From 3eddfd4117b6a5a302e8404781a4441a5099b23c Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Fri, 2 Aug 2024 22:40:47 -0400 Subject: [PATCH] update spectator-py docs for the 1.0 release (#179) --- docs/spectator/lang/go/usage.md | 13 +- docs/spectator/lang/py/meters/age-gauge.md | 18 +- docs/spectator/lang/py/meters/counter.md | 24 +- docs/spectator/lang/py/meters/dist-summary.md | 8 +- docs/spectator/lang/py/meters/gauge.md | 16 +- docs/spectator/lang/py/meters/max-gauge.md | 8 +- docs/spectator/lang/py/meters/mono-counter.md | 13 - .../lang/py/meters/monotonic-counter-uint.md | 18 ++ .../lang/py/meters/monotonic-counter.md | 17 ++ ...-summary.md => percentile-dist-summary.md} | 10 +- .../{pct-timer.md => percentile-timer.md} | 22 +- docs/spectator/lang/py/meters/timer.md | 25 +- docs/spectator/lang/py/migrations.md | 106 ++++++- docs/spectator/lang/py/usage.md | 265 ++++++++++++------ mkdocs.yml | 27 +- 15 files changed, 422 insertions(+), 168 deletions(-) delete mode 100644 docs/spectator/lang/py/meters/mono-counter.md create mode 100644 docs/spectator/lang/py/meters/monotonic-counter-uint.md create mode 100644 docs/spectator/lang/py/meters/monotonic-counter.md rename docs/spectator/lang/py/meters/{pct-dist-summary.md => percentile-dist-summary.md} (62%) rename docs/spectator/lang/py/meters/{pct-timer.md => percentile-timer.md} (50%) diff --git a/docs/spectator/lang/go/usage.md b/docs/spectator/lang/go/usage.md index 2b87c56c..682e9112 100644 --- a/docs/spectator/lang/go/usage.md +++ b/docs/spectator/lang/go/usage.md @@ -96,14 +96,15 @@ func main() { ## Logging -Logging is implemented with the standard Golang [slog package](https://pkg.go.dev/log/slog). The logger defines interfaces -for [Debugf, Infof, and Errorf]. There are useful messages implemented at the Debug level which can -help diagnose the metric publishing workflow. The logger can be overridden by providing one as the -third parameter of the `Config` constructor. +Logging is implemented with the standard Golang [slog package](https://pkg.go.dev/log/slog). The +logger defines interfaces for [Debugf, Infof, and Errorf]. There are useful messages implemented at +the Debug level which can help diagnose the metric publishing workflow. The logger can be overridden +by providing one as the third parameter of the `Config` constructor. [Debugf, Infof, and Errorf]: https://github.com/Netflix/spectator-go/blob/main/spectator/logger/logger.go ## Runtime Metrics -Use [spectator-go-runtime-metrics](https://github.com/Netflix/spectator-go-runtime-metrics). Follow instructions -in the [README](https://github.com/Netflix/spectator-go-runtime-metrics) to enable collection. +Use [spectator-go-runtime-metrics](https://github.com/Netflix/spectator-go-runtime-metrics). Follow +instructions in the [README](https://github.com/Netflix/spectator-go-runtime-metrics) to enable +collection. diff --git a/docs/spectator/lang/py/meters/age-gauge.md b/docs/spectator/lang/py/meters/age-gauge.md index 418d4894..ade5e2cb 100644 --- a/docs/spectator/lang/py/meters/age-gauge.md +++ b/docs/spectator/lang/py/meters/age-gauge.md @@ -7,17 +7,25 @@ Time Since Last Success alerting pattern. To set a specific time as the last success: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.age_gauge("time.sinceLastSuccess").set(1611081000) +registry = Registry() +registry.age_gauge("time.sinceLastSuccess").set(1611081000) + +last_success = registry.new_id("time.sinceLastSuccess") +registry.age_gauge_with_id(last_success).set(1611081000) ``` To set `now()` as the last success: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry + +registry = Registry() +registry.age_gauge("time.sinceLastSuccess").now() -GlobalRegistry.age_gauge("time.sinceLastSuccess").set(0) +last_success = registry.new_id("time.sinceLastSuccess") +registry.age_gauge_with_id(last_success).now() ``` By default, a maximum of `1000` Age Gauges are allowed per `spectatord` process, because there is no @@ -28,7 +36,7 @@ Since Age Gauges are long-lived entities that reside in the memory of the Specta you need to delete and re-create them for any reason, then you can use the [SpectatorD admin server] to accomplish this task. You can delete all Age Gauges or a single Age Gauge. -Example: +**Example:** ``` curl -X DELETE \ diff --git a/docs/spectator/lang/py/meters/counter.md b/docs/spectator/lang/py/meters/counter.md index efa83c47..aba31da8 100644 --- a/docs/spectator/lang/py/meters/counter.md +++ b/docs/spectator/lang/py/meters/counter.md @@ -1,22 +1,30 @@ -A Counter is used to measure the rate at which an event is occurring. Considering an API -endpoint, a Counter could be used to measure the rate at which it is being accessed. +A Counter is used to measure the rate at which an event is occurring. Considering an API endpoint, +a Counter could be used to measure the rate at which it is being accessed. -Counters are reported to the backend as a rate-per-second. In Atlas, the `:per-step` operator -can be used to convert them back into a value-per-step on a graph. +Counters are reported to the backend as a rate-per-second. In Atlas, the `:per-step` operator can +be used to convert them back into a value-per-step on a graph. Call `increment()` when an event occurs: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.counter("server.numRequests").increment() +registry = Registry() +registry.counter("server.numRequests").increment() + +num_requests = registry.new_id("server.numRequests") +registry.counter_with_id(num_requests).increment() ``` You can also pass a value to `increment()`. This is useful when a collection of events happens together: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry + +registry = Registry() +registry.counter("queue.itemsAdded").increment(10) -GlobalRegistry.counter("queue.itemsAdded").increment(10) +num_requests = registry.new_id("server.numRequests") +registry.counter_with_id(num_requests).increment(10) ``` diff --git a/docs/spectator/lang/py/meters/dist-summary.md b/docs/spectator/lang/py/meters/dist-summary.md index f644a003..e97f5b87 100644 --- a/docs/spectator/lang/py/meters/dist-summary.md +++ b/docs/spectator/lang/py/meters/dist-summary.md @@ -9,7 +9,11 @@ This means that a `4K` tick label will represent 4 kilobytes, rather than 4 kilo Call `record()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.distribution_summary("server.requestSize").record(10) +registry = Registry() +registry.distribution_summary("server.requestSize").record(10) + +request_size = registry.new_id("server.requestSize") +registry.distribution_summary_with_id(request_size).record(10) ``` diff --git a/docs/spectator/lang/py/meters/gauge.md b/docs/spectator/lang/py/meters/gauge.md index dca3e7e7..1ce10636 100644 --- a/docs/spectator/lang/py/meters/gauge.md +++ b/docs/spectator/lang/py/meters/gauge.md @@ -9,9 +9,13 @@ higher or lower at some point during interval, but that is not known. Call `set()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.gauge("server.queueSize").set(10) +registry = Registry() +registry.gauge("server.queueSize").set(10) + +queue_size = registry.new_id("server.queueSize") +registry.gauge_with_id(queue_size).set(10) ``` Gauges will report the last set value for 15 minutes. This done so that updates to the values do @@ -19,7 +23,11 @@ not need to be collected on a tight 1-minute schedule to ensure that Atlas shows graphs. A custom TTL may be configured for gauges. SpectatorD enforces a minimum TTL of 5 seconds. ```python -from spectator import GlobalRegistry +from spectator.registry import Registry + +registry = Registry() +registry.gauge("server.queueSize", ttl_seconds=120).set(10) -GlobalRegistry.gauge("server.queueSize", ttl_seconds=120).set(10) +queue_size = registry.new_id("server.queueSize") +registry.gauge_with_id(queue_size, ttl_seconds=120).set(10) ``` diff --git a/docs/spectator/lang/py/meters/max-gauge.md b/docs/spectator/lang/py/meters/max-gauge.md index 8b01e772..010d5367 100644 --- a/docs/spectator/lang/py/meters/max-gauge.md +++ b/docs/spectator/lang/py/meters/max-gauge.md @@ -6,7 +6,11 @@ standard Gauges, Max Gauges do not continue to report to the backend, and there Call `set()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.max_gauge("server.queueSize").set(10) +registry = Registry() +registry.max_gauge("server.queueSize").set(10) + +queue_size = registry.new_id("server.queueSize") +registry.max_gauge_with_id(queue_size).set(10) ``` diff --git a/docs/spectator/lang/py/meters/mono-counter.md b/docs/spectator/lang/py/meters/mono-counter.md deleted file mode 100644 index bb90b23c..00000000 --- a/docs/spectator/lang/py/meters/mono-counter.md +++ /dev/null @@ -1,13 +0,0 @@ -A Monotonic Counter is used to measure the rate at which an event is occurring, when the source -data is a monotonically increasing number. A minimum of two samples must be sent, in order to -calculate a delta value and report it to the backend as a rate-per-second. A variety of networking -metrics may be reported monotonically, and this metric type provides a convenient means of recording -these values, at the expense of a slower time-to-first metric. - -Call `set()` when an event occurs: - -```python -from spectator import GlobalRegistry - -GlobalRegistry.monotonic_counter("iface.bytes").set(10) -``` diff --git a/docs/spectator/lang/py/meters/monotonic-counter-uint.md b/docs/spectator/lang/py/meters/monotonic-counter-uint.md new file mode 100644 index 00000000..4eacf9ce --- /dev/null +++ b/docs/spectator/lang/py/meters/monotonic-counter-uint.md @@ -0,0 +1,18 @@ +A Monotonic Counter (uint64) is used to measure the rate at which an event is occurring, when the +source data is a monotonically increasing number. A minimum of two samples must be sent, in order to +calculate a delta value and report it to the backend as a rate-per-second. A variety of networking +metrics may be reported monotonically, and this metric type provides a convenient means of recording +these values, at the expense of a slower time-to-first metric. + +Call `set()` when an event occurs: + +```python +from ctypes import c_uint64 +from spectator.registry import Registry + +registry = Registry() +registry.monotonic_counter_uint("iface.bytes").set(c_uint64(1)) + +iface_bytes = registry.new_id("iface.bytes") +registry.monotonic_counter_uint_with_id(iface_bytes).set(c_uint64(1)) +``` diff --git a/docs/spectator/lang/py/meters/monotonic-counter.md b/docs/spectator/lang/py/meters/monotonic-counter.md new file mode 100644 index 00000000..f14835e3 --- /dev/null +++ b/docs/spectator/lang/py/meters/monotonic-counter.md @@ -0,0 +1,17 @@ +A Monotonic Counter (float) is used to measure the rate at which an event is occurring, when the +source data is a monotonically increasing number. A minimum of two samples must be sent, in order to +calculate a delta value and report it to the backend as a rate-per-second. A variety of networking +metrics may be reported monotonically, and this metric type provides a convenient means of recording +these values, at the expense of a slower time-to-first metric. + +Call `set()` when an event occurs: + +```python +from spectator.registry import Registry + +registry = Registry() +registry.monotonic_counter("iface.bytes").set(10) + +iface_bytes = registry.new_id("iface.bytes") +registry.monotonic_counter_with_id(iface_bytes).set(10) +``` diff --git a/docs/spectator/lang/py/meters/pct-dist-summary.md b/docs/spectator/lang/py/meters/percentile-dist-summary.md similarity index 62% rename from docs/spectator/lang/py/meters/pct-dist-summary.md rename to docs/spectator/lang/py/meters/percentile-dist-summary.md index 83831df9..0b67b888 100644 --- a/docs/spectator/lang/py/meters/pct-dist-summary.md +++ b/docs/spectator/lang/py/meters/percentile-dist-summary.md @@ -1,5 +1,5 @@ The value tracks the distribution of events, with percentile estimates. It is similar to a -Percentile Timer, but more general, because the size does not have to be a period of time. +`PercentileTimer`, but more general, because the size does not have to be a period of time. For example, it can be used to measure the payload sizes of requests hitting a server or the number of records returned from a query. @@ -11,7 +11,11 @@ added to Percentile Distribution Summaries and ensure that they have a small bou Call `record()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.pct_distribution_summary("server.requestSize").record(10) +registry = Registry() +registry.pct_distribution_summary("server.requestSize").record(10) + +request_size = registry.new_id("server.requestSize") +registry.pct_distribution_summary_with_id(request_size).record(10) ``` diff --git a/docs/spectator/lang/py/meters/pct-timer.md b/docs/spectator/lang/py/meters/percentile-timer.md similarity index 50% rename from docs/spectator/lang/py/meters/pct-timer.md rename to docs/spectator/lang/py/meters/percentile-timer.md index 2bf2940a..184d3d64 100644 --- a/docs/spectator/lang/py/meters/pct-timer.md +++ b/docs/spectator/lang/py/meters/percentile-timer.md @@ -11,20 +11,28 @@ Timers and ensure that they have a small bounded cardinality. Call `record()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.pct_timer("server.requestLatency").record(0.01) +registry = Registry() +registry.pct_timer("server.requestLatency").record(0.01) + +request_latency = registry.new_id("server.requestLatency") +registry.pct_timer_with_id(request_latency).record(0.01) ``` -A `stopwatch()` method is available which may be used as a [Context Manager](https://docs.python.org/3/reference/datamodel.html#context-managers) -to automatically record the number of seconds that have elapsed while executing a block of code: +A `StopWatch` class is available, which may be used as a [Context Manager] to automatically record +the number of seconds that have elapsed while executing a block of code: ```python import time -from spectator import GlobalRegistry +from spectator.registry import Registry +from spectator.stopwatch import StopWatch -t = GlobalRegistry.pct_timer("thread.sleep") +registry = Registry() +thread_sleep = registry.pct_timer("thread.sleep") -with t.stopwatch(): +with StopWatch(thread_sleep): time.sleep(5) ``` + +[Context Manager]: https://docs.python.org/3/reference/datamodel.html#context-managers diff --git a/docs/spectator/lang/py/meters/timer.md b/docs/spectator/lang/py/meters/timer.md index 8232a1f0..ee4212c9 100644 --- a/docs/spectator/lang/py/meters/timer.md +++ b/docs/spectator/lang/py/meters/timer.md @@ -3,29 +3,28 @@ A Timer is used to measure how long (in seconds) some event is taking. Call `record()` with a value: ```python -from spectator import GlobalRegistry +from spectator.registry import Registry -GlobalRegistry.timer("server.requestLatency").record(0.01) +registry = Registry() +registry.timer("server.requestLatency").record(0.01) + +request_latency = registry.new_id("server.requestLatency") +registry.timer_with_id(request_latency).record(0.01) ``` -A `stopwatch()` method is available which may be used as a [Context Manager] to automatically record +A `StopWatch` class is available, which may be used as a [Context Manager] to automatically record the number of seconds that have elapsed while executing a block of code: ```python import time -from spectator import GlobalRegistry +from spectator.registry import Registry +from spectator.stopwatch import StopWatch -t = GlobalRegistry.timer("thread.sleep") +registry = Registry() +thread_sleep = registry.timer("thread.sleep") -with t.stopwatch(): +with StopWatch(thread_sleep): time.sleep(5) ``` -Internally, Timers will keep track of the following statistics as they are used: - -* `count` -* `totalTime` -* `totalOfSquares` -* `max` - [Context Manager]: https://docs.python.org/3/reference/datamodel.html#context-managers diff --git a/docs/spectator/lang/py/migrations.md b/docs/spectator/lang/py/migrations.md index 302415c6..24a6165d 100644 --- a/docs/spectator/lang/py/migrations.md +++ b/docs/spectator/lang/py/migrations.md @@ -1,4 +1,108 @@ -## Migrating from 0.1.X to 0.2.X +## Migrating from 0.2 to 1.0 + +Version 1.0 consists of a major rewrite that cleans up and simplifies the `spectator-py` thin client +API. It is designed to send metrics through [spectatord](https://github.com/Netflix-Skunkworks/spectatord). +As a result, some functionality has been moved to other modules, or removed. + +### New + +#### Config + +* Replace the `SidecarConfig` with `Config`, and simplify usage. +* The `location` configuration is clarified, with a default set to the `spectatord` UDP port, and a +new option for picking the default Unix Domain Socket for `spectatord`. +* The `extra_common_tags` concept is clarified. Any extra common tags provided through the `Config` +object are merged with two process-specific tags that may be present in environment variables. +* Any `MeterId` or `Meter` objects created through `Registry` methods will contain these extra tags. + +#### Meters + +* The `AgeGauge` meter added a `now()` method, which sets `0` as the value, so you do not need to +remember this special value. +* Add `MonotonicCounterUint` with a `c_uint64` data type, to support `uint64` data types. These are +not commonly encountered, as they usually only show up in networking metrics, such as bytes/sec in +high-volume contexts. When you need it, you need it, else wise, it can be ignored. +* The `MonotonicCounter` with a `float` data type continues to exist, for the more common use case. +* Note that monotonic counters are convenience meter types provided by `spectatord`, because they +help you avoid the work of tracking previous values and calculating deltas. + +#### Registry + +* Add a `new_id()` method and `*_with_id()` methods for all meter types, to support more complex +tag operations related to `MeterId` objects. This follows the way they work in the other clients. + +### Moved + +#### Meters + +* Separate classes for each `Meter` type. Relocated to a new module, `spectator.meter`. + +#### StopWatch + +* The `StopWatch` context manager is no longer part of the `Timer` class; it is now a standalone +class. It has been preserved, because it continues to fulfill the purpose of simplifying how `Timer` +and `PercentileTimer` meters record their values after exiting a block of code, and there are a few +uses of this class across the organization. + +**Before:** + +```python +import time +from spectator import GlobalRegistry + +server_latency = GlobalRegistry.pct_timer("serverLatency") + +with server_latency.stopwatch(): + time.sleep(5) +``` + +**After:** + +```python +import time +from spectator.registry import Registry +from spectator.stopwatch import StopWatch + +registry = Registry() +server_latency = registry.pct_timer("serverLatency") + +with StopWatch(server_latency): + time.sleep(5) +``` + +#### Writers + +* Separate classes for each `Writer` type. Relocated to a new module, `spectator.writer`. + +### Removed + +* All remnants of the previous thick-client API. + +### Deprecated + +* The `GlobalRegistry` is a hold-over from the thick-client version of this library, but it has been +maintained to help minimize the amount of code change that application owners need to implement +when adopting the thin-client version of the library. Replace with direct use of `Registry`. +* There are no plans to remove the `GlobalRegistry`, until we know that all uses have been removed. + +**Before:** + +```python +from spectator import GlobalRegistry + +GlobalRegistry.gauge("server.queueSize", ttl_seconds=120).set(10) +``` + +**After:** + +```python +from spectator.registry import Registry + +registry = Registry() +registry.gauge("server.queueSize", ttl_seconds=120).set(10) +``` + +## Migrating from 0.1 to 0.2 * This library no longer publishes directly to the Atlas backends. It now publishes to the [SpectatorD] sidecar which is bundled with all standard AMIs and containers. If you must diff --git a/docs/spectator/lang/py/usage.md b/docs/spectator/lang/py/usage.md index 5d89f31b..d9059a35 100644 --- a/docs/spectator/lang/py/usage.md +++ b/docs/spectator/lang/py/usage.md @@ -2,50 +2,126 @@ Python thin-client [metrics library] for use with [Atlas] and [SpectatorD]. -Supports Python >= 3.5. This version is chosen as the baseline, because it is the oldest system -Python available in our operating environments. - [metrics library]: https://github.com/Netflix/spectator-py [Atlas]: ../../../overview.md [SpectatorD]: ../../agent/usage.md +## Supported Python Versions + +This library currently targets the Python >= 3.8. + ## Installing Install this library for your project as follows: ```shell -pip3 install netflix-spectator-py +pip install netflix-spectator-py ``` -Publishing metrics requires a [SpectatorD] process running on your instance. +## Instrumenting Code + +```python +import logging + +from flask import Flask, request, Response +from flask.logging import default_handler +from spectator.config import Config +from spectator.registry import Registry +from spectator.stopwatch import StopWatch + +root_logger = logging.getLogger() +root_logger.setLevel(logging.DEBUG) +root_logger.addHandler(default_handler) + +config = Config(location="none", extra_common_tags={"nf.platform": "my_platform"}) +registry = Registry(config) + +request_count_id = registry.new_id("server.requestCount", {"version": "v1"}) +request_latency = registry.timer("server.requestLatency") +response_size = registry.distribution_summary("server.responseSize") + +app = Flask(__name__) + +@app.route("/") +def root(): + return Response("Usage: /api/v1/play?country=foo&title=bar") +@app.route("/api/v1/play", methods=["GET", "POST"]) +def play(): + if request.method == "GET": + with StopWatch(request_latency): + status_code = 200 + country = request.args.get("country", default="none") + title = request.args.get("title", default="none") + + tags = {"country": country, "title": title, "status": str(status_code)} + request_count_with_tags = request_count_id.with_tags(tags) + counter = registry.counter_with_id(request_count_with_tags) + counter.increment() + + message = f"requested play for country={country} title={title}" + response_size.record(len(message)) + return Response(message, status=status_code) + else: + status_code = 405 + + tags = {"status": str(status_code)} + request_count_with_tags = request_count_id.with_tags(tags) + counter = registry.counter_with_id(request_count_with_tags) + counter.increment() + + return Response("unsupported request method", status=status_code) +``` + +Save this snippet as `app.py`, then `flask --app app run`. + ## Importing ### Standard Usage -Importing the `GlobalRegistry` instantiates a `Registry` with a default configuration that applies -process-specific common tags based on environment variables and opens a socket to the [SpectatorD] -agent. The remainder of the instance-specific common tags are provided by [SpectatorD]. +Instantiate a `Registry` object, with either a default or custom `Config`, and use it to create and +manage `MeterId` and `Meter` objects. ```python -from spectator import GlobalRegistry +from spectator.registry import Registry + +registry = Registry() +registry.counter("server.requestCount").increment() ``` -Once the `GlobalRegistry` is imported, it is used to create and manage Meters. +### Legacy Usage + +The `GlobalRegistry` concept is a hold-over from the thick-client version of this library, but it +has been maintained to help minimize the amount of code change that application owners need to +implement when adopting the thin client version of the library. It existed as a concept in the +thick client because it was stateful, and required starting background threads. The thin client +version is stateless. + +Importing the `GlobalRegistry` instantiates a `Registry` with a default `Config` that applies +process-specific common tags based on environment variables and opens a UDP socket to the local +[SpectatorD] agent. The remainder of the instance-specific common tags are provided by [SpectatorD]. +Once imported, the `GlobalRegistry` can be used to create and manage Meters. + +```python +from spectator import GlobalRegistry + +GlobalRegistry.counter("server.requestCount").increment() +``` ### Logging This package provides the following loggers: -* `spectator.MeterId` -* `spectator.SidecarWriter` - -The `MeterId` logger is used to report invalid meters which have not-a-str tag keys or values. +* `spectator.meter.meter_id`, which reports invalid tags at WARNING level. +* `spectator.registry`, which reports Registry status messages at INFO level, and errors closing +writers at ERROR level. +* `spectator.writer`, which reports the protocol lines written at DEBUG level, and writing errors +at ERROR level. -When troubleshooting metrics collection and reporting, you should set the `SidecarWriter` logging -to the `DEBUG` level, before the first metric is recorded. For example: +When troubleshooting metrics collection and reporting, you should set the `spectator.meter.meter_id` +logger to `DEBUG` level, before the first metric is recorded. For example: ```python import logging @@ -56,32 +132,39 @@ logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(thread)d - %(message)s' ) -logging.getLogger('spectator.SidecarWriter').setLevel(logging.DEBUG) +logging.getLogger('spectator.meter.meter_id').setLevel(logging.DEBUG) ``` -There is approximately a 10% performance penalty in UDP write performance when debug logging is -enabled. It may be more, depending on the exact logging configuration (i.e. flushing to slow disk). - -## Working with IDs +## Working with MeterId Objects -The IDs used for identifying a meter in the `GlobalRegistry` consist of a name and a set of tags. -IDs will be consumed by users many times after the data has been reported, so they should be -chosen thoughtfully, while considering how they will be used. See the [naming conventions] page -for general guidelines. +Each metric stored in Atlas is uniquely identified by the combination of the name and the tags +associated with it. In `spectator-py`, this data is represented with `MeterId` objects, created +by the `Registry`. The `new_id()` method returns new `MeterId` objects, which have extra common +tags applied, and which can be further customized by calling the `with_tag()` and `with_tags()` +methods. Each `MeterId` will create and store a validated subset of the `spectatord` protocol line +to be written for each `Meter`, when it is instantiated. `MeterId` objects are immutable, so they +can be freely passed around and used concurrently. Manipulating the tags with the provided methods +will create new `MeterId` objects, to assist with maintaining immutability. -IDs are immutable, so they can be freely passed around and used in a concurrent context. Tags can -be added to an ID when it is created, to track the dimensionality of the metric. **All tag keys -and values must be strings.** For example, if you want to keep track of the number of successful -requests, you must cast integers to strings. +Note that **all tag keys and values must be strings.** For example, if you want to keep track of the +number of successful requests, then you must cast integers to strings. The `MeterId` class will +validate these values, dropping or changing any that are not valid, and reporting a warning log. ```python -from spectator import GlobalRegistry +from spectator.registry import Registry + +registry = Registry() +registry.counter("server.numRequests", {"statusCode": str(200)}).increment() -requests_id = GlobalRegistry.counter("server.numRequests", {"statusCode": str(200)}) -requests_id.increment() +num_requests_id = registry.new_id("server.numRequests", {"statusCode": str(200)}) +registry.counter_with_id(num_requests_id).increment() ``` -[naming conventions]: https://netflix.github.io/atlas-docs/concepts/naming/ +Atlas metrics will be consumed by users many times after the data has been reported, so they should +be chosen thoughtfully, while considering how they will be used. See the [naming conventions] page +for general guidelines on metrics naming and restrictions. + +[naming conventions]: ../../../concepts/naming.md ## Meter Types @@ -90,20 +173,21 @@ requests_id.increment() * [Distribution Summary](./meters/dist-summary.md) * [Gauge](./meters/gauge.md) * [Max Gauge](./meters/max-gauge.md) -* [Monotonic Counter](./meters/mono-counter.md) -* [Percentile Distribution Summary](./meters/pct-dist-summary.md) -* [Percentile Timer](./meters/pct-timer.md) +* [Monotonic Counter](./meters/monotonic-counter.md) +* [Monotonic Counter Uint](./meters/monotonic-counter-uint.md) +* [Percentile Distribution Summary](./meters/percentile-dist-summary.md) +* [Percentile Timer](./meters/percentile-timer.md) * [Timer](./meters/timer.md) ## asyncio Support -The `GlobalRegistry` provides a `UdpWriter` implementation of the `SidecarWriter` by default. UDP -is a non-blocking, unordered and connectionless protocol, which is ideal for communicating with a -local [SpectatorD] process in a variety of circumstances. The `UdpWriter` should be used in asyncio +The `Registry` provides a `UdpWriter` by default. UDP is a non-blocking, unordered and +connectionless protocol, which is ideal for communicating with a local [SpectatorD] +process in a variety of circumstances. The `UdpWriter` should be used in asyncio applications. -The `PrintWriter` implementation, which can be used to communicate with the [SpectatorD] Unix domain -socket, does not offer asyncio support at this time. +The `FileWriter` implementation, which can be used to communicate with the [SpectatorD] Unix domain +socket, for slightly higher performance, does not offer asyncio support at this time. ## IPv6 Support @@ -111,98 +195,97 @@ By default, [SpectatorD] will listen on `IPv6 UDP *:1234`, without setting the ` flag. On dual-stacked systems, this means that it will receive packets from both IPv4 and IPv6, and the IPv4 addresses will show up on the server as IPv4-mapped IPv6 addresses. -By default, the `GlobalRegistry` will write UDP packets to `127.0.0.1:1234`, which will allow -for communication with [SpectatorD] on dual-stacked systems. +By default, the `UdpWriter` will send UDP packets to `127.0.0.1:1234`, which will allow for +communication with [SpectatorD] on dual-stacked systems. On IPv6-only systems, it may be necessary to change the default configuration using one of the following methods: -* Configure the following environment variable, which will override the default configuration of -the `GlobalRegistry`: +* Configure the following environment variable, which will override the default location `Config` +in the `Registry`: - export SPECTATOR_OUTPUT_LOCATION="udp://[::1]:1234" +```shell +export SPECTATOR_OUTPUT_LOCATION="udp://[::1]:1234" +``` -* Configure a custom Registry, instead of using the `GlobalRegistry`: +* Provide a custom `Config` for the `Registry`: - from spectator import Registry - from spectator.sidecarconfig import SidecarConfig - - r = Registry(config=SidecarConfig({"sidecar.output-location": "udp://[::1]:1234"})) - r.counter("test").increment() +```python +from spectator.config import Config +from spectator.registry import Registry + +config = Config(location="udp://[::1]:1234") +registry = Registry(config) +registry.counter("server.numRequests").increment() +``` ## Writing Tests -To write tests against this library, instantiate a test instance of the Registry and configure it -to use the [MemoryWriter](https://github.com/Netflix/spectator-py/blob/main/spectator/sidecarwriter.py#L63-L80), -which stores all updates in a List. Use the `writer()` method on the Registry to access the writer, -then inspect the `last_line()` or `get()` all messages to verify your metrics updates. +To write tests against this library, instantiate an instance of the `Registry` and provide a `Config` +that selects the [MemoryWriter](https://github.com/Netflix/spectator-py/blob/main/spectator/writer/memory_writer.py). +This `Writer` stores all updates in a `List[str]`. Use the `writer()` method on the `Registry` to +access the writer, then inspect the `last_line()` or `get()` all messages to verify your metrics +updates. ```python import unittest -from spectator import Registry -from spectator.sidecarconfig import SidecarConfig +from spectator.config import Config +from spectator.registry import Registry class MetricsTest(unittest.TestCase): def test_counter(self): - r = Registry(config=SidecarConfig({"sidecar.output-location": "memory"})) + r = Registry(Config("memory")) - c = r.counter("test") + c = r.counter("server.numRequests") self.assertTrue(r.writer().is_empty()) c.increment() - self.assertEqual("c:test:1", r.writer().last_line()) + self.assertEqual("c:server.numRequests:1", r.writer().last_line()) ``` -If you need to override the default output location (udp) of the `GlobalRegistry`, then you can -set a `SPECTATOR_OUTPUT_LOCATION` environment variable to one of the following values supported -by the `SidecarConfig` class: +### Overriding Output Location + +If you need to override the default output location (UDP) of the `Registry`, then you can set a +`Config` class location to one of the following supported values: -* `none` - Disable output. +* `none` - Disable output. * `memory` - Write to memory. -* `stdout` - Write to standard out for the process. * `stderr` - Write to standard error for the process. -* `file://$path_to_file` - Write to a file (e.g. `file:///tmp/foo/bar`). -* `udp://$host:$port` - Write to a UDP socket. +* `stdout` - Write to standard out for the process. +* `udp` - Write to the default UDP port for spectatord. +* `unix` - Write to the default unix datagram socket for spectatord. +* `file://$path_to_file` - Write to a custom file (e.g. `file:///tmp/foo/bar`). +* `udp://$host:$port` - Write to a custom UDP socket. -If you want to disable metrics publishing from the `GlobalRegistry`, then you can set: +The `SPECTATOR_OUTPUT_LOCATION` environment variable accepts the same values, and can be used to +override the value provided to the `Config` class, which may be useful in CI/CD contexts. For +example, if you want to disable metrics publishing from the `Registry`, then you can set: ```shell export SPECTATOR_OUTPUT_LOCATION=none ``` -If you want to validate the metrics that will be published through the `GlobalRegistry` -in tests, then you can set: +### Protocol Parser -```shell -export SPECTATOR_OUTPUT_LOCATION=memory -``` - -The `MemoryWriter` subclass offers a few methods to inspect the values that it captures: - -* `clear()` - Delete the contents of the internal list. -* `get()` - Return the internal list. -* `is_empty()` - Is the internal list empty? -* `last_line()` - Return the last element of the internal list. - -Lastly, a [SpectatorD] line protocol parser is available, which is intended to be used for validating -the results captured by a `MemoryWriter`. It may be used as follows: +A [SpectatorD] line protocol parser is available, which ca be used for validating the results +captured by a `MemoryWriter`. ```python import unittest -from spectator.counter import Counter -from spectator.protocolparser import parse_protocol_line - +from spectator.meter.counter import Counter +from spectator.protocol_parser import get_meter_class, parse_protocol_line class ProtocolParserTest(unittest.TestCase): def test_parse_counter_with_multiple_tags(self): - meter_class, meter_id, value = parse_protocol_line("c:test,foo=bar,baz=quux:1") - self.assertEqual(Counter, meter_class) - self.assertEqual("test", meter_id.name) - self.assertEqual({"foo": "bar", "baz": "quux"}, meter_id.tags()) + symbol, id, value = parse_protocol_line("c:counter,foo=bar,baz=quux:1") + self.assertEqual("c", symbol) + self.assertEqual(Counter, get_meter_class(symbol)) + self.assertEqual("counter", id.name()) + self.assertEqual({"foo": "bar", "baz": "quux"}, id.tags()) self.assertEqual("1", value) ``` diff --git a/mkdocs.yml b/mkdocs.yml index 3abf5a98..94fcca19 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -268,23 +268,24 @@ nav: - Heap Space: spectator/lang/nodejs/ext/nodejs-heapspace.md - Memory: spectator/lang/nodejs/ext/nodejs-memory.md - Meters: - - Counters: spectator/lang/nodejs/meters/counter.md - - Distribution Summaries: spectator/lang/nodejs/meters/dist-summary.md - - Gauges: spectator/lang/nodejs/meters/gauge.md - - Percentile Timers: spectator/lang/nodejs/meters/percentile-timer.md - - Timers: spectator/lang/nodejs/meters/timer.md + - Counter: spectator/lang/nodejs/meters/counter.md + - Distribution Summary: spectator/lang/nodejs/meters/dist-summary.md + - Gauge: spectator/lang/nodejs/meters/gauge.md + - Percentile Timer: spectator/lang/nodejs/meters/percentile-timer.md + - Timer: spectator/lang/nodejs/meters/timer.md - Python: - Usage: spectator/lang/py/usage.md - Meters: - - Age Gauges: spectator/lang/py/meters/age-gauge.md - - Counters: spectator/lang/py/meters/counter.md - - Distribution Summaries: spectator/lang/py/meters/dist-summary.md + - Age Gauge: spectator/lang/py/meters/age-gauge.md + - Counter: spectator/lang/py/meters/counter.md + - Distribution Summary: spectator/lang/py/meters/dist-summary.md - Gauges: spectator/lang/py/meters/gauge.md - - Max Gauges: spectator/lang/py/meters/max-gauge.md - - Monotonic Counters: spectator/lang/py/meters/mono-counter.md - - Percentile Distribution Summaries: spectator/lang/py/meters/pct-dist-summary.md - - Percentile Timers: spectator/lang/py/meters/pct-timer.md - - Timers: spectator/lang/py/meters/timer.md + - Max Gauge: spectator/lang/py/meters/max-gauge.md + - Monotonic Counter: spectator/lang/py/meters/monotonic-counter.md + - Monotonic Counter Uint: spectator/lang/py/meters/monotonic-counter-uint.md + - Percentile Distribution Summary: spectator/lang/py/meters/percentile-dist-summary.md + - Percentile Timer: spectator/lang/py/meters/percentile-timer.md + - Timer: spectator/lang/py/meters/timer.md - Migrations: spectator/lang/py/migrations.md - Ruby: - Deprecated: spectator/lang/rb/deprecated.md