set up opentelemetry metrics for new infra (and now)

Cargo.toml

@@ -41,6 +41,10 @@ comrak = { version = "0.47.0", default-features = false }
 syntect = { version = "5.0.0", default-features = false, features = ["parsing", "html", "dump-load", "regex-onig"] }
 toml = "0.9.2"
 prometheus = { version = "0.14.0", default-features = false }
+opentelemetry = "0.31.0"
+opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
+opentelemetry-resource-detectors = "0.10.0"
+opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
 rustwide = { version = "0.20.0", features = ["unstable-toolchain-ci", "unstable"] }
 mime_guess = "2"
 zstd = "0.13.0"
@@ -123,6 +127,7 @@ rand = "0.9"
 mockito = "1.0.2"
 test-case = "3.0.0"
 tower = { version = "0.5.1", features = ["util"] }
+opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio", "testing"] }
 aws-smithy-types = "1.0.1"
 aws-smithy-runtime = {version = "1.0.1", features = ["client", "test-util"]}
 aws-smithy-http = "0.62.0"

README.md

@@ -198,6 +198,22 @@ $ just compose-down
 $ just compose-down-and-wipe
 ```
 
+#### testing opentelemetry metrics
+
+When you add or update any metrics you might want to test them. While there is
+a way to check metric in unit-tests (see `TestEnvironment::collected_metrics`),
+you might also want to test manually.
+
+We have set up a small docker-compose service (`opentelemetry`) you can start up
+via `docker compose up opentelemetry`. This start up a local instance of
+the [opentelemetry collector
+contrib](https://hub.docker.com/r/otel/opentelemetry-collector-contrib) image,
+configured for debug-logging.
+
+After configuring your local environment for `OTEL_EXPORTER_OTLP_ENDPOINT` => `http://localhost:4317`
+(either in `.env` or `.docker.env`, depending on how you run the webserver), you
+can see any metrics you report and how they are exported to your collector.
+
 #### FAQ
 
 ##### I see the error `standard_init_linux.go:211: exec user process caused "no such file or directory"` when I use docker-compose.

docker-compose.yml

@@ -22,6 +22,7 @@
 #
 # optional profile: `metrics`:
 # * `prometheus` -> configured prometheus instance
+# * `opentelemetry` -> a debug opentelemetry receiver
 #
 # optional profile: `full`: all of the above.
 #
@@ -296,6 +297,23 @@ services:
             - metrics
             - full
 
+    opentelemetry:
+        build:
+            context: ./dockerfiles
+            dockerfile: ./Dockerfile-opentelemetry
+            <<: *docker-cache
+        ports:
+            - "127.0.0.1:4317:4317"
+        healthcheck:
+            <<: *healthcheck-interval
+            test: curl --silent --fail http://localhost:13133/health
+
+        profiles:
+            # we rarely need to test with actual prometheus, so always running
+            # it is a waste.
+            - metrics
+            - full
+
     gui_tests:
         platform: "linux/amd64"
         build:

dockerfiles/Dockerfile-opentelemetry

@@ -0,0 +1,2 @@
+FROM otel/opentelemetry-collector-contrib:0.139.0 AS base
+COPY collector-config-dev.yaml /etc/otelcol-contrib/config.yaml

dockerfiles/collector-config-dev.yaml

@@ -0,0 +1,30 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+exporters:
+  debug:
+    # the debug exporter will just print everything to the console
+    verbosity: detailed
+
+extensions:
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/09d7cdeb075f51bf90a604c95b106521bba9962e/extension/healthcheckextension
+  health_check:
+    endpoint: "0.0.0.0:13133"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: []
+      exporters: [debug]
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [debug]
+    logs:
+      receivers: [otlp]
+      processors: []
+      exporters: [debug]

src/bin/cratesfyi.rs

@@ -5,8 +5,9 @@ use docs_rs::{
     db::{self, CrateId, Overrides, ReleaseId, add_path_into_database, types::version::Version},
     start_background_metrics_webserver, start_web_server,
     utils::{
-        ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities,
-        queue_builder, remove_crate_priority, set_config, set_crate_priority,
+        ConfigName, daemon::start_background_service_metric_collector, get_config,
+        get_crate_pattern_and_priority, list_crate_priorities, queue_builder,
+        remove_crate_priority, set_config, set_crate_priority,
     },
 };
 use futures_util::StreamExt;
@@ -199,6 +200,10 @@ impl CommandLine {
                     docs_rs::utils::daemon::start_background_queue_rebuild(&ctx)?;
                 }
 
+                // When people run the services separately, we assume that we can collect service
+                // metrics from the registry watcher, which should only run once, and all the time.
+                start_background_service_metric_collector(&ctx)?;
+
                 start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?;
 
                 ctx.runtime.block_on(async move {