From ed436937b0a268c660b67643443122b3c3eea3c0 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:12:37 +0100 Subject: [PATCH 01/26] add otel_grpc param --- crates/embucketd/src/cli.rs | 8 ++++++++ crates/embucketd/src/main.rs | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/embucketd/src/cli.rs b/crates/embucketd/src/cli.rs index f6db5875..6c8b3b83 100644 --- a/crates/embucketd/src/cli.rs +++ b/crates/embucketd/src/cli.rs @@ -118,6 +118,14 @@ pub struct CliOpts { )] pub auth_demo_password: Option, + #[arg( + long, + env = "OTEL_GRPC", + default_value = "true", + help = "Enable OTLP gRPC exporter (requires 'otel-grpc' feature)" + )] + pub otel_grpc: bool, + #[arg( long, value_enum, diff --git a/crates/embucketd/src/main.rs b/crates/embucketd/src/main.rs index d758572b..904a58d7 100644 --- a/crates/embucketd/src/main.rs +++ b/crates/embucketd/src/main.rs @@ -197,11 +197,19 @@ async fn async_main( #[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] fn setup_tracing(opts: &cli::CliOpts) -> SdkTracerProvider { - // Initialize OTLP exporter using gRPC (Tonic) - let exporter = opentelemetry_otlp::SpanExporter::builder() - .with_tonic() - .build() - .expect("Failed to create OTLP exporter"); + let exporter = if opts.otel_grpc { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } else { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + }; let resource = Resource::builder().with_service_name("Em").build(); From 1b3f97298fe80f2f81f1dc080e26ad42a1afaf3c Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:13:29 +0100 Subject: [PATCH 02/26] use registry for both logs and spans --- Cargo.lock | 9 ++- crates/embucket-lambda/Cargo.toml | 9 ++- crates/embucket-lambda/src/config.rs | 8 +++ crates/embucket-lambda/src/main.rs | 96 ++++++++++++++++++++++++++-- 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee056747..775bf8e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,11 +1110,11 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.5", + "aws-smithy-http 0.62.6", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types 1.3.4", + "aws-smithy-types 1.3.5", "aws-types", "bytes", "fastrand", @@ -3309,6 +3309,9 @@ dependencies = [ "http 1.4.0", "http-body-util", "lambda_http", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "serde", "serde_json", "serde_urlencoded", @@ -3316,6 +3319,8 @@ dependencies = [ "tower 0.5.2", "tower-http", "tracing", + "tracing-allocations", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index 8f54e70c..fb7921d9 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -11,8 +11,6 @@ catalog-metastore = { path = "../catalog-metastore" } executor = { path = "../executor" } lambda_http = "0.17" tokio = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } base64 = "0.22" serde = { workspace = true } serde_json = { workspace = true } @@ -25,6 +23,13 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"] tower = { workspace = true } tower-http = { workspace = true } cfg-if = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { version = "0.3.20", features = ["env-filter", "registry", "fmt", "json"] } +tracing-opentelemetry = { version = "0.31.0" } +tracing-allocations = { version = "0.1.0", optional = true } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic"] } +opentelemetry_sdk = { version = "0.30.0" } +opentelemetry = { version = "0.30.0" } [lints] workspace = true diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index 37ec5316..fba001c2 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -24,6 +24,10 @@ pub struct EnvConfig { pub iceberg_catalog_timeout_secs: u64, pub object_store_timeout_secs: u64, pub object_store_connect_timeout_secs: u64, + pub otel_grpc: bool, + pub tracing_level: String, + pub log_format: String, + pub log_filter: String, } impl EnvConfig { @@ -57,6 +61,10 @@ impl EnvConfig { object_store_timeout_secs: parse_env("OBJECT_STORE_TIMEOUT_SECS").unwrap_or(30), object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS") .unwrap_or(3), + otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), + tracing_level: env_or_default("TRACING_LEVEL", "INFO"), + log_format: env_or_default("LOG_FORMAT", "pretty"), + log_filter: parse_env("RUST_LOG").unwrap_or("INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index a7c49571..a56b6b3c 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -14,11 +14,20 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; +use tracing::instrument::WithSubscriber; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SdkTracerProvider; +use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { use lambda_http::run_with_streaming_response as run; @@ -29,11 +38,15 @@ cfg_if::cfg_if! { type InitResult = Result>; +const DISABLED_TARGETS: [&str; 2] = ["h2", "aws_smithy_runtime"]; + #[tokio::main] async fn main() -> Result<(), LambdaError> { - init_tracing(); - let env_config = EnvConfig::from_env(); + + init_tracing(&env_config); + init_tracing_and_logs(&env_config); + info!( data_format = %env_config.data_format, max_concurrency = env_config.max_concurrency_level, @@ -212,14 +225,83 @@ fn extract_socket_addr(headers: &HeaderMap) -> Option { .map(|ip| SocketAddr::new(ip, 0)) } -fn init_tracing() { +#[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] +fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { + let exporter = if config.otel_grpc { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } else { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + }; + + let resource = Resource::builder().build(); + + let tracing_provider = SdkTracerProvider::builder() + .with_span_processor(BatchSpanProcessor::builder(exporter).build()) + .with_resource(resource) + .build(); + + let targets_with_level = + |targets: &[&'static str], level: LevelFilter| -> Vec<(&str, LevelFilter)> { + // let default_log_targets: Vec<(String, LevelFilter)> = + targets.iter().map(|t| ((*t), level)).collect() + }; + + + let registry = tracing_subscriber::registry() + // Telemetry filtering + .with( + tracing_opentelemetry::OpenTelemetryLayer::new(tracing_provider.tracer("embucket")) + .with_level(true) + .with_filter( + Targets::default() + .with_targets(targets_with_level(&DISABLED_TARGETS, LevelFilter::OFF)) + .with_default(config.tracing_level.parse().unwrap_or(tracing::Level::INFO)), + ), + ); + // Logs filtering + // fmt::layer has different types for json vs plain + if config.log_format == "json" { + registry.with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(false) + .json() + .with_current_span(true) + .with_span_list(true) + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); + } else { + registry.with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(std::io::stdout().is_terminal()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::ENTER + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); + }; + + tracing_provider +} + + +fn init_tracing(config: &EnvConfig) { let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); let emit_ansi = std::io::stdout().is_terminal(); - // Use json format if requested via env var, otherwise use pretty format with span events - let format = std::env::var("LOG_FORMAT").unwrap_or_else(|_| "pretty".to_string()); - - if format == "json" { + if config.log_format == "json" { let _ = tracing_subscriber::fmt() .with_env_filter(filter) .with_target(true) From e8dfc704d28e136aac6a00b4923a08922f9fef8b Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:57:42 +0100 Subject: [PATCH 03/26] add collector artifacts into gitignore --- .gitignore | 2 ++ crates/embucket-lambda/Cargo.toml | 2 +- crates/embucket-lambda/src/main.rs | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6372e0a5..f537c353 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ debug/ target/ data/ +collector-config/ +extensions/ .env metastore.yaml diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index fb7921d9..943caed7 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -57,7 +57,7 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config"] +include = ["config", "collector-config", "extensions"] [package.metadata.lambda.deploy.env] LOG_FORMAT = "json" diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index a56b6b3c..bc6c9bf5 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -14,7 +14,6 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; -use tracing::instrument::WithSubscriber; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; @@ -24,9 +23,8 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::BatchSpanProcessor; use opentelemetry_sdk::trace::SdkTracerProvider; -use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; +use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::EnvFilter; -use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { From 580698b3271e5337c3ce518d54a09e60848b8d49 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Mon, 15 Dec 2025 13:08:55 +0100 Subject: [PATCH 04/26] shitdown tracing provider --- crates/embucket-lambda/src/config.rs | 2 +- crates/embucket-lambda/src/main.rs | 98 +++++++++++----------------- 2 files changed, 40 insertions(+), 60 deletions(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index fba001c2..72c3067b 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -64,7 +64,7 @@ impl EnvConfig { otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), log_format: env_or_default("LOG_FORMAT", "pretty"), - log_filter: parse_env("RUST_LOG").unwrap_or("INFO".to_string()), + log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index bc6c9bf5..cbdfc3a5 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -14,17 +14,17 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; -use opentelemetry::trace::TracerProvider; -use opentelemetry_sdk::Resource; -use opentelemetry_sdk::trace::BatchSpanProcessor; -use opentelemetry_sdk::trace::SdkTracerProvider; -use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { @@ -42,8 +42,7 @@ const DISABLED_TARGETS: [&str; 2] = ["h2", "aws_smithy_runtime"]; async fn main() -> Result<(), LambdaError> { let env_config = EnvConfig::from_env(); - init_tracing(&env_config); - init_tracing_and_logs(&env_config); + let tracing_provider = init_tracing_and_logs(&env_config); info!( data_format = %env_config.data_format, @@ -66,11 +65,18 @@ async fn main() -> Result<(), LambdaError> { err })?); - run(service_fn(move |event: Request| { + let err = run(service_fn(move |event: Request| { let app = Arc::clone(&app); async move { app.handle_event(event).await } })) - .await + .await; + + tracing_provider.shutdown().map_err(|err| { + error!(error = %err, "Failed to shutdown TracerProvider"); + err + })?; + + err } struct LambdaApp { @@ -252,7 +258,6 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { targets.iter().map(|t| ((*t), level)).collect() }; - let registry = tracing_subscriber::registry() // Telemetry filtering .with( @@ -267,56 +272,31 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Logs filtering // fmt::layer has different types for json vs plain if config.log_format == "json" { - registry.with( - tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(false) - .json() - .with_current_span(true) - .with_span_list(true) - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); - } else { - registry.with( - tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(std::io::stdout().is_terminal()) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::ENTER - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); - }; - - tracing_provider -} - - -fn init_tracing(config: &EnvConfig) { - let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); - let emit_ansi = std::io::stdout().is_terminal(); - // Use json format if requested via env var, otherwise use pretty format with span events - if config.log_format == "json" { - let _ = tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(true) - .with_ansi(false) - .json() - .with_current_span(true) - .with_span_list(true) - .try_init(); + registry + .with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(false) + .json() + .with_current_span(true) + .with_span_list(true), + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); } else { - let _ = tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(true) - .with_ansi(emit_ansi) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::ENTER - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + registry + .with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(std::io::stdout().is_terminal()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::ENTER + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ), ) - .try_init(); + .with(EnvFilter::new(config.log_filter.clone())) + .init(); } + + tracing_provider } From 0560d41a3719589459fd7a369b9c49163e83bbe8 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 21:37:40 +0200 Subject: [PATCH 05/26] fixes for honeycomb --- crates/embucket-lambda/src/config.rs | 2 +- crates/embucket-lambda/src/main.rs | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index 72c3067b..4618acc1 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -63,7 +63,7 @@ impl EnvConfig { .unwrap_or(3), otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), - log_format: env_or_default("LOG_FORMAT", "pretty"), + log_format: env_or_default("LOG_FORMAT", "json"), log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index cbdfc3a5..390b3be0 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -21,6 +21,7 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use opentelemetry_otlp::WithExportConfig; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -235,17 +236,19 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Initialize OTLP exporter using gRPC (Tonic) opentelemetry_otlp::SpanExporter::builder() .with_tonic() + .with_endpoint("http://127.0.0.1:4317".to_string()) .build() .expect("Failed to create OTLP gRPC exporter") } else { // Initialize OTLP exporter using HTTP opentelemetry_otlp::SpanExporter::builder() .with_http() + .with_endpoint("http://127.0.0.1:4318".to_string()) .build() .expect("Failed to create OTLP HTTP exporter") }; - let resource = Resource::builder().build(); + let resource = Resource::builder().with_service_name("embucket-lambda-api").build(); let tracing_provider = SdkTracerProvider::builder() .with_span_processor(BatchSpanProcessor::builder(exporter).build()) @@ -275,11 +278,12 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { registry .with( tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(false) .json() - .with_current_span(true) - .with_span_list(true), + .with_target(false) + .with_ansi(false) + .with_current_span(false) + .with_span_list(false) + .without_time(), ) .with(EnvFilter::new(config.log_filter.clone())) .init(); From 23315dad5963bffbe20e79b176f1a1651bed33e7 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 22:55:24 +0200 Subject: [PATCH 06/26] fixes for honeycomb and otlp layers --- crates/embucket-lambda/Makefile | 6 ++++-- crates/embucket-lambda/src/main.rs | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index fc71cc79..664990b0 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -10,11 +10,13 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build - cd ../.. && cargo lambda deploy --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:14 + --binary-name bootstrap $(FUNCTION_NAME) # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:14 + --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development watch: diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 390b3be0..f56a2baf 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -7,7 +7,7 @@ use api_snowflake_rest::server::make_snowflake_router; use api_snowflake_rest::server::server_models::RestApiConfig as SnowflakeServerConfig; use api_snowflake_rest::server::state::AppState; use api_snowflake_rest_sessions::session::SESSION_EXPIRATION_SECONDS; -use axum::Router; +use axum::{middleware, Router}; use axum::body::Body as AxumBody; use axum::extract::connect_info::ConnectInfo; use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; @@ -27,6 +27,9 @@ use tracing::{error, info}; use tracing_subscriber::EnvFilter; use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; +use axum::extract::{Request as AxumRequest, State}; +use axum::middleware::Next; +use axum::response::IntoResponse; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { use lambda_http::run_with_streaming_response as run; @@ -236,14 +239,12 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Initialize OTLP exporter using gRPC (Tonic) opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .with_endpoint("http://127.0.0.1:4317".to_string()) .build() .expect("Failed to create OTLP gRPC exporter") } else { // Initialize OTLP exporter using HTTP opentelemetry_otlp::SpanExporter::builder() .with_http() - .with_endpoint("http://127.0.0.1:4318".to_string()) .build() .expect("Failed to create OTLP HTTP exporter") }; @@ -303,4 +304,4 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { } tracing_provider -} +} \ No newline at end of file From b7f5c3a84504a265ffe1eb34a8192ca946ce46e8 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:09:46 +0200 Subject: [PATCH 07/26] fixes for honeycomb and otlp layers --- .../src/tests/create_test_server.rs | 1 + crates/embucket-lambda/src/main.rs | 16 +++++----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/crates/api-snowflake-rest/src/tests/create_test_server.rs b/crates/api-snowflake-rest/src/tests/create_test_server.rs index ebd0f535..d22874fd 100644 --- a/crates/api-snowflake-rest/src/tests/create_test_server.rs +++ b/crates/api-snowflake-rest/src/tests/create_test_server.rs @@ -30,6 +30,7 @@ pub fn executor_default_cfg() -> UtilsConfig { UtilsConfig::default().with_max_concurrency_level(2) } +#[must_use] pub fn metastore_default_settings_cfg() -> MetastoreSettingsConfig { MetastoreSettingsConfig::default() .with_object_store_connect_timeout(1) diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index f56a2baf..d965da82 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -7,7 +7,7 @@ use api_snowflake_rest::server::make_snowflake_router; use api_snowflake_rest::server::server_models::RestApiConfig as SnowflakeServerConfig; use api_snowflake_rest::server::state::AppState; use api_snowflake_rest_sessions::session::SESSION_EXPIRATION_SECONDS; -use axum::{middleware, Router}; +use axum::Router; use axum::body::Body as AxumBody; use axum::extract::connect_info::ConnectInfo; use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; @@ -21,15 +21,11 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; -use opentelemetry_otlp::WithExportConfig; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; -use axum::extract::{Request as AxumRequest, State}; -use axum::middleware::Next; -use axum::response::IntoResponse; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { use lambda_http::run_with_streaming_response as run; @@ -157,10 +153,6 @@ impl LambdaApp { ); } - // if let Err(err) = ensure_session_header(&mut parts.headers, &self.state).await { - // return Ok(snowflake_error_response(&err)); - // } - let mut axum_request = to_axum_request(parts, body_bytes); if let Some(addr) = extract_socket_addr(axum_request.headers()) { axum_request.extensions_mut().insert(ConnectInfo(addr)); @@ -249,7 +241,9 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { .expect("Failed to create OTLP HTTP exporter") }; - let resource = Resource::builder().with_service_name("embucket-lambda-api").build(); + let resource = Resource::builder() + .with_service_name("embucket-lambda-api") + .build(); let tracing_provider = SdkTracerProvider::builder() .with_span_processor(BatchSpanProcessor::builder(exporter).build()) @@ -304,4 +298,4 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { } tracing_provider -} \ No newline at end of file +} From 9a8e24eef6e96e57dcd996f74c67ac376aea964b Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:14:03 +0200 Subject: [PATCH 08/26] fmt fix --- crates/api-snowflake-rest/src/tests/create_test_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/api-snowflake-rest/src/tests/create_test_server.rs b/crates/api-snowflake-rest/src/tests/create_test_server.rs index d22874fd..96b3a1be 100644 --- a/crates/api-snowflake-rest/src/tests/create_test_server.rs +++ b/crates/api-snowflake-rest/src/tests/create_test_server.rs @@ -30,7 +30,7 @@ pub fn executor_default_cfg() -> UtilsConfig { UtilsConfig::default().with_max_concurrency_level(2) } -#[must_use] +#[must_use] pub fn metastore_default_settings_cfg() -> MetastoreSettingsConfig { MetastoreSettingsConfig::default() .with_object_store_connect_timeout(1) From c4efc7ad261864b120b403d7123e4f887500712d Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:15:42 +0200 Subject: [PATCH 09/26] fix --- crates/embucket-lambda/src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index d965da82..2a1b614c 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -241,9 +241,7 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { .expect("Failed to create OTLP HTTP exporter") }; - let resource = Resource::builder() - .with_service_name("embucket-lambda-api") - .build(); + let resource = Resource::builder().build(); let tracing_provider = SdkTracerProvider::builder() .with_span_processor(BatchSpanProcessor::builder(exporter).build()) From 050869d990c7454e08a94c8676d244dbc2d9c6ed Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:17:11 +0200 Subject: [PATCH 10/26] fix --- crates/embucket-lambda/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 664990b0..1718d360 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -10,12 +10,12 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:14 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:13 --binary-name bootstrap $(FUNCTION_NAME) # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:14 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:13 --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development From 6e2df29d2b36cc146cd22569f799b26782272b54 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:19:17 +0200 Subject: [PATCH 11/26] hardcoded --- crates/embucket-lambda/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 1718d360..af2d8c23 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -10,12 +10,12 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:13 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:17 --binary-name bootstrap $(FUNCTION_NAME) # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:13 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:17 --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development From 238af18da78a6bfb826a59f187f51972d0b2cb0f Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:30:36 +0200 Subject: [PATCH 12/26] not hardcoded --- crates/embucket-lambda/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index af2d8c23..424d7a61 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -10,12 +10,12 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:17 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 --binary-name bootstrap $(FUNCTION_NAME) # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:17 + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development From 65a92638afcda0e01837e4bf20ae795360938543 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:48:15 +0200 Subject: [PATCH 13/26] remove duplication --- crates/embucket-lambda/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 424d7a61..b29c1fe2 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -9,9 +9,8 @@ build: cd ../.. && cargo lambda build --release --arm64 --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) # Deploy to AWS (must run from workspace root for include paths to work) -deploy: build - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 - --binary-name bootstrap $(FUNCTION_NAME) +deploy: build deploy-only + @echo "Deployed $(FUNCTION_NAME) to AWS" # Quick deploy without rebuild deploy-only: From 413dd929162a8257e7f494387e6cce2c7868af13 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 23:50:18 +0200 Subject: [PATCH 14/26] remove duplication --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index b29c1fe2..ccb25b46 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -15,7 +15,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 - --binary-name bootstrap $(FUNCTION_NAME) + --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE) # Watch locally for development watch: From 0e7604ec427545fa73fc6c0f01a5f05ddf86ff81 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Wed, 17 Dec 2025 22:50:31 +0200 Subject: [PATCH 15/26] WIP --- config/.env.exmaple | 4 ++++ crates/embucket-lambda/Makefile | 27 ++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 config/.env.exmaple diff --git a/config/.env.exmaple b/config/.env.exmaple new file mode 100644 index 00000000..3d849888 --- /dev/null +++ b/config/.env.exmaple @@ -0,0 +1,4 @@ +METASTORE_CONFIG=config/metastore.yaml +JWT_SECRET=secret +RUST_LOG=info +LOG_FORMAT=json \ No newline at end of file diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index cfdc6b37..dfde0f1e 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -2,21 +2,38 @@ # Function name (override with: make deploy FUNCTION_NAME=your-function) FUNCTION_NAME ?= embucket-lambda -ENV_FILE ?= .env +ENV_FILE ?= config/.env +IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole +OTEL_COLLECTOR_LAYERS ?= # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) build: - cd ../.. && cargo lambda build --release --arm64 --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) + cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --include config/metastore.yaml --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only - @echo "Deployed $(FUNCTION_NAME) to AWS" + @echo "Deployed $(FUNCTION_NAME) to AWS" # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 - --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE) + cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true + @$(MAKE) public-url FUNCTION_NAME=$(FUNCTION_NAME) + +public-url: + @set -e; \ + echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \ + aws lambda create-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null 2>&1 || \ + aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \ + echo "Ensuring public invoke permission exists..." ; \ + aws lambda add-permission --function-name "$(FUNCTION_NAME)" \ + --statement-id AllowPublicURLInvoke \ + --action lambda:InvokeFunctionUrl \ + --principal "*" \ + --function-url-auth-type NONE >/dev/null 2>&1 || true ; \ + URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \ + echo "$$URL" # Watch locally for development watch: From 234f5791aca6111bad4538bd6cd614eb3fcc0e65 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Thu, 18 Dec 2025 23:33:13 +0200 Subject: [PATCH 16/26] readme --- crates/embucket-lambda/README.md | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index 9cb44dbb..db31fb01 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -77,6 +77,66 @@ cargo lambda deploy --binary-name bootstrap ``` - It will deploy envs from `.env` if `ENV_FILE` not specified +### Observability + +#### AWS traces +We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces. +- `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". + +#### Exporting telemetry + +To work with Opentelemtry, you need an Opentelemetry Collector running in your environment with open telemetry config. +The easiest way is to add two layers to your lambda deployment. One of which would be your config file with the remote exporter. + +1. Create a folder called collector-config and add a file called `config.yml` with the OpenTelemetry Collector [**configuration**](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/). +2. After which zip the folder with this ocmmand: `zip -r .zip collector-config` +3. Then publish it to AWS (change the file name nad layer name if you want): `aws lambda publish-layer-version + --layer-name + --zip-file fileb://.zip + --compatible-runtimes provided.al2 provided.al2023 + --compatible-architectures arm64` +4. After which add this to the Makefile (the first layer is the collector itself): `OTEL_COLLECTOR_LAYERS ?= \ + --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1\ + --layer-arn arn:aws:lambda:::layer::` +5. Now you can deploy the function with the new layer. + +If you later update the configratuin and publish the layer again remember to change the layer `` number, after the first publish it is `1`. + +#### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/) + +OpenTelemrty Collector config example for Honeycomb: +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + http: + endpoint: localhost:4318 + +processors: + batch: + +exporters: + otlp: + # You can name these envs anything you want as long as they are the same as in .env file + endpoint: "${env:HONEYCOMB_ENDPOINT_URL}" + headers: + x-honeycomb-team: "${env:HONEYCOMB_API_KEY}" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] +``` + +- Environment variables configuration: + * `HONEYCOMB_API_KEY` - this is the full ingestion key (not the key id or management key) + * `HONEYCOMB_ENDPOINT_URL` - check the region it can start be `api.honeycomb.io` or `api.eu1.honeycomb.io` + * `OTEL_SERVICE_NAME` - is the x-honeycomb-dataset name + ### Test locally ```bash From 59efa7e61fa6382cfa6d8e1e08defcbfcba35936 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Thu, 18 Dec 2025 23:34:54 +0200 Subject: [PATCH 17/26] readme --- crates/embucket-lambda/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index db31fb01..946f9928 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -83,7 +83,7 @@ cargo lambda deploy --binary-name bootstrap We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces. - `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". -#### Exporting telemetry +#### OpenTelemetry configuration To work with Opentelemtry, you need an Opentelemetry Collector running in your environment with open telemetry config. The easiest way is to add two layers to your lambda deployment. One of which would be your config file with the remote exporter. From a558127f8adf41aa98bbf12802022aac8ef6ee06 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:14:08 +0200 Subject: [PATCH 18/26] fixes --- crates/embucket-lambda/Cargo.toml | 2 +- crates/embucket-lambda/Makefile | 4 +- crates/embucket-lambda/src/main.rs | 64 +++++++++++++++--------------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index e70dbca4..516023d7 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -58,7 +58,7 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config", "collector-config", "extensions"] +include = ["config"] [package.metadata.lambda.deploy.env] LOG_FORMAT = "json" diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index dfde0f1e..c95238d3 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -3,13 +3,13 @@ # Function name (override with: make deploy FUNCTION_NAME=your-function) FUNCTION_NAME ?= embucket-lambda ENV_FILE ?= config/.env -IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole +IAM_ROLE ?= OTEL_COLLECTOR_LAYERS ?= # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) build: - cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --include config/metastore.yaml --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) + cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index e324082f..080877cd 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -25,8 +25,10 @@ use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::{LevelFilter, Targets}; +use tracing_subscriber::filter::{filter_fn, FilterExt, LevelFilter, Targets}; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::fmt::format::FmtSpan; + cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { use lambda_http::run_with_streaming_response as run; @@ -282,36 +284,36 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { .with_targets(targets_with_level(&DISABLED_TARGETS, LevelFilter::OFF)) .with_default(config.tracing_level.parse().unwrap_or(tracing::Level::INFO)), ), - ); - // Logs filtering - // fmt::layer has different types for json vs plain - if config.log_format == "json" { - registry - .with( - tracing_subscriber::fmt::layer() - .json() - .with_target(false) - .with_ansi(false) - .with_current_span(false) - .with_span_list(false) - .without_time(), - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); - } else { - registry - .with( - tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(std::io::stdout().is_terminal()) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::ENTER - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ), - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); - } + ) + .with({ + let fmt_filter = match std::env::var("RUST_LOG") { + Ok(val) => match val.parse::() { + Ok(log_targets_from_env) => log_targets_from_env, + Err(err) => { + eprintln!("Failed to parse RUST_LOG: {err:?}"); + Targets::default() + .with_targets(targets_with_level(&DISABLED_TARGETS, LevelFilter::OFF)) + .with_default(LevelFilter::DEBUG) + } + }, + _ => Targets::default() + .with_targets(targets_with_level(&DISABLED_TARGETS, LevelFilter::OFF)) + .with_default(LevelFilter::INFO), + }; + // Skip memory allocations spans + let spans_always = filter_fn(|meta| meta.is_span()); + let not_alloc_event = filter_fn(|meta| { + meta.target() != "alloc" && meta.target() != "tracing_allocations" + }); + + tracing_subscriber::fmt::layer() + .with_target(true) + .with_level(true) + .with_span_events(FmtSpan::NONE) + .json() + .with_filter(spans_always.or(not_alloc_event.and(fmt_filter))) + }); + registry.init(); tracing_provider } From b8d8463d6dc8537a0fae52d4907e28e543349d8d Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:14:18 +0200 Subject: [PATCH 19/26] fixes --- crates/embucket-lambda/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 080877cd..8656c444 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -25,9 +25,9 @@ use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::{filter_fn, FilterExt, LevelFilter, Targets}; -use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { @@ -46,7 +46,7 @@ async fn main() -> Result<(), LambdaError> { let env_config = EnvConfig::from_env(); let tracing_provider = init_tracing_and_logs(&env_config); - + // Log version and build information on startup info!( version = %BuildInfo::GIT_DESCRIBE, From 1ceebaaa4b2a8d2ff2bcd68c16996d77cc33cbae Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:28:18 +0200 Subject: [PATCH 20/26] fixes --- crates/embucket-lambda/src/config.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index 90d4194f..caa619c7 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -25,10 +25,8 @@ pub struct EnvConfig { pub iceberg_catalog_timeout_secs: u64, pub object_store_timeout_secs: u64, pub object_store_connect_timeout_secs: u64, - pub otel_grpc: bool, + pub otel_exporter_otlp_protocol: String, pub tracing_level: String, - pub log_format: String, - pub log_filter: String, } impl EnvConfig { @@ -62,10 +60,8 @@ impl EnvConfig { object_store_timeout_secs: parse_env("OBJECT_STORE_TIMEOUT_SECS").unwrap_or(30), object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS") .unwrap_or(3), - otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), + otel_exporter_otlp_protocol: parse_env("OTEL_EXPORTER_OTLP_PROTOCOL").unwrap_or("grpc".to_string()), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), - log_format: env_or_default("LOG_FORMAT", "json"), - log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()), } } From 717be7e72a030dd5c5816db9ed31a8afa12eaeed Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:28:53 +0200 Subject: [PATCH 21/26] fmt --- crates/embucket-lambda/src/config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index caa619c7..bfc058ad 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -60,7 +60,8 @@ impl EnvConfig { object_store_timeout_secs: parse_env("OBJECT_STORE_TIMEOUT_SECS").unwrap_or(30), object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS") .unwrap_or(3), - otel_exporter_otlp_protocol: parse_env("OTEL_EXPORTER_OTLP_PROTOCOL").unwrap_or("grpc".to_string()), + otel_exporter_otlp_protocol: parse_env("OTEL_EXPORTER_OTLP_PROTOCOL") + .unwrap_or("grpc".to_string()), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), } } From 3865a110850aa669d67b596ae5e58bffae39b43d Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:36:19 +0200 Subject: [PATCH 22/26] fix --- crates/embucket-lambda/src/main.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 8656c444..1289a7aa 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -19,12 +19,10 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::BatchSpanProcessor; use opentelemetry_sdk::trace::SdkTracerProvider; -use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; -use tracing_subscriber::EnvFilter; use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; @@ -247,18 +245,22 @@ fn extract_socket_addr(headers: &HeaderMap) -> Option { #[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { - let exporter = if config.otel_grpc { - // Initialize OTLP exporter using gRPC (Tonic) - opentelemetry_otlp::SpanExporter::builder() - .with_tonic() - .build() - .expect("Failed to create OTLP gRPC exporter") - } else { - // Initialize OTLP exporter using HTTP - opentelemetry_otlp::SpanExporter::builder() - .with_http() - .build() - .expect("Failed to create OTLP HTTP exporter") + let exporter = match config.otel_exporter_otlp_protocol.as_str() { + "grpc" => { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } + "http/json" => { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + } + protocol => panic!("Unsupported OTLP exporter protocol: {protocol}"), }; let resource = Resource::builder().build(); From 4ab2b5d8464d75f91452b2ef0da90ee3ac5991fb Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:41:36 +0200 Subject: [PATCH 23/26] fix --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index f537c353..6372e0a5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ debug/ target/ data/ -collector-config/ -extensions/ .env metastore.yaml From 1f62d9b06365cd187991761d2a68bf390b30b626 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 00:43:16 +0200 Subject: [PATCH 24/26] fix --- config/.env.exmaple | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/.env.exmaple b/config/.env.exmaple index 3d849888..9b66cd05 100644 --- a/config/.env.exmaple +++ b/config/.env.exmaple @@ -1,4 +1,3 @@ METASTORE_CONFIG=config/metastore.yaml JWT_SECRET=secret -RUST_LOG=info -LOG_FORMAT=json \ No newline at end of file +TRACING_LEVEL=info \ No newline at end of file From 701e9d61013b29b20f184b4876f9ecdd78d4fd2c Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Fri, 19 Dec 2025 01:59:22 +0200 Subject: [PATCH 25/26] requested changes --- crates/embucket-lambda/Makefile | 3 +-- crates/embucket-lambda/src/main.rs | 2 +- crates/embucketd/src/cli.rs | 8 ++++---- crates/embucketd/src/main.rs | 28 ++++++++++++++++------------ 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index c95238d3..c164bdeb 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -12,14 +12,13 @@ build: cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) # Deploy to AWS (must run from workspace root for include paths to work) -deploy: build deploy-only +deploy: build deploy-only public-url @echo "Deployed $(FUNCTION_NAME) to AWS" # Quick deploy without rebuild deploy-only: cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true - @$(MAKE) public-url FUNCTION_NAME=$(FUNCTION_NAME) public-url: @set -e; \ diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 1289a7aa..5e1d0799 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -245,7 +245,7 @@ fn extract_socket_addr(headers: &HeaderMap) -> Option { #[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { - let exporter = match config.otel_exporter_otlp_protocol.as_str() { + let exporter = match config.otel_exporter_otlp_protocol.to_lowercase().as_str() { "grpc" => { // Initialize OTLP exporter using gRPC (Tonic) opentelemetry_otlp::SpanExporter::builder() diff --git a/crates/embucketd/src/cli.rs b/crates/embucketd/src/cli.rs index 6c8b3b83..e1b76e84 100644 --- a/crates/embucketd/src/cli.rs +++ b/crates/embucketd/src/cli.rs @@ -120,11 +120,11 @@ pub struct CliOpts { #[arg( long, - env = "OTEL_GRPC", - default_value = "true", - help = "Enable OTLP gRPC exporter (requires 'otel-grpc' feature)" + env = "OTEL_EXPORTER_OTLP_PROTOCOL", + default_value = "grpc", + help = "OpenTelemetry Exporter Protocol" )] - pub otel_grpc: bool, + pub otel_exporter_otlp_protocol: String, #[arg( long, diff --git a/crates/embucketd/src/main.rs b/crates/embucketd/src/main.rs index 20aea838..e113e53a 100644 --- a/crates/embucketd/src/main.rs +++ b/crates/embucketd/src/main.rs @@ -207,18 +207,22 @@ async fn async_main( #[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] fn setup_tracing(opts: &cli::CliOpts) -> SdkTracerProvider { - let exporter = if opts.otel_grpc { - // Initialize OTLP exporter using gRPC (Tonic) - opentelemetry_otlp::SpanExporter::builder() - .with_tonic() - .build() - .expect("Failed to create OTLP gRPC exporter") - } else { - // Initialize OTLP exporter using HTTP - opentelemetry_otlp::SpanExporter::builder() - .with_http() - .build() - .expect("Failed to create OTLP HTTP exporter") + let exporter = match opts.otel_exporter_otlp_protocol.to_lowercase().as_str() { + "grpc" => { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } + "http/json" => { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + } + protocol => panic!("Unsupported OTLP protocol: {}", protocol), }; let resource = Resource::builder().with_service_name("Em").build(); From 292d4bda5102ed0422861ce06d851609cca6b02c Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 23 Dec 2025 13:39:20 +0200 Subject: [PATCH 26/26] requested changes --- crates/embucket-lambda/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index 946f9928..063eda07 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -90,12 +90,12 @@ The easiest way is to add two layers to your lambda deployment. One of which wou 1. Create a folder called collector-config and add a file called `config.yml` with the OpenTelemetry Collector [**configuration**](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/). 2. After which zip the folder with this ocmmand: `zip -r .zip collector-config` -3. Then publish it to AWS (change the file name nad layer name if you want): `aws lambda publish-layer-version +3. Then publish it to AWS (change the file name and layer name if you want): `aws lambda publish-layer-version --layer-name --zip-file fileb://.zip --compatible-runtimes provided.al2 provided.al2023 --compatible-architectures arm64` -4. After which add this to the Makefile (the first layer is the collector itself): `OTEL_COLLECTOR_LAYERS ?= \ +4. After which provide this as an external env variable (the first layer is the collector itself): `OTEL_COLLECTOR_LAYERS ?= \ --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1\ --layer-arn arn:aws:lambda:::layer::` 5. Now you can deploy the function with the new layer.