From f0a6298963f9cf26df2f0114eaf932023cf6e51a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 26 Nov 2025 10:53:17 -0700 Subject: [PATCH 1/5] add enabled_logs variable to control which logs get the json_streaming_ treatment --- scripts/main.zeek | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/main.zeek b/scripts/main.zeek index ecbf229..1592f2f 100644 --- a/scripts/main.zeek +++ b/scripts/main.zeek @@ -27,6 +27,9 @@ export { ## JSON streaming logs. This is set separately since these logs are ephemeral ## and meant to be immediately carried off to some other storage and search system. const JSONStreaming::rotation_interval = 15mins &redef; + + ## Set of log names to get the json_streaming_ treatment. If empty, do all logs. + const JSONStreaming::enabled_logs: set[string] = set() &redef; } type JsonStreamingExtension: record { @@ -87,6 +90,10 @@ event zeek_init() &priority=-5 for ( stream in Log::active_streams ) { + ## Skip streams not in the enabled set (unless enabled_logs is empty) + if ( |JSONStreaming::enabled_logs| > 0 && !(stream in JSONStreaming::enabled_logs) ) + next; + for ( filter_name in Log::get_filter_names(stream) ) { # This is here because we're modifying the list of filters right now... From 20a30c6650d8a7979802261a3130a24eaec46b18 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 26 Nov 2025 10:55:48 -0700 Subject: [PATCH 2/5] add enabled_logs variable to control which logs get the json_streaming_ treatment --- scripts/main.zeek | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/main.zeek b/scripts/main.zeek index 1592f2f..09c36a3 100644 --- a/scripts/main.zeek +++ b/scripts/main.zeek @@ -91,7 +91,8 @@ event zeek_init() &priority=-5 for ( stream in Log::active_streams ) { ## Skip streams not in the enabled set (unless enabled_logs is empty) - if ( |JSONStreaming::enabled_logs| > 0 && !(stream in JSONStreaming::enabled_logs) ) + local stream_name = Log::id_name(stream); + if ( |JSONStreaming::enabled_logs| > 0 && !(stream_name in JSONStreaming::enabled_logs) ) next; for ( filter_name in Log::get_filter_names(stream) ) From e4343732271cc1802a283adb672fa3a2e6ed77a9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 26 Nov 2025 11:21:04 -0700 Subject: [PATCH 3/5] add enabled_logs variable to control which logs get the json_streaming_ treatment --- scripts/main.zeek | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/main.zeek b/scripts/main.zeek index 09c36a3..872e43e 100644 --- a/scripts/main.zeek +++ b/scripts/main.zeek @@ -90,11 +90,6 @@ event zeek_init() &priority=-5 for ( stream in Log::active_streams ) { - ## Skip streams not in the enabled set (unless enabled_logs is empty) - local stream_name = Log::id_name(stream); - if ( |JSONStreaming::enabled_logs| > 0 && !(stream_name in JSONStreaming::enabled_logs) ) - next; - for ( filter_name in Log::get_filter_names(stream) ) { # This is here because we're modifying the list of filters right now... @@ -113,6 +108,12 @@ event zeek_init() &priority=-5 else if ( filt?$path_func ) filt$path = "json_streaming_" + filt$path_func(stream, "", []); + # Skip this filter if it's not in the enabled set (unless enabled_logs is empty) + # Remove leading directories, json_streaming_, and log extension + local log_type = sub(sub(sub(filt$path, /^.*\//, ""), /^json_streaming_?/, ""), /\.[^\.]+/, ""); + if ( |JSONStreaming::enabled_logs| > 0 && !(log_type in JSONStreaming::enabled_logs) ) + next; + filt$writer = Log::WRITER_ASCII; if ( JSONStreaming::enable_log_rotation ) From 0246ffa197ce709fdd260682fdffd254be7b2af5 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 26 Nov 2025 11:27:45 -0700 Subject: [PATCH 4/5] Added new test for enabled_files --- testing/tests/logs-filtered.zeek | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 testing/tests/logs-filtered.zeek diff --git a/testing/tests/logs-filtered.zeek b/testing/tests/logs-filtered.zeek new file mode 100644 index 0000000..80f9509 --- /dev/null +++ b/testing/tests/logs-filtered.zeek @@ -0,0 +1,10 @@ +# @TEST-DOC: Verifies that Zeek by default writes both the usual logs and the (filtered) json-streaming ones. +# @TEST-EXEC: zeek -r $TRACES/http.pcap $PACKAGE %INPUT +# @TEST-EXEC: for f in conn files http packet_filter; do test -f $f.log; done +# @TEST-EXEC: for f in files http; do test -f json_streaming_$f.log; done +# @TEST-EXEC: for f in conn packet_filter; do ! test -f json_streaming_$f.log; done + +# Filter the list of files +redef JSONStreaming::enabled_logs = set("http","files"); +# Turn off log rotation handling because it only kicks in for some of the files: +redef JSONStreaming::enable_log_rotation = F; From 77f7a518412d7fa4e6c4fc94ad3e00217607ea96 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 1 Dec 2025 15:57:00 +0000 Subject: [PATCH 5/5] make enabled_logs filter a set of log streams, not strings --- scripts/main.zeek | 14 ++++++-------- testing/tests/logs-filtered.zeek | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/main.zeek b/scripts/main.zeek index 872e43e..4b5c3e6 100644 --- a/scripts/main.zeek +++ b/scripts/main.zeek @@ -28,8 +28,8 @@ export { ## and meant to be immediately carried off to some other storage and search system. const JSONStreaming::rotation_interval = 15mins &redef; - ## Set of log names to get the json_streaming_ treatment. If empty, do all logs. - const JSONStreaming::enabled_logs: set[string] = set() &redef; + ## Set of log streams to get the json_streaming_ treatment. If empty, do all logs. + const JSONStreaming::enabled_logs: set[Log::ID] = set() &redef; } type JsonStreamingExtension: record { @@ -90,6 +90,10 @@ event zeek_init() &priority=-5 for ( stream in Log::active_streams ) { + # Skip this filter if it's not in the enabled set (unless enabled_logs is empty) + if ( |JSONStreaming::enabled_logs| > 0 && !(stream in JSONStreaming::enabled_logs) ) + next; + for ( filter_name in Log::get_filter_names(stream) ) { # This is here because we're modifying the list of filters right now... @@ -108,12 +112,6 @@ event zeek_init() &priority=-5 else if ( filt?$path_func ) filt$path = "json_streaming_" + filt$path_func(stream, "", []); - # Skip this filter if it's not in the enabled set (unless enabled_logs is empty) - # Remove leading directories, json_streaming_, and log extension - local log_type = sub(sub(sub(filt$path, /^.*\//, ""), /^json_streaming_?/, ""), /\.[^\.]+/, ""); - if ( |JSONStreaming::enabled_logs| > 0 && !(log_type in JSONStreaming::enabled_logs) ) - next; - filt$writer = Log::WRITER_ASCII; if ( JSONStreaming::enable_log_rotation ) diff --git a/testing/tests/logs-filtered.zeek b/testing/tests/logs-filtered.zeek index 80f9509..c6d7d89 100644 --- a/testing/tests/logs-filtered.zeek +++ b/testing/tests/logs-filtered.zeek @@ -5,6 +5,6 @@ # @TEST-EXEC: for f in conn packet_filter; do ! test -f json_streaming_$f.log; done # Filter the list of files -redef JSONStreaming::enabled_logs = set("http","files"); +redef JSONStreaming::enabled_logs = set(HTTP::LOG, Files::LOG); # Turn off log rotation handling because it only kicks in for some of the files: redef JSONStreaming::enable_log_rotation = F;