From 101f3b2b90509a75c8c5dc03c328f56d8bccbe7c Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:04:12 -0500 Subject: [PATCH 01/61] create event_import v1alpha --- ingestion/v1alpha/event_import.py | 105 ++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 ingestion/v1alpha/event_import.py diff --git a/ingestion/v1alpha/event_import.py b/ingestion/v1alpha/event_import.py new file mode 100644 index 00000000..03f4672f --- /dev/null +++ b/ingestion/v1alpha/event_import.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for importing events into Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/import +""" +# pylint: enable=line-too-long + +import argparse +import json + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def import_events( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + json_events: str) -> None: + """Import events into Chronicle using the Events Import API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + json_events: Events in (serialized) JSON format. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the parent resource: + chronicle.events.import + """ + parent = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{parent}/events:import" + + body = { + "events": json.loads(json_events), + } + + response = http_session.request("POST", url, json=body) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + result = response.json() + if "successCount" in result: + print(f"Successfully imported {result['successCount']} events") + if "failureCount" in result: + print(f"Failed to import {result['failureCount']} events") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--json_events_file", + type=argparse.FileType("r"), + required=True, + help="path to a file (or \"-\" for STDIN) containing events in JSON format") + + args = parser.parse_args() + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + import_events( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.json_events_file.read()) From 3ab7f9919420de3e5787289bbdf5446d48f41b53 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:24:19 -0500 Subject: [PATCH 02/61] add events_get --- ingestion/v1alpha/events_get.py | 99 +++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 ingestion/v1alpha/events_get.py diff --git a/ingestion/v1alpha/events_get.py b/ingestion/v1alpha/events_get.py new file mode 100644 index 00000000..5a501261 --- /dev/null +++ b/ingestion/v1alpha/events_get.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for getting event details from Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/get +""" +# pylint: enable=line-too-long + +import argparse +import base64 + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def get_event( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + event_id: str) -> None: + """Get event details from Chronicle using the Events Get API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + event_id: The ID of the event to retrieve. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the parent resource: + chronicle.events.get + """ + # URL encode the event_id in Base64 + encoded_event_id = base64.urlsafe_b64encode(event_id.encode()).decode() + name = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}/events/{encoded_event_id}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{name}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + print(response.json()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--event_id", + type=str, + required=True, + help="The ID of the event to retrieve") + + args = parser.parse_args() + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + get_event( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.event_id) From 3db7cf9efd854ed2bc103aa5f1efe0f92b3e4cea Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:26:34 -0500 Subject: [PATCH 03/61] events_batch_get --- ingestion/v1alpha/events_batch_get.py | 105 ++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 ingestion/v1alpha/events_batch_get.py diff --git a/ingestion/v1alpha/events_batch_get.py b/ingestion/v1alpha/events_batch_get.py new file mode 100644 index 00000000..acbf00c9 --- /dev/null +++ b/ingestion/v1alpha/events_batch_get.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for batch getting events from Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/batchGet +""" +# pylint: enable=line-too-long + +import argparse +import base64 +import json + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def batch_get_events( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + event_ids: str) -> None: + """Batch get events from Chronicle using the Events BatchGet API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + event_ids: JSON string containing a list of event IDs to retrieve. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the parent resource: + chronicle.events.batchGet + """ + parent = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{parent}/events:batchGet" + + # Convert event IDs to URL-encoded Base64 and create query parameters + event_ids_list = json.loads(event_ids) + encoded_ids = [base64.urlsafe_b64encode(id.encode()).decode() for id in event_ids_list] + query_params = "&".join([f"names={id}" for id in encoded_ids]) + + url = f"{url}?{query_params}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + print(json.dumps(response.json(), indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--event_ids", + type=str, + required=True, + help='JSON string containing a list of event IDs to retrieve (e.g., \'["id1", "id2"]\')') + + args = parser.parse_args() + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + batch_get_events( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.event_ids) From f0b93008c6681a52833befb2e7ac14f2e963cdee Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:29:38 -0500 Subject: [PATCH 04/61] udm_events_find --- search/v1alpha/udm_events_find.py | 144 ++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 search/v1alpha/udm_events_find.py diff --git a/search/v1alpha/udm_events_find.py b/search/v1alpha/udm_events_find.py new file mode 100644 index 00000000..338ca1db --- /dev/null +++ b/search/v1alpha/udm_events_find.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for finding UDM events in Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyFindUdmEvents +""" +# pylint: enable=line-too-long + +import argparse +import json +from typing import List, Optional + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def find_udm_events( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + tokens: Optional[List[str]] = None, + event_ids: Optional[List[str]] = None, + return_unenriched_data: bool = False, + return_all_events_for_log: bool = False) -> None: + """Find UDM events in Chronicle using the Legacy Find UDM Events API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + tokens: Optional list of tokens, with each token referring to a group of UDM/Entity events. + event_ids: Optional list of UDM/Entity event ids that should be returned. + If both tokens and event_ids are provided, tokens will be discarded. + return_unenriched_data: Optional boolean to return unenriched data. Default is False. + return_all_events_for_log: Optional boolean to return all events generated from the ingested log. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the instance resource: + chronicle.legacies.legacyFindUdmEvents + """ + instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindUdmEvents" + + # Build query parameters + params = [] + if tokens and not event_ids: # event_ids take precedence over tokens + for token in tokens: + params.append(f"tokens={token}") + if event_ids: + for event_id in event_ids: + params.append(f"ids={event_id}") + if return_unenriched_data: + params.append("returnUnenrichedData=true") + if return_all_events_for_log: + params.append("returnAllEventsForLog=true") + + if params: + url = f"{url}?{'&'.join(params)}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + print(json.dumps(response.json(), indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--tokens", + type=str, + help='JSON string containing a list of tokens (e.g., \'["token1", "token2"]\')') + parser.add_argument( + "--event_ids", + type=str, + help='JSON string containing a list of event IDs (e.g., \'["id1", "id2"]\')') + parser.add_argument( + "--return_unenriched_data", + action="store_true", + help="Whether to return unenriched data") + parser.add_argument( + "--return_all_events_for_log", + action="store_true", + help="Whether to return all events generated from the ingested log") + + args = parser.parse_args() + + # Convert JSON strings to lists if provided + tokens_list = json.loads(args.tokens) if args.tokens else None + event_ids_list = json.loads(args.event_ids) if args.event_ids else None + + # Validate that at least one of tokens or event_ids is provided + if not tokens_list and not event_ids_list: + parser.error("At least one of --tokens or --event_ids must be provided") + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + find_udm_events( + auth_session, + args.project_id, + args.project_instance, + args.region, + tokens_list, + event_ids_list, + args.return_unenriched_data, + args.return_all_events_for_log) From a8ebfdb4098063208c65c0555219b6221860d022 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:33:14 -0500 Subject: [PATCH 05/61] find assets and raw logs --- search/v1alpha/asset_events_find.py | 170 ++++++++++++++++++++++++++++ search/v1alpha/raw_logs_find.py | 162 ++++++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 search/v1alpha/asset_events_find.py create mode 100644 search/v1alpha/raw_logs_find.py diff --git a/search/v1alpha/asset_events_find.py b/search/v1alpha/asset_events_find.py new file mode 100644 index 00000000..6fc1801e --- /dev/null +++ b/search/v1alpha/asset_events_find.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for finding asset events in Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyFindAssetEvents +""" +# pylint: enable=line-too-long + +import argparse +import json +from datetime import datetime, timezone +from typing import Optional + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + +DEFAULT_MAX_RESULTS = 10000 +MAX_RESULTS_LIMIT = 250000 + + +def find_asset_events( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + asset_indicator: str, + start_time: str, + end_time: str, + reference_time: Optional[str] = None, + max_results: Optional[int] = None) -> None: + """Find asset events in Chronicle using the Legacy Find Asset Events API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + asset_indicator: JSON string containing the asset indicator to search for. + start_time: Start time in RFC3339 format (e.g., "2024-01-01T00:00:00Z"). + end_time: End time in RFC3339 format (e.g., "2024-01-02T00:00:00Z"). + reference_time: Optional reference time in RFC3339 format for asset aliasing. + max_results: Optional maximum number of results to return (default: 10000, max: 250000). + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + ValueError: If the time format is invalid. + + Requires the following IAM permission on the instance resource: + chronicle.legacies.legacyFindAssetEvents + """ + # Validate and parse the times to ensure they're in RFC3339 format + for time_str in [start_time, end_time, reference_time] if reference_time else [start_time, end_time]: + try: + datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as e: + if "does not match format" in str(e): + raise ValueError( + f"Time '{time_str}' must be in RFC3339 format (e.g., '2024-01-01T00:00:00Z')") from e + raise + + instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindAssetEvents" + + # Build query parameters + params = [ + f"assetIndicator={asset_indicator}", + f"timeRange.startTime={start_time}", + f"timeRange.endTime={end_time}" + ] + + if reference_time: + params.append(f"referenceTime={reference_time}") + + if max_results: + # Ensure max_results is within bounds + max_results = min(max(1, max_results), MAX_RESULTS_LIMIT) + params.append(f"maxResults={max_results}") + + url = f"{url}?{'&'.join(params)}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + result = response.json() + print(json.dumps(result, indent=2)) + + if result.get("more_data_available"): + print("\nWarning: More data is available but was not returned due to maxResults limit.") + + if result.get("uri"): + print("\nBackstory UI URLs:") + for uri in result["uri"]: + print(f" {uri}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--asset_indicator", + type=str, + required=True, + help="JSON string containing the asset indicator (e.g., '{\"hostname\": \"example.com\"}')") + parser.add_argument( + "--start_time", + type=str, + required=True, + help="Start time in RFC3339 format (e.g., '2024-01-01T00:00:00Z')") + parser.add_argument( + "--end_time", + type=str, + required=True, + help="End time in RFC3339 format (e.g., '2024-01-02T00:00:00Z')") + parser.add_argument( + "--reference_time", + type=str, + help="Optional reference time in RFC3339 format for asset aliasing") + parser.add_argument( + "--max_results", + type=int, + help=f"Maximum number of results to return (default: {DEFAULT_MAX_RESULTS}, max: {MAX_RESULTS_LIMIT})") + + args = parser.parse_args() + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + find_asset_events( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.asset_indicator, + args.start_time, + args.end_time, + args.reference_time, + args.max_results) diff --git a/search/v1alpha/raw_logs_find.py b/search/v1alpha/raw_logs_find.py new file mode 100644 index 00000000..bd974763 --- /dev/null +++ b/search/v1alpha/raw_logs_find.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for finding raw logs in Chronicle. + +API reference: +https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyFindRawLogs +""" +# pylint: enable=line-too-long + +import argparse +import json +from typing import List, Optional + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + +DEFAULT_MAX_RESPONSE_SIZE = 52428800 # 50MiB in bytes + + +def find_raw_logs( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + query: str, + batch_tokens: Optional[List[str]] = None, + log_ids: Optional[List[str]] = None, + regex_search: bool = False, + case_sensitive: bool = False, + max_response_size: Optional[int] = None) -> None: + """Find raw logs in Chronicle using the Legacy Find Raw Logs API. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + query: Required search parameters that expand or restrict the search. + batch_tokens: Optional list of tokens that should be downloaded. + log_ids: Optional list of raw log ids that should be downloaded. + If both batch_tokens and log_ids are provided, batch_tokens will be discarded. + regex_search: Optional boolean to treat query as regex. Default is False. + case_sensitive: Optional boolean for case-sensitive search. Default is False. + max_response_size: Optional maximum response size in bytes. Default is 50MiB. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the instance resource: + chronicle.legacies.legacyFindRawLogs + """ + instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindRawLogs" + + # Build query parameters + params = [f"query={query}"] + if batch_tokens and not log_ids: # log_ids take precedence over batch_tokens + for token in batch_tokens: + params.append(f"batchToken={token}") + if log_ids: + for log_id in log_ids: + params.append(f"ids={log_id}") + if regex_search: + params.append("regexSearch=true") + if case_sensitive: + params.append("caseSensitive=true") + if max_response_size: + params.append(f"maxResponseByteSize={max_response_size}") + + url = f"{url}?{'&'.join(params)}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + result = response.json() + print(json.dumps(result, indent=2)) + + if result.get("too_many_results"): + print("\nWarning: Some results were omitted due to too many matches.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--query", + type=str, + required=True, + help="Search parameters that expand or restrict the search") + parser.add_argument( + "--batch_tokens", + type=str, + help='JSON string containing a list of batch tokens (e.g., \'["token1", "token2"]\')') + parser.add_argument( + "--log_ids", + type=str, + help='JSON string containing a list of raw log IDs (e.g., \'["id1", "id2"]\')') + parser.add_argument( + "--regex_search", + action="store_true", + help="Whether to treat the query as a regex pattern") + parser.add_argument( + "--case_sensitive", + action="store_true", + help="Whether to perform a case-sensitive search") + parser.add_argument( + "--max_response_size", + type=int, + help=f"Maximum response size in bytes (default: {DEFAULT_MAX_RESPONSE_SIZE})") + + args = parser.parse_args() + + # Convert JSON strings to lists if provided + batch_tokens_list = json.loads(args.batch_tokens) if args.batch_tokens else None + log_ids_list = json.loads(args.log_ids) if args.log_ids else None + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + find_raw_logs( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.query, + batch_tokens_list, + log_ids_list, + args.regex_search, + args.case_sensitive, + args.max_response_size) From a0581fde6a02d4297f0b0b4c4480dd8ca8bb49cf Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:38:43 -0500 Subject: [PATCH 06/61] url_always_prepend_region --- ingestion/v1alpha/event_import.py | 7 ++++++- ingestion/v1alpha/events_batch_get.py | 7 ++++++- ingestion/v1alpha/events_get.py | 7 ++++++- search/v1alpha/asset_events_find.py | 7 ++++++- search/v1alpha/raw_logs_find.py | 7 ++++++- search/v1alpha/udm_events_find.py | 11 ++++++++--- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/ingestion/v1alpha/event_import.py b/ingestion/v1alpha/event_import.py index 03f4672f..f5f6cc15 100644 --- a/ingestion/v1alpha/event_import.py +++ b/ingestion/v1alpha/event_import.py @@ -32,6 +32,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -59,8 +60,12 @@ def import_events( Requires the following IAM permission on the parent resource: chronicle.events.import """ + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) parent = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{parent}/events:import" + url = f"{base_url_with_region}/v1alpha/{parent}/events:import" body = { "events": json.loads(json_events), diff --git a/ingestion/v1alpha/events_batch_get.py b/ingestion/v1alpha/events_batch_get.py index acbf00c9..64cd67b9 100644 --- a/ingestion/v1alpha/events_batch_get.py +++ b/ingestion/v1alpha/events_batch_get.py @@ -33,6 +33,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -60,8 +61,12 @@ def batch_get_events( Requires the following IAM permission on the parent resource: chronicle.events.batchGet """ + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) parent = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{parent}/events:batchGet" + url = f"{base_url_with_region}/v1alpha/{parent}/events:batchGet" # Convert event IDs to URL-encoded Base64 and create query parameters event_ids_list = json.loads(event_ids) diff --git a/ingestion/v1alpha/events_get.py b/ingestion/v1alpha/events_get.py index 5a501261..3e5d5536 100644 --- a/ingestion/v1alpha/events_get.py +++ b/ingestion/v1alpha/events_get.py @@ -32,6 +32,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -61,8 +62,12 @@ def get_event( """ # URL encode the event_id in Base64 encoded_event_id = base64.urlsafe_b64encode(event_id.encode()).decode() + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) name = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}/events/{encoded_event_id}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{name}" + url = f"{base_url_with_region}/v1alpha/{name}" response = http_session.request("GET", url) if response.status_code >= 400: diff --git a/search/v1alpha/asset_events_find.py b/search/v1alpha/asset_events_find.py index 6fc1801e..ef0e856f 100644 --- a/search/v1alpha/asset_events_find.py +++ b/search/v1alpha/asset_events_find.py @@ -34,6 +34,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -83,8 +84,12 @@ def find_asset_events( f"Time '{time_str}' must be in RFC3339 format (e.g., '2024-01-01T00:00:00Z')") from e raise + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindAssetEvents" + url = f"{base_url_with_region}/v1alpha/{instance}/legacy:legacyFindAssetEvents" # Build query parameters params = [ diff --git a/search/v1alpha/raw_logs_find.py b/search/v1alpha/raw_logs_find.py index bd974763..2d927e83 100644 --- a/search/v1alpha/raw_logs_find.py +++ b/search/v1alpha/raw_logs_find.py @@ -33,6 +33,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -73,8 +74,12 @@ def find_raw_logs( Requires the following IAM permission on the instance resource: chronicle.legacies.legacyFindRawLogs """ + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindRawLogs" + url = f"{base_url_with_region}/v1alpha/{instance}/legacy:legacyFindRawLogs" # Build query parameters params = [f"query={query}"] diff --git a/search/v1alpha/udm_events_find.py b/search/v1alpha/udm_events_find.py index 338ca1db..30d98fda 100644 --- a/search/v1alpha/udm_events_find.py +++ b/search/v1alpha/udm_events_find.py @@ -33,6 +33,7 @@ from common import project_instance from common import regions +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" SCOPES = [ "https://www.googleapis.com/auth/cloud-platform", ] @@ -64,11 +65,15 @@ def find_udm_events( requests.exceptions.HTTPError: HTTP request resulted in an error (response.status_code >= 400). - Requires the following IAM permission on the instance resource: - chronicle.legacies.legacyFindUdmEvents + Requires the following IAM permission on the parent resource: + chronicle.events.batchGet """ + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) instance = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" - url = f"https://{proj_region}-chronicle.googleapis.com/v1alpha/{instance}/legacy:legacyFindUdmEvents" + url = f"{base_url_with_region}/v1alpha/{instance}/legacy:legacyFindUdmEvents" # Build query parameters params = [] From 7d55b44c48cd56d36793cd954d2946e34891a0b5 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:41:19 -0500 Subject: [PATCH 07/61] get_detection --- detect/v1alpha/get_detection.py | 110 ++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 detect/v1alpha/get_detection.py diff --git a/detect/v1alpha/get_detection.py b/detect/v1alpha/get_detection.py new file mode 100644 index 00000000..cb0681d2 --- /dev/null +++ b/detect/v1alpha/get_detection.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +r"""Executable and reusable sample for getting a Detection. + +Usage: + python -m detect.v1alpha.get_detection \ + --project_id= \ + --project_instance= \ + --detection_id= + +# pylint: disable=line-too-long +API reference: + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyGetDetection +# pylint: enable=line-too-long +""" + +import argparse +import json +from typing import Any, Mapping + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +from google.auth.transport import requests + +CHRONICLE_API_BASE_URL = "https://chronicle.googleapis.com" +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def get_detection( + http_session: requests.AuthorizedSession, + proj_id: str, + proj_instance: str, + proj_region: str, + detection_id: str, +) -> Mapping[str, Any]: + """Gets a Detection. + + Args: + http_session: Authorized session for HTTP requests. + proj_id: GCP project id or number to which the target instance belongs. + proj_instance: Customer ID (uuid with dashes) for the Chronicle instance. + proj_region: region in which the target project is located. + detection_id: Identifier for the detection. + + Returns: + Dictionary representation of the Detection + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + """ + base_url_with_region = regions.url_always_prepend_region( + CHRONICLE_API_BASE_URL, + proj_region + ) + parent = f"projects/{proj_id}/locations/{proj_region}/instances/{proj_instance}" + url = f"{base_url_with_region}/v1alpha/{parent}/legacy:legacyGetDetection" + + query_params = {"detectionId": detection_id} + + response = http_session.request("GET", url, params=query_params) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + return response.json() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + parser.add_argument( + "--detection_id", type=str, required=True, + help="identifier for the detection" + ) + args = parser.parse_args() + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + detection = get_detection( + auth_session, + args.project_id, + args.project_instance, + args.region, + args.detection_id, + ) + print(json.dumps(detection, indent=2)) From a380a969849ad1d649887ec1c84eb7c8126765b6 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 6 Mar 2025 21:48:42 -0500 Subject: [PATCH 08/61] Add SDK wrapper --- README.md | 78 ++++++++- chronicle_api.egg-info/PKG-INFO | 8 + chronicle_api.egg-info/SOURCES.txt | 141 ++++++++++++++++ chronicle_api.egg-info/dependency_links.txt | 1 + chronicle_api.egg-info/entry_points.txt | 2 + chronicle_api.egg-info/requires.txt | 3 + chronicle_api.egg-info/top_level.txt | 10 ++ requirements.txt | 1 + sdk/cli.py | 70 ++++++++ sdk/commands/__init__.py | 1 + sdk/commands/detect.py | 88 ++++++++++ sdk/commands/ingestion.py | 102 ++++++++++++ sdk/commands/search.py | 173 ++++++++++++++++++++ setup.py | 39 +++++ 14 files changed, 716 insertions(+), 1 deletion(-) create mode 100644 chronicle_api.egg-info/PKG-INFO create mode 100644 chronicle_api.egg-info/SOURCES.txt create mode 100644 chronicle_api.egg-info/dependency_links.txt create mode 100644 chronicle_api.egg-info/entry_points.txt create mode 100644 chronicle_api.egg-info/requires.txt create mode 100644 chronicle_api.egg-info/top_level.txt create mode 100644 sdk/cli.py create mode 100644 sdk/commands/__init__.py create mode 100644 sdk/commands/detect.py create mode 100644 sdk/commands/ingestion.py create mode 100644 sdk/commands/search.py create mode 100644 setup.py diff --git a/README.md b/README.md index e6c0055c..ce606d8a 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,84 @@ python3 -m lists. -h ### Lists API v1alpha -``` +```shell python -m lists.v1alpha.create_list -h python -m lists.v1alpha.get_list -h python -m lists.v1alpha.patch_list -h ``` + +## SDK CLI Wrapper + +In addition to running individual sample scripts, you can use the unified CLI wrapper that provides access to all Chronicle APIs through a single command-line interface. + +### Installation + +Install the SDK in development mode: + +```shell +pip install -e . +``` + +This will install the `chronicle` command-line tool. + +### Usage + +The CLI follows this general pattern: +```shell +chronicle [common options] COMMAND_GROUP COMMAND [command options] +``` + +Common options (required for all commands): +- `--credentials-file`: Path to service account credentials file +- `--project-id`: GCP project id or number +- `--project-instance`: Customer ID for the Chronicle instance +- `--region`: Region of the target project + +Available command groups: + +1. Detection API (`detect`): +```shell +# Get an alert +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + detect get-alert --alert-id + +# Get a detection +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + detect get-detection --detection-id +``` + +2. Ingestion API (`ingestion`): +```shell +# Import events +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + ingestion import-events --json-events '' + +# Get an event +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + ingestion get-event --event-id + +# Batch get events +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + ingestion batch-get-events --event-ids '[,]' +``` + +3. Search API (`search`): +```shell +# Find asset events +chronicle --credentials-file creds.json --project-id proj --project-instance inst --region reg \ + search find-asset-events --asset-indicator --start-time