From c6c9922d4f8bc41792bc0714561621589bbdc4e0 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Tue, 18 Mar 2025 08:48:53 -0400 Subject: [PATCH 1/4] known working logs:import --- ingestion/v1alpha/logs_import.py | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ingestion/v1alpha/logs_import.py diff --git a/ingestion/v1alpha/logs_import.py b/ingestion/v1alpha/logs_import.py new file mode 100644 index 00000000..1ceca2e7 --- /dev/null +++ b/ingestion/v1alpha/logs_import.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_instance +from common import project_id +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + +#LOGS = base64.b64encode(b"""2024-01-29 09:03:06.000000001 client 192.168.109.254#12345: query: google.com IN A + (1.2.3.4) +#2024-01-29 09:03:07.000000001 client 192.168.109.252#12345: query: bbc.com IN A + (5.6.7.8) +#""").decode("utf-8") + +def logs_import(http_session: requests.AuthorizedSession, logs_file) -> None: + log_type = "GCP_CLOUDAUDIT" + parent = f"projects/{args.project_id}/" \ + f"locations/{args.region}/" \ + f"instances/{args.project_instance}/" \ + f"logTypes/{log_type}" + url = f"https://{args.region}-chronicle.googleapis.com/" \ + f"v1alpha/{parent}/logs:import" + logs = logs_file.read() + # Reset file pointer to beginning in case it needs to be read again + logs_file.seek(0) + logs = base64.b64encode(logs.encode("utf-8")).decode("utf-8") + body = { + "inline_source": { + "logs": [ + { + "data": logs, + "log_entry_time": "2025-01-29T15:01:23.045123456Z", + "collection_time": "2025-01-29T16:01:23.045123456Z", + }, + ], + "forwarder": f"projects/{args.project_id}/" + f"locations/{args.region}/" + f"instances/{args.project_instance}/" + f"forwarders/{args.forwarder_id}" + } + } + response = http_session.request("POST", url, json=body) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + print(response.status_code) + return response.json() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--forwarder_id", + type=str, + required=True, + help="UUID4 of the forwarder") + parser.add_argument( + "--logs_file", + type=argparse.FileType("r"), + required=True, + help="path to a log file (or \"-\" for STDIN)") + args = parser.parse_args() + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + print(json.dumps(logs_import(auth_session, args.logs_file))) From 033b1a0c055ea4116eb41410930f6a62829b0507 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Tue, 18 Mar 2025 08:52:09 -0400 Subject: [PATCH 2/4] logging --- ingestion/v1alpha/logs_import.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/ingestion/v1alpha/logs_import.py b/ingestion/v1alpha/logs_import.py index 1ceca2e7..30da5fef 100644 --- a/ingestion/v1alpha/logs_import.py +++ b/ingestion/v1alpha/logs_import.py @@ -2,7 +2,9 @@ import argparse import base64 +import datetime import json +import logging from google.auth.transport import requests @@ -15,11 +17,8 @@ "https://www.googleapis.com/auth/cloud-platform", ] -#LOGS = base64.b64encode(b"""2024-01-29 09:03:06.000000001 client 192.168.109.254#12345: query: google.com IN A + (1.2.3.4) -#2024-01-29 09:03:07.000000001 client 192.168.109.252#12345: query: bbc.com IN A + (5.6.7.8) -#""").decode("utf-8") -def logs_import(http_session: requests.AuthorizedSession, logs_file) -> None: +def logs_import(http_session: requests.AuthorizedSession, logs_file) -> dict: log_type = "GCP_CLOUDAUDIT" parent = f"projects/{args.project_id}/" \ f"locations/{args.region}/" \ @@ -31,13 +30,14 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> None: # Reset file pointer to beginning in case it needs to be read again logs_file.seek(0) logs = base64.b64encode(logs.encode("utf-8")).decode("utf-8") + now = datetime.datetime.now(datetime.timezone.utc).isoformat() body = { "inline_source": { "logs": [ { "data": logs, - "log_entry_time": "2025-01-29T15:01:23.045123456Z", - "collection_time": "2025-01-29T16:01:23.045123456Z", + "log_entry_time": now, + "collection_time": now, }, ], "forwarder": f"projects/{args.project_id}/" @@ -48,13 +48,20 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> None: } response = http_session.request("POST", url, json=body) if response.status_code >= 400: - print(response.text) + logging.error(f"Error response: {response.text}") response.raise_for_status() - print(response.status_code) + logging.info(f"Request successful with status code: {response.status_code}") return response.json() if __name__ == "__main__": + # Configure logging + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + logger = logging.getLogger(__name__) + parser = argparse.ArgumentParser() # common chronicle_auth.add_argument_credentials_file(parser) @@ -77,4 +84,9 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> None: args.credentials_file, SCOPES, ) - print(json.dumps(logs_import(auth_session, args.logs_file))) + try: + result = logs_import(auth_session, args.logs_file) + logging.info("Import operation completed successfully") + print(json.dumps(result, indent=2)) + except Exception as e: + logging.error(f"Import operation failed: {str(e)}") From 7917ce2c146d672e94d2c337dcbd8c3a7b5c549e Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Tue, 18 Mar 2025 08:55:29 -0400 Subject: [PATCH 3/4] Googly python style updates --- ingestion/v1alpha/logs_import.py | 85 ++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/ingestion/v1alpha/logs_import.py b/ingestion/v1alpha/logs_import.py index 30da5fef..26bfcbf2 100644 --- a/ingestion/v1alpha/logs_import.py +++ b/ingestion/v1alpha/logs_import.py @@ -18,14 +18,31 @@ ] -def logs_import(http_session: requests.AuthorizedSession, logs_file) -> dict: +def logs_import(http_session: requests.AuthorizedSession, logs_file, project_id: str, + region: str, project_instance: str, forwarder_id: str) -> dict: + """Imports logs to Chronicle using the GCP CLOUDAUDIT log type. + + Args: + http_session: Authorized session for HTTP requests. + logs_file: File-like object containing the logs to import. + project_id: Google Cloud project ID. + region: Chronicle region. + project_instance: Chronicle instance. + forwarder_id: UUID4 of the forwarder. + + Returns: + dict: JSON response from the API. + + Raises: + requests.HTTPError: If the request fails. + """ log_type = "GCP_CLOUDAUDIT" - parent = f"projects/{args.project_id}/" \ - f"locations/{args.region}/" \ - f"instances/{args.project_instance}/" \ - f"logTypes/{log_type}" - url = f"https://{args.region}-chronicle.googleapis.com/" \ - f"v1alpha/{parent}/logs:import" + parent = (f"projects/{project_id}/" + f"locations/{region}/" + f"instances/{project_instance}/" + f"logTypes/{log_type}") + url = (f"https://{region}-chronicle.googleapis.com/" + f"v1alpha/{parent}/logs:import") logs = logs_file.read() # Reset file pointer to beginning in case it needs to be read again logs_file.seek(0) @@ -38,23 +55,24 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> dict: "data": logs, "log_entry_time": now, "collection_time": now, - }, + }, ], - "forwarder": f"projects/{args.project_id}/" - f"locations/{args.region}/" - f"instances/{args.project_instance}/" - f"forwarders/{args.forwarder_id}" + "forwarder": (f"projects/{project_id}/" + f"locations/{region}/" + f"instances/{project_instance}/" + f"forwarders/{forwarder_id}") } } response = http_session.request("POST", url, json=body) if response.status_code >= 400: - logging.error(f"Error response: {response.text}") + logging.error("Error response: %s", response.text) response.raise_for_status() - logging.info(f"Request successful with status code: {response.status_code}") + logging.info("Request successful with status code: %d", response.status_code) return response.json() -if __name__ == "__main__": +def main(): + """Main entry point for the logs import script.""" # Configure logging logging.basicConfig( level=logging.INFO, @@ -62,7 +80,7 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> dict: ) logger = logging.getLogger(__name__) - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description="Import logs to Chronicle.") # common chronicle_auth.add_argument_credentials_file(parser) project_instance.add_argument_project_instance(parser) @@ -70,23 +88,36 @@ def logs_import(http_session: requests.AuthorizedSession, logs_file) -> dict: regions.add_argument_region(parser) # local parser.add_argument( - "--forwarder_id", - type=str, - required=True, - help="UUID4 of the forwarder") + "--forwarder_id", + type=str, + required=True, + help="UUID4 of the forwarder") parser.add_argument( - "--logs_file", - type=argparse.FileType("r"), - required=True, - help="path to a log file (or \"-\" for STDIN)") + "--logs_file", + type=argparse.FileType("r"), + required=True, + help="path to a log file (or \"-\" for STDIN)") args = parser.parse_args() auth_session = chronicle_auth.initialize_http_session( args.credentials_file, SCOPES, ) try: - result = logs_import(auth_session, args.logs_file) + result = logs_import( + auth_session, + args.logs_file, + args.project_id, + args.region, + args.project_instance, + args.forwarder_id + ) logging.info("Import operation completed successfully") print(json.dumps(result, indent=2)) - except Exception as e: - logging.error(f"Import operation failed: {str(e)}") + except Exception as e: # pylint: disable=broad-except + logging.error("Import operation failed: %s", str(e)) + return 1 + return 0 + + +if __name__ == "__main__": + main() From b703f094745dc042aaa73e4436c01528518ca189 Mon Sep 17 00:00:00 2001 From: DanDye Date: Thu, 20 Mar 2025 12:25:03 -0400 Subject: [PATCH 4/4] Add copyright and license --- ingestion/v1alpha/logs_import.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ingestion/v1alpha/logs_import.py b/ingestion/v1alpha/logs_import.py index 26bfcbf2..c988a0e0 100644 --- a/ingestion/v1alpha/logs_import.py +++ b/ingestion/v1alpha/logs_import.py @@ -1,5 +1,19 @@ #!/usr/bin/env python3 +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import base64 import datetime