From 5fd995a08ca5db76ef4f96215f118b47cf2eaaac Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 06:55:55 +0000 Subject: [PATCH 1/8] chore: Prevent OS Login key accumulation in e2e tests --- cloudbuild/zb-system-tests-cloudbuild.yaml | 32 ++++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index be790ebd4..e81c6e1d7 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -4,6 +4,20 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} steps: + # Step 0: Generate a persistent SSH key for this build run. + # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + # Generate the SSH key + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + # Save the public key content to a file for the cleanup step + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + waitFor: ["-"] # Step 1 Create a GCE VM to run the tests. # The VM is created in the same zone as the buckets to test rapid storage features. @@ -38,19 +52,19 @@ steps: set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do - if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="echo VM is ready"; then + if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then break fi echo "Waiting for VM to become available... (attempt $i/10)" sleep 15 done # copy the script to the VM - gcloud compute scp cloudbuild/run_zonal_tests.sh gcsfs-test-vm-${_SHORT_BUILD_ID}:~ --zone=${_ZONE} --internal-ip + gcloud compute scp cloudbuild/run_zonal_tests.sh gcsfs-test-vm-${_SHORT_BUILD_ID}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} bash run_zonal_tests.sh" + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} bash run_zonal_tests.sh" EXIT_CODE=$? set -e @@ -62,6 +76,18 @@ steps: waitFor: - "create-vm" + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "run-tests-and-delete-vm" + timeout: "3600s" # 60 minutes options: From f36c8e374f9312418679f0adb15db3c49eba093a Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 07:57:50 +0000 Subject: [PATCH 2/8] wait for "generate-ssh-key" --- cloudbuild/zb-system-tests-cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index e81c6e1d7..832c0b63f 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -75,6 +75,7 @@ steps: exit $$EXIT_CODE waitFor: - "create-vm" + - "generate-ssh-key" - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "cleanup-ssh-key" From 190b58443e447d40dc31186c6ef766e807717077 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 08:05:19 +0000 Subject: [PATCH 3/8] change vm name and keep it in a variable --- cloudbuild/zb-system-tests-cloudbuild.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 832c0b63f..5138e1144 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -2,6 +2,7 @@ substitutions: _REGION: "us-central1" _ZONE: "us-central1-a" _SHORT_BUILD_ID: ${BUILD_ID:0:8} + _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}" steps: # Step 0: Generate a persistent SSH key for this build run. @@ -29,7 +30,7 @@ steps: - "compute" - "instances" - "create" - - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "${_VM_NAME}" - "--project=${PROJECT_ID}" - "--zone=${_ZONE}" - "--machine-type=e2-medium" @@ -52,24 +53,24 @@ steps: set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do - if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then + if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then break fi echo "Waiting for VM to become available... (attempt $i/10)" sleep 15 done # copy the script to the VM - gcloud compute scp cloudbuild/run_zonal_tests.sh gcsfs-test-vm-${_SHORT_BUILD_ID}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + gcloud compute scp cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} bash run_zonal_tests.sh" + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} bash run_zonal_tests.sh" EXIT_CODE=$? set -e echo "--- Deleting GCE VM ---" - gcloud compute instances delete "gcsfs-test-vm-${_SHORT_BUILD_ID}" --zone=${_ZONE} --quiet + gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet # Exit with the original exit code from the test script. exit $$EXIT_CODE @@ -94,4 +95,4 @@ timeout: "3600s" # 60 minutes options: logging: CLOUD_LOGGING_ONLY pool: - name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" \ No newline at end of file + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" From 3eb8d7af1233e99518e0f0d4a44aba1a30126e74 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 08:17:46 +0000 Subject: [PATCH 4/8] describe profile to debug --- cloudbuild/zb-system-tests-cloudbuild.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 5138e1144..1927c6cb8 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -4,7 +4,17 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}" + + steps: +- name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "os-login" + - "describe-profile" + waitFor: ["-"] # Step 0: Generate a persistent SSH key for this build run. # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" From f9a85ea7e699ba7d5802bae4687c1be09f37d853 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 08:19:52 +0000 Subject: [PATCH 5/8] fix indent --- cloudbuild/zb-system-tests-cloudbuild.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 1927c6cb8..51eebedb4 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -7,14 +7,14 @@ substitutions: steps: -- name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-vm" - entrypoint: "gcloud" - args: - - "compute" - - "os-login" - - "describe-profile" - waitFor: ["-"] + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "os-login" + - "describe-profile" + waitFor: ["-"] # Step 0: Generate a persistent SSH key for this build run. # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" From c7942117b8f40ed2f5ad99d580cab7451abe2d13 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 08:21:09 +0000 Subject: [PATCH 6/8] make cloudbuild steps unique --- cloudbuild/zb-system-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 51eebedb4..c4b27abdf 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -8,7 +8,7 @@ substitutions: steps: - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-vm" + id: "describe-profile" entrypoint: "gcloud" args: - "compute" From ddc649a35530856f548addae2ecfdd6fa1adb4e9 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 08:49:36 +0000 Subject: [PATCH 7/8] remove os login describe step --- cloudbuild/zb-system-tests-cloudbuild.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index c4b27abdf..383c4fa96 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -7,14 +7,6 @@ substitutions: steps: - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "describe-profile" - entrypoint: "gcloud" - args: - - "compute" - - "os-login" - - "describe-profile" - waitFor: ["-"] # Step 0: Generate a persistent SSH key for this build run. # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" From fe3851d55fc1ba7d6cbe741871ee97fee4ae578e Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 06:26:58 +0000 Subject: [PATCH 8/8] skip notification tests until b/470069573 is fixed --- tests/system/test_notification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/system/test_notification.py b/tests/system/test_notification.py index 9b631c29b..c21d836a3 100644 --- a/tests/system/test_notification.py +++ b/tests/system/test_notification.py @@ -69,6 +69,7 @@ def notification_topic(storage_client, publisher_client, topic_path, no_mtls): publisher_client.set_iam_policy(request={"resource": topic_path, "policy": policy}) +@pytest.mark.skip(reason="until b/470069573 is fixed") def test_notification_create_minimal( storage_client, buckets_to_delete, @@ -94,6 +95,7 @@ def test_notification_create_minimal( notification.delete() +@pytest.mark.skip(reason="until b/470069573 is fixed") def test_notification_create_explicit( storage_client, buckets_to_delete, @@ -128,6 +130,7 @@ def test_notification_create_explicit( notification.delete() +@pytest.mark.skip(reason="until b/470069573 is fixed") def test_notification_create_w_user_project( storage_client, buckets_to_delete, @@ -156,6 +159,7 @@ def test_notification_create_w_user_project( notification.delete() +@pytest.mark.skip(reason="until b/470069573 is fixed") def test_notification_create_wo_topic_name( storage_client, buckets_to_delete, @@ -184,6 +188,7 @@ def test_notification_create_wo_topic_name( notification.create() +@pytest.mark.skip(reason="until b/470069573 is fixed") def test_bucket_get_notification( storage_client, buckets_to_delete,