From 2c27bbf96ea459b1dc260b378c52318d9e08be2a Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 17 Dec 2025 21:06:00 +0100 Subject: [PATCH 01/26] Add integration tests to simtools-prod --- .github/workflows/CI-integrationtests.yml | 34 ++++++++++++++--------- .github/workflows/build-simtools-prod.yml | 19 +++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 95a97f53e6..fe24e7bd7c 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -10,6 +10,18 @@ on: required: false type: string default: 'main' + workflow_call: + inputs: + container_image: + description: 'Container image to use for tests (default: simtools-dev:latest)' + required: false + type: string + default: 'ghcr.io/gammasim/simtools-dev:latest' + model_versions: + description: 'JSON array of model versions to test (e.g. ["7.0.0","6.0.2"])' + required: false + type: string + default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' pull_request: types: [opened, synchronize] paths-ignore: @@ -45,7 +57,7 @@ jobs: integration_tests: runs-on: ubuntu-latest container: - image: ghcr.io/gammasim/simtools-dev:latest + image: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }} options: --user 0 services: @@ -63,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - model_version: ['7.0.0', '6.0.2', '5.0.0', '6.0.2,6.1.1'] + model_version: ${{ fromJSON(inputs.model_versions || '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]') }} defaults: run: @@ -153,23 +165,19 @@ jobs: echo "DB Simulation Model Version: $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" echo "DB Simulation Model Branch: $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" + - name: Install simtools (dev container only) + if: github.event_name != 'schedule' && !contains(inputs.container_image, 'simtools-prod') + run: | + pip install --no-cache-dir -e '.[tests,dev,doc]' + - name: Upload data to MongoDB if: github.event_name != 'schedule' run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e '.[tests,dev,doc]' simtools-db-upload-model-repository \ --db_simulation_model ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} \ --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \ --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" - - name: Print versions - run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e . - cat .env - simtools-print-version - - name: Integration tests shell: bash -l {0} env: @@ -177,7 +185,7 @@ jobs: SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} SIMTOOLS_DB_SERVER: ${{ github.event_name == 'schedule' && secrets.DB_SERVER || 'mongodb' }} run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e '.[tests,dev,doc]' + cat .env + simtools-print-version pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \ -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 3ca45d658e..851ea5fbee 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -22,6 +22,8 @@ jobs: permissions: contents: read packages: write + outputs: + no_opt_tag: ${{ steps.export-tags.outputs.no_opt_tag }} strategy: matrix: version: @@ -68,6 +70,15 @@ jobs: images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} flavor: latest=false + - name: Export no_opt tag for testing + id: export-tags + if: matrix.version.avx_flag == 'no_opt' + run: | + TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) + TAG=${TAG##*/} + echo "no_opt_tag=$TAG" >> "$GITHUB_OUTPUT" + echo "Exported no_opt tag: $TAG" + - name: Build and push uses: docker/build-push-action@v6 with: @@ -83,3 +94,11 @@ jobs: file: ./docker/Dockerfile-simtools-prod tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + test-simtools-prod: + needs: build-simtools-prod + if: github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/CI-integrationtests.yml + with: + container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }} + model_versions: '["7.0.0","6.0.2"]' From f213ae20948c170faa2ca8ff5080e9a2b3fb6e1e Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 17 Dec 2025 21:15:01 +0100 Subject: [PATCH 02/26] avoid test building --- .github/workflows/CI-integrationtests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index fe24e7bd7c..1de5c732f5 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -37,6 +37,7 @@ jobs: test_building: # Build the package and check if it is installable # (tests among others that all components are there) + if: github.event_name != 'workflow_call' runs-on: ubuntu-latest defaults: From f1b346d684120036d70a8ff8d47e220d7ba604e9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 09:36:13 +0100 Subject: [PATCH 03/26] testing --- .github/workflows/build-simtools-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 851ea5fbee..6e73f55596 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -5,8 +5,8 @@ name: build-simtools-prod on: workflow_dispatch: pull_request: - paths: - - docker/Dockerfile-simtools-prod + # paths: + # - docker/Dockerfile-simtools-prod release: types: [published] schedule: From fdff2775af91b68d4eebd1c39c5173dbeef61e28 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 09:37:20 +0100 Subject: [PATCH 04/26] testing --- .github/workflows/build-simtools-prod.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 6e73f55596..69b980c9f1 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -5,7 +5,7 @@ name: build-simtools-prod on: workflow_dispatch: pull_request: - # paths: + # TODO paths: # - docker/Dockerfile-simtools-prod release: types: [published] @@ -84,7 +84,8 @@ jobs: with: context: . # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific - platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + # TODO platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + platforms: 'linux/arm64/v8' build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} From 305791adeda9bf0193a3c2d335662c7f0a6111b8 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 09:38:14 +0100 Subject: [PATCH 05/26] testing --- .github/workflows/build-simtools-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 69b980c9f1..5fec811019 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -98,7 +98,7 @@ jobs: test-simtools-prod: needs: build-simtools-prod - if: github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + # TODO if: github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' uses: ./.github/workflows/CI-integrationtests.yml with: container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }} From de2edde01acb2a5a61322cdc13bf4887cc026b5f Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 09:47:36 +0100 Subject: [PATCH 06/26] testing --- .github/workflows/build-simtools-prod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 5fec811019..b4d27f060b 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -92,6 +92,7 @@ jobs: AVX_FLAG=${{ matrix.version.avx_flag }} SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + load: true file: ./docker/Dockerfile-simtools-prod tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 2d4a9eee702902d85b11efe3a72d3cb5963e2e85 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 09:57:59 +0100 Subject: [PATCH 07/26] Amd! --- .github/workflows/build-simtools-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 9a93d493e1..8583081288 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -85,7 +85,7 @@ jobs: context: . # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific # TODO platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} - platforms: 'linux/arm64/v8' + platforms: 'linux/amd64' build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} From 11d27cff6bfc63ea223cd2e2ba8b5e771743d6ce Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 10:08:53 +0100 Subject: [PATCH 08/26] tag change --- .github/workflows/build-simtools-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 8583081288..e4a27ee7c4 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -75,7 +75,7 @@ jobs: if: matrix.version.avx_flag == 'no_opt' run: | TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - TAG=${TAG##*/} + TAG=${TAG##*:} echo "no_opt_tag=$TAG" >> "$GITHUB_OUTPUT" echo "Exported no_opt tag: $TAG" From 55633bef182f16f43d6a061e22907f8f30b522fd Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 10:28:21 +0100 Subject: [PATCH 09/26] run tests --- .github/workflows/build-simtools-prod.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index e4a27ee7c4..1e711e1e6d 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -91,7 +91,7 @@ jobs: SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} AVX_FLAG=${{ matrix.version.avx_flag }} SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} - push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }} load: true file: ./docker/Dockerfile-simtools-prod tags: ${{ steps.meta.outputs.tags }} @@ -99,7 +99,6 @@ jobs: test-simtools-prod: needs: build-simtools-prod - # TODO if: github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' uses: ./.github/workflows/CI-integrationtests.yml with: container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }} From 9bf9207fd4d8847925fcc97a7844650fd63676bc Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 11:05:50 +0100 Subject: [PATCH 10/26] path --- .github/workflows/CI-integrationtests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 1de5c732f5..d0900c9557 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -88,9 +88,11 @@ jobs: with: fetch-depth: 0 - - name: Set sim_telarray path + - name: Set PATH entries run: | - echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV" + echo "/usr/bin" >> "$GITHUB_PATH" + echo "/usr/local/bin" >> "$GITHUB_PATH" + if [ -n "$SIMTOOLS_SIMTEL_PATH" ]; then echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH"; fi - name: Create environment file (remote DB; schedule only) if: ${{ github.event_name == 'schedule' }} From f3fefd033c5dd406a60bfa14797ce6e6783ea7f7 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 11:11:59 +0100 Subject: [PATCH 11/26] pytest --- .github/workflows/CI-integrationtests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index d0900c9557..ec30c659d0 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -173,6 +173,11 @@ jobs: run: | pip install --no-cache-dir -e '.[tests,dev,doc]' + - name: Install pytest (prod container only) + if: contains(inputs.container_image, 'simtools-prod') + run: | + pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist + - name: Upload data to MongoDB if: github.event_name != 'schedule' run: | From 25d5447a59f43e6310bfdbb9707d148e689c044d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 11:18:31 +0100 Subject: [PATCH 12/26] pytest --- .github/workflows/CI-integrationtests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index ec30c659d0..ed89cdb145 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -176,7 +176,7 @@ jobs: - name: Install pytest (prod container only) if: contains(inputs.container_image, 'simtools-prod') run: | - pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist + pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist pytest-retry - name: Upload data to MongoDB if: github.event_name != 'schedule' From 449b7097dd7aedc67593ac9751e494e0a7f7edb2 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 12:16:47 +0100 Subject: [PATCH 13/26] simplify --- .github/workflows/CI-integrationtests.yml | 168 ++++++++++------------ 1 file changed, 75 insertions(+), 93 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index ed89cdb145..e4efe6eb31 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -1,6 +1,5 @@ --- name: CI-integrationtests -# Integration tests for applications on: workflow_dispatch: @@ -10,34 +9,36 @@ on: required: false type: string default: 'main' + workflow_call: inputs: container_image: - description: 'Container image to use for tests (default: simtools-dev:latest)' + description: 'Container image to use for tests' required: false type: string default: 'ghcr.io/gammasim/simtools-dev:latest' model_versions: - description: 'JSON array of model versions to test (e.g. ["7.0.0","6.0.2"])' + description: 'Model versions to test' required: false type: string default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' + pull_request: types: [opened, synchronize] paths-ignore: - 'docs/**' - 'tests/unit_tests/**' + schedule: - cron: "0 0 * * *" + release: types: [published] jobs: test_building: - # Build the package and check if it is installable - # (tests among others that all components are there) - if: github.event_name != 'workflow_call' + if: ${{ github.event_name != 'workflow_call' }} runs-on: ubuntu-latest defaults: @@ -45,14 +46,11 @@ jobs: shell: bash -leo pipefail {0} steps: + - uses: actions/checkout@v6 - - name: checkout - uses: actions/checkout@v6 - - - name: Build the package + - name: Build package run: | - python -m pip install --upgrade pip - python -m pip install --upgrade build + python -m pip install --upgrade pip build python -m build integration_tests: @@ -83,117 +81,101 @@ jobs: shell: bash -leo pipefail {0} steps: - - name: checkout - uses: actions/checkout@v6 + - uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Set PATH entries + - name: Extend PATH (optional) run: | - echo "/usr/bin" >> "$GITHUB_PATH" - echo "/usr/local/bin" >> "$GITHUB_PATH" - if [ -n "$SIMTOOLS_SIMTEL_PATH" ]; then echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH"; fi + [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" - - name: Create environment file (remote DB; schedule only) - if: ${{ github.event_name == 'schedule' }} - run: | - { - echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}" - echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}" - echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}" - echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env - - - name: Extract SIMTOOLS_DB_SIMULATION_MODEL - run: | - SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"') - SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"') - SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" - echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - - - name: Create environment file (local DB) - if: github.event_name != 'schedule' - run: | - { - echo "SIMTOOLS_DB_SERVER=mongodb" - echo "SIMTOOLS_DB_API_USER=api" - echo "SIMTOOLS_DB_API_PW=password" - echo "SIMTOOLS_DB_API_PORT=27017" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env - - - name: Determine simulation model branch + - name: Create runtime environment (.env) env: - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} EVENT_NAME: ${{ github.event_name }} INPUT_BRANCH: ${{ github.event.inputs.simulation_model_branch }} + HEAD_REF: ${{ github.head_ref }} + REF_NAME: ${{ github.ref_name }} + SECRET_DB_SERVER: ${{ secrets.DB_SERVER }} + SECRET_DB_API_USER: ${{ secrets.DB_API_USER }} + SECRET_DB_API_PW: ${{ secrets.DB_API_PW }} + SECRET_DB_API_PORT: ${{ secrets.DB_API_PORT }} run: | + set -a + source .env_template + set +a + + DB_SERVER=mongodb + DB_API_USER=api + DB_API_PW=password + DB_API_PORT=27017 + + if [ "$EVENT_NAME" = "schedule" ]; then + DB_SERVER="$SECRET_DB_SERVER" + DB_API_USER="$SECRET_DB_API_USER" + DB_API_PW="$SECRET_DB_API_PW" + DB_API_PORT="$SECRET_DB_API_PORT" + fi + + # Determine simulation model branch + BRANCH="main" + if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_BRANCH" ]; then BRANCH="$INPUT_BRANCH" - else - BRANCH="main" fi + + CURRENT_REF="$REF_NAME" if [ "$EVENT_NAME" = "pull_request" ]; then CURRENT_REF="$HEAD_REF" - else - CURRENT_REF="$REF_NAME" fi - # release candidates (e.g., v1.2.3-rc or v1.2.3-rc3) should be tested against released versions of the - # simulation model database + if [[ "$CURRENT_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]*$ ]]; then - echo "Detected RC version branch: $CURRENT_REF - setting branch to $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" BRANCH="$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" fi - echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV" - - name: Print DB connection variables - run: | - echo "Event name: ${{ github.event_name }}" - echo "DB Server: $SIMTOOLS_DB_SERVER" - echo "DB API User: $SIMTOOLS_DB_API_USER" - echo "DB API Port: $SIMTOOLS_DB_API_PORT" - echo "DB Simulation Model: $SIMTOOLS_DB_SIMULATION_MODEL" - echo "DB Simulation Model Version: $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - echo "DB Simulation Model Branch: $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" - - - name: Install simtools (dev container only) - if: github.event_name != 'schedule' && !contains(inputs.container_image, 'simtools-prod') - run: | - pip install --no-cache-dir -e '.[tests,dev,doc]' + cat > .env < Date: Thu, 18 Dec 2025 12:31:57 +0100 Subject: [PATCH 14/26] sim --- .github/workflows/CI-integrationtests.yml | 121 ++++++++++++---------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index e4efe6eb31..1e730094af 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -9,7 +9,6 @@ on: required: false type: string default: 'main' - workflow_call: inputs: container_image: @@ -22,16 +21,13 @@ on: required: false type: string default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' - pull_request: types: [opened, synchronize] paths-ignore: - 'docs/**' - 'tests/unit_tests/**' - schedule: - cron: "0 0 * * *" - release: types: [published] @@ -90,64 +86,76 @@ jobs: [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" - name: Create runtime environment (.env) + if: ${{ github.event_name == 'schedule' }} + run: | + { + echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}" + echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}" + echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}" + echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}" + echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" + } > .env + + - name: Extract SIMTOOLS_DB_SIMULATION_MODEL + run: | + SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"') + SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"') + SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}" + echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" + echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" + echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" + + - name: Create environment file (local DB) + if: github.event_name != 'schedule' + run: | + { + echo "SIMTOOLS_DB_SERVER=mongodb" + echo "SIMTOOLS_DB_API_USER=api" + echo "SIMTOOLS_DB_API_PW=password" + echo "SIMTOOLS_DB_API_PORT=27017" + echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}" + echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}" + echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" + echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" + } > .env + + - name: Determine simulation model branch env: - EVENT_NAME: ${{ github.event_name }} - INPUT_BRANCH: ${{ github.event.inputs.simulation_model_branch }} HEAD_REF: ${{ github.head_ref }} REF_NAME: ${{ github.ref_name }} - SECRET_DB_SERVER: ${{ secrets.DB_SERVER }} - SECRET_DB_API_USER: ${{ secrets.DB_API_USER }} - SECRET_DB_API_PW: ${{ secrets.DB_API_PW }} - SECRET_DB_API_PORT: ${{ secrets.DB_API_PORT }} + EVENT_NAME: ${{ github.event_name }} + INPUT_BRANCH: ${{ github.event.inputs.simulation_model_branch }} run: | - set -a - source .env_template - set +a - - DB_SERVER=mongodb - DB_API_USER=api - DB_API_PW=password - DB_API_PORT=27017 - - if [ "$EVENT_NAME" = "schedule" ]; then - DB_SERVER="$SECRET_DB_SERVER" - DB_API_USER="$SECRET_DB_API_USER" - DB_API_PW="$SECRET_DB_API_PW" - DB_API_PORT="$SECRET_DB_API_PORT" - fi - - # Determine simulation model branch BRANCH="main" - if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_BRANCH" ]; then BRANCH="$INPUT_BRANCH" fi - CURRENT_REF="$REF_NAME" if [ "$EVENT_NAME" = "pull_request" ]; then CURRENT_REF="$HEAD_REF" fi - + # release candidates (e.g., v1.2.3-rc or v1.2.3-rc3) should be tested against released versions of the + # simulation model database if [[ "$CURRENT_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]*$ ]]; then + echo "Detected RC version branch: $CURRENT_REF - setting branch to $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" BRANCH="$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" fi + echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV" - cat > .env < Date: Thu, 18 Dec 2025 12:36:12 +0100 Subject: [PATCH 15/26] path --- .github/workflows/CI-integrationtests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 1e730094af..6e404fc601 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -81,10 +81,6 @@ jobs: with: fetch-depth: 0 - - name: Extend PATH (optional) - run: | - [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" - - name: Create runtime environment (.env) if: ${{ github.event_name == 'schedule' }} run: | @@ -124,6 +120,11 @@ jobs: echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" } > .env + - name: Extend PATH (simtel) + run: | + source .env + [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" + - name: Determine simulation model branch env: HEAD_REF: ${{ github.head_ref }} From 03b5c5918f94e8d76b8e6af3902cbe7aaf861480 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 13:08:11 +0100 Subject: [PATCH 16/26] download qgsjet tables --- .github/workflows/CI-integrationtests.yml | 11 +++++++++++ docker/Dockerfile-simtools-prod | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 6e404fc601..576112310e 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -31,6 +31,9 @@ on: release: types: [published] +env: + CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" + jobs: test_building: @@ -176,6 +179,14 @@ jobs: --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \ --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" + - name: Download QGSJet (prod images without tables) + if: contains(inputs.container_image, 'simtools-prod') + run: | + wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download -O qgsdat-III.bz2 + bunzip2 qgsdat-III.bz2 + source .env + mv -v -f qgsdat-III ${{ env.SIMTOOLS_CORSIKA_PATH }}/ + - name: Integration tests shell: bash -l {0} env: diff --git a/docker/Dockerfile-simtools-prod b/docker/Dockerfile-simtools-prod index 7498bed833..ee9a95b0ec 100644 --- a/docker/Dockerfile-simtools-prod +++ b/docker/Dockerfile-simtools-prod @@ -27,7 +27,7 @@ COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulatio COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools RUN microdnf update -y && microdnf install -y \ - bc findutils gcc-c++ git gsl libgfortran procps wget zstd \ + bc bzip2 findutils gcc-c++ git gsl libgfortran procps wget zstd \ python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ microdnf clean all && \ rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ From 35ada5918f931681344bf97d84337bf6d0001ae9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 13:22:27 +0100 Subject: [PATCH 17/26] inherit secrets --- .github/workflows/CI-integrationtests.yml | 11 +++++++++++ .github/workflows/build-simtools-prod.yml | 1 + 2 files changed, 12 insertions(+) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 576112310e..e67f720d4e 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -21,6 +21,17 @@ on: required: false type: string default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' + secrets: + CLOUD_QGSJET3: + required: false + DB_SERVER: + required: false + DB_API_USER: + required: false + DB_API_PW: + required: false + DB_API_PORT: + required: false pull_request: types: [opened, synchronize] paths-ignore: diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 1e711e1e6d..b43f863c9e 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -100,6 +100,7 @@ jobs: test-simtools-prod: needs: build-simtools-prod uses: ./.github/workflows/CI-integrationtests.yml + secrets: inherit with: container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }} model_versions: '["7.0.0","6.0.2"]' From 007ba16b23aa08116e4431a59ae2c09fd4b3041c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 13:59:51 +0100 Subject: [PATCH 18/26] testing --- .github/workflows/CI-integrationtests.yml | 64 +++++++++++------------ 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index e67f720d4e..d6aaa8e1a2 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -96,43 +96,38 @@ jobs: fetch-depth: 0 - name: Create runtime environment (.env) - if: ${{ github.event_name == 'schedule' }} run: | - { - echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}" - echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}" - echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}" - echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env - - - name: Extract SIMTOOLS_DB_SIMULATION_MODEL - run: | - SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"') - SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"') + set -a + source .env_template + set +a + + DB_SERVER="mongodb" + DB_API_USER="api" + DB_API_PW="password" + DB_API_PORT="27017" + if [ "${{ github.event_name }}" = "schedule" ]; then + DB_SERVER="${{ secrets.DB_SERVER }}" + DB_API_USER="${{ secrets.DB_API_USER }}" + DB_API_PW="${{ secrets.DB_API_PW }}" + DB_API_PORT="${{ secrets.DB_API_PORT }}" + fi + SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}" echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" - echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - - name: Create environment file (local DB) - if: github.event_name != 'schedule' - run: | - { - echo "SIMTOOLS_DB_SERVER=mongodb" - echo "SIMTOOLS_DB_API_USER=api" - echo "SIMTOOLS_DB_API_PW=password" - echo "SIMTOOLS_DB_API_PORT=27017" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env + cat > .env < Date: Thu, 18 Dec 2025 14:19:20 +0100 Subject: [PATCH 19/26] more about env --- .github/workflows/CI-integrationtests.yml | 38 +++++++++-------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index d6aaa8e1a2..c26dfc3f93 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -113,25 +113,21 @@ jobs: fi SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" - - cat > .env <> "$GITHUB_ENV" - name: Extend PATH (simtel) run: | - source .env [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" - name: Determine simulation model branch @@ -188,17 +184,11 @@ jobs: - name: Download QGSJet (prod images without tables) if: contains(inputs.container_image, 'simtools-prod') run: | - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download -O qgsdat-III.bz2 - bunzip2 qgsdat-III.bz2 - source .env - mv -v -f qgsdat-III "$SIMTOOLS_CORSIKA_PATH/" + wget -qO- "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \ + bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III" - name: Integration tests shell: bash -l {0} - env: - SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} - SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} - SIMTOOLS_DB_SERVER: ${{ github.event_name == 'schedule' && secrets.DB_SERVER || 'mongodb' }} run: | cat .env simtools-print-version From 402655cdf3d73e0ef6cb23b6181718720b056cfa Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 14:23:40 +0100 Subject: [PATCH 20/26] changelog --- docs/changes/1955.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/1955.feature.md diff --git a/docs/changes/1955.feature.md b/docs/changes/1955.feature.md new file mode 100644 index 0000000000..7ea0535a5a --- /dev/null +++ b/docs/changes/1955.feature.md @@ -0,0 +1 @@ +Add integration test run to updated production image building. Reuse existing integration tests and simplify building steps. From 9f0ace1f77f2570963a0016440340bce9a350421 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 14:28:23 +0100 Subject: [PATCH 21/26] full tests --- .github/workflows/CI-integrationtests.yml | 13 +------------ .github/workflows/build-simtools-prod.yml | 3 +-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index c26dfc3f93..3529729863 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -153,16 +153,6 @@ jobs: fi echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV" - - name: Print DB connection variables - run: | - echo "Event name: ${{ github.event_name }}" - echo "DB Server: $SIMTOOLS_DB_SERVER" - echo "DB API User: $SIMTOOLS_DB_API_USER" - echo "DB API Port: $SIMTOOLS_DB_API_PORT" - echo "DB Simulation Model: $SIMTOOLS_DB_SIMULATION_MODEL" - echo "DB Simulation Model Version: $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - echo "DB Simulation Model Branch: $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" - - name: Install Python dependencies (prod container) if: contains(inputs.container_image, 'simtools-prod') run: | @@ -193,5 +183,4 @@ jobs: cat .env simtools-print-version pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \ - 'tests/integration_tests/test_applications_from_config.py::test_applications_from_config[simtools-simulate-prod_gamma_40_deg_south_check_output]' - # -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ + -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index b43f863c9e..5f63b57b7f 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -84,8 +84,7 @@ jobs: with: context: . # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific - # TODO platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} - platforms: 'linux/amd64' + platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} From 529761d08df51818ab3351743f64e39bf538b3b3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 14:31:06 +0100 Subject: [PATCH 22/26] full tests --- .github/workflows/build-simtools-prod.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index 5f63b57b7f..f25dad1c02 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -83,8 +83,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific - platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + platforms: 'linux/amd64' build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} From 57578ad91b484608685da56650df661d96d598ff Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 18 Dec 2025 16:08:26 +0100 Subject: [PATCH 23/26] no load --- .github/workflows/build-simtools-prod.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index f25dad1c02..f10f1593e2 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -83,14 +83,13 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: 'linux/amd64' + platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} AVX_FLAG=${{ matrix.version.avx_flag }} SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }} - load: true file: ./docker/Dockerfile-simtools-prod tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 0ea16867b1ba9a3ed28c72fd4ff6613aaf831f4a Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 19 Dec 2025 08:26:40 +0100 Subject: [PATCH 24/26] minor change in wget --- .github/workflows/CI-integrationtests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 3529729863..4800c86b86 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -174,7 +174,7 @@ jobs: - name: Download QGSJet (prod images without tables) if: contains(inputs.container_image, 'simtools-prod') run: | - wget -qO- "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \ + wget -nv -O - "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \ bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III" - name: Integration tests From ddb723842e0510aa9ae3610c50aba21e9124ad0c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 19 Dec 2025 10:59:40 +0100 Subject: [PATCH 25/26] remove last todo --- .github/workflows/build-simtools-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index f10f1593e2..d0471a29b9 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -5,8 +5,8 @@ name: build-simtools-prod on: workflow_dispatch: pull_request: - # TODO paths: - # - docker/Dockerfile-simtools-prod + paths: + - docker/Dockerfile-simtools-prod release: types: [published] schedule: From a076d941c437b6cc68c1206329b8bf084700d09f Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 5 Jan 2026 11:47:31 +0100 Subject: [PATCH 26/26] add job permissions --- .github/workflows/build-simtools-prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index d0471a29b9..870aba36a5 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -95,6 +95,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} test-simtools-prod: + permissions: + contents: read needs: build-simtools-prod uses: ./.github/workflows/CI-integrationtests.yml secrets: inherit