diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 95a97f53e6..4800c86b86 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -1,6 +1,5 @@ --- name: CI-integrationtests -# Integration tests for applications on: workflow_dispatch: @@ -10,6 +9,29 @@ on: required: false type: string default: 'main' + workflow_call: + inputs: + container_image: + description: 'Container image to use for tests' + required: false + type: string + default: 'ghcr.io/gammasim/simtools-dev:latest' + model_versions: + description: 'Model versions to test' + required: false + type: string + default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' + secrets: + CLOUD_QGSJET3: + required: false + DB_SERVER: + required: false + DB_API_USER: + required: false + DB_API_PW: + required: false + DB_API_PORT: + required: false pull_request: types: [opened, synchronize] paths-ignore: @@ -20,11 +42,13 @@ on: release: types: [published] +env: + CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" + jobs: test_building: - # Build the package and check if it is installable - # (tests among others that all components are there) + if: ${{ github.event_name != 'workflow_call' }} runs-on: ubuntu-latest defaults: @@ -32,20 +56,17 @@ jobs: shell: bash -leo pipefail {0} steps: + - uses: actions/checkout@v6 - - name: checkout - uses: actions/checkout@v6 - - - name: Build the package + - name: Build package run: | - python -m pip install --upgrade pip - python -m pip install --upgrade build + python -m pip install --upgrade pip build python -m build integration_tests: runs-on: ubuntu-latest container: - image: ghcr.io/gammasim/simtools-dev:latest + image: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }} options: --user 0 services: @@ -63,60 +84,51 @@ jobs: strategy: fail-fast: false matrix: - model_version: ['7.0.0', '6.0.2', '5.0.0', '6.0.2,6.1.1'] + model_version: ${{ fromJSON(inputs.model_versions || '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]') }} defaults: run: shell: bash -leo pipefail {0} steps: - - name: checkout - uses: actions/checkout@v6 + - uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Set sim_telarray path + - name: Create runtime environment (.env) run: | - echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV" + set -a + source .env_template + set +a + + DB_SERVER="mongodb" + DB_API_USER="api" + DB_API_PW="password" + DB_API_PORT="27017" + if [ "${{ github.event_name }}" = "schedule" ]; then + DB_SERVER="${{ secrets.DB_SERVER }}" + DB_API_USER="${{ secrets.DB_API_USER }}" + DB_API_PW="${{ secrets.DB_API_PW }}" + DB_API_PORT="${{ secrets.DB_API_PORT }}" + fi - - name: Create environment file (remote DB; schedule only) - if: ${{ github.event_name == 'schedule' }} - run: | - { - echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}" - echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}" - echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}" - echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env - - - name: Extract SIMTOOLS_DB_SIMULATION_MODEL - run: | - SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"') - SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"') SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" - echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - - - name: Create environment file (local DB) - if: github.event_name != 'schedule' - run: | { - echo "SIMTOOLS_DB_SERVER=mongodb" - echo "SIMTOOLS_DB_API_USER=api" - echo "SIMTOOLS_DB_API_PW=password" - echo "SIMTOOLS_DB_API_PORT=27017" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}" - echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}" - echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')" - } > .env + echo "SIMTOOLS_DB_SERVER=$DB_SERVER" + echo "SIMTOOLS_DB_API_USER=$DB_API_USER" + echo "SIMTOOLS_DB_API_PW=$DB_API_PW" + echo "SIMTOOLS_DB_API_PORT=$DB_API_PORT" + echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" + echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" + echo "SIMTOOLS_SIMTEL_PATH=$SIMTOOLS_SIMTEL_PATH" + echo "SIMTOOLS_CORSIKA_PATH=$SIMTOOLS_CORSIKA_PATH" + echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$SIMTOOLS_CORSIKA_HE_INTERACTION" + echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$SIMTOOLS_CORSIKA_LE_INTERACTION" + } | tee .env >> "$GITHUB_ENV" + + - name: Extend PATH (simtel) + run: | + [ -n "$SIMTOOLS_SIMTEL_PATH" ] && echo "$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_PATH" - name: Determine simulation model branch env: @@ -125,15 +137,13 @@ jobs: EVENT_NAME: ${{ github.event_name }} INPUT_BRANCH: ${{ github.event.inputs.simulation_model_branch }} run: | + BRANCH="main" if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_BRANCH" ]; then BRANCH="$INPUT_BRANCH" - else - BRANCH="main" fi + CURRENT_REF="$REF_NAME" if [ "$EVENT_NAME" = "pull_request" ]; then CURRENT_REF="$HEAD_REF" - else - CURRENT_REF="$REF_NAME" fi # release candidates (e.g., v1.2.3-rc or v1.2.3-rc3) should be tested against released versions of the # simulation model database @@ -143,41 +153,34 @@ jobs: fi echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV" - - name: Print DB connection variables + - name: Install Python dependencies (prod container) + if: contains(inputs.container_image, 'simtools-prod') + run: | + pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist pytest-retry + + - name: Install Python dependencies (dev container) + if: ${{ !contains(inputs.container_image, 'simtools-prod') }} run: | - echo "Event name: ${{ github.event_name }}" - echo "DB Server: $SIMTOOLS_DB_SERVER" - echo "DB API User: $SIMTOOLS_DB_API_USER" - echo "DB API Port: $SIMTOOLS_DB_API_PORT" - echo "DB Simulation Model: $SIMTOOLS_DB_SIMULATION_MODEL" - echo "DB Simulation Model Version: $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - echo "DB Simulation Model Branch: $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" + pip install --no-cache-dir -e '.[tests,dev,doc]' - name: Upload data to MongoDB if: github.event_name != 'schedule' run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e '.[tests,dev,doc]' simtools-db-upload-model-repository \ --db_simulation_model ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} \ --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \ --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" - - name: Print versions + - name: Download QGSJet (prod images without tables) + if: contains(inputs.container_image, 'simtools-prod') run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e . - cat .env - simtools-print-version + wget -nv -O - "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \ + bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III" - name: Integration tests shell: bash -l {0} - env: - SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} - SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} - SIMTOOLS_DB_SERVER: ${{ github.event_name == 'schedule' && secrets.DB_SERVER || 'mongodb' }} run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e '.[tests,dev,doc]' + cat .env + simtools-print-version pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \ - -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ + -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml index c1c6857ea3..870aba36a5 100644 --- a/.github/workflows/build-simtools-prod.yml +++ b/.github/workflows/build-simtools-prod.yml @@ -22,6 +22,8 @@ jobs: permissions: contents: read packages: write + outputs: + no_opt_tag: ${{ steps.export-tags.outputs.no_opt_tag }} strategy: matrix: version: @@ -68,18 +70,36 @@ jobs: images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} flavor: latest=false + - name: Export no_opt tag for testing + id: export-tags + if: matrix.version.avx_flag == 'no_opt' + run: | + TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) + TAG=${TAG##*:} + echo "no_opt_tag=$TAG" >> "$GITHUB_OUTPUT" + echo "Exported no_opt tag: $TAG" + - name: Build and push uses: docker/build-push-action@v6 with: context: . - # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} build-args: | CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} AVX_FLAG=${{ matrix.version.avx_flag }} SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} - push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }} file: ./docker/Dockerfile-simtools-prod tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + test-simtools-prod: + permissions: + contents: read + needs: build-simtools-prod + uses: ./.github/workflows/CI-integrationtests.yml + secrets: inherit + with: + container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }} + model_versions: '["7.0.0","6.0.2"]' diff --git a/docker/Dockerfile-simtools-prod b/docker/Dockerfile-simtools-prod index 7498bed833..ee9a95b0ec 100644 --- a/docker/Dockerfile-simtools-prod +++ b/docker/Dockerfile-simtools-prod @@ -27,7 +27,7 @@ COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulatio COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools RUN microdnf update -y && microdnf install -y \ - bc findutils gcc-c++ git gsl libgfortran procps wget zstd \ + bc bzip2 findutils gcc-c++ git gsl libgfortran procps wget zstd \ python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ microdnf clean all && \ rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ diff --git a/docs/changes/1955.feature.md b/docs/changes/1955.feature.md new file mode 100644 index 0000000000..7ea0535a5a --- /dev/null +++ b/docs/changes/1955.feature.md @@ -0,0 +1 @@ +Add integration test run to updated production image building. Reuse existing integration tests and simplify building steps.