diff --git a/.github/workflows/build-corsika-simtel.yml b/.github/workflows/build-corsika-simtel-legacy.yml similarity index 87% rename from .github/workflows/build-corsika-simtel.yml rename to .github/workflows/build-corsika-simtel-legacy.yml index 622fd6d3fd..2ec8f9a247 100644 --- a/.github/workflows/build-corsika-simtel.yml +++ b/.github/workflows/build-corsika-simtel-legacy.yml @@ -1,5 +1,6 @@ --- -name: build-corsika-simtel +name: build-corsika-simtel-legacy +# **legacy build** - will be deprecated in near future # Build and push CORSIKA / sim_telarray base images for different CPU architectures. # The no_opt (non-optimized) images are built as multi-platform images (x86_64 and arm64). # @@ -73,22 +74,22 @@ jobs: matrix: version: # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'avx2'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'avx512f'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'sse4'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'no_opt'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'avx512f'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'sse4'} + - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx512f'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'sse4'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'no_opt'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx512f'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'sse4'} production: # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used - {build_opt: 'prod6-baseline', extra_def: ''} env: - BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_instruction }}" + BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" steps: - name: Checkout repository @@ -144,12 +145,12 @@ jobs: with: context: . # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific - platforms: ${{ matrix.version.avx_instruction == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} build-args: | BUILD_OPT=${{ matrix.production.build_opt }} EXTRA_DEF=${{ matrix.production.extra_def }} HADRONIC_MODEL=${{ matrix.version.hadronic_model }} - AVX_FLAG=${{ matrix.version.avx_instruction }} + AVX_FLAG=${{ matrix.version.avx_flag }} CORSIKA_VERSION=${{ matrix.version.corsika_version }} BERNLOHR_VERSION=${{ matrix.version.bernlohr_version }} secrets: | diff --git a/.github/workflows/build-corsika7.yml b/.github/workflows/build-corsika7.yml index d2de0ec299..58a8e07553 100644 --- a/.github/workflows/build-corsika7.yml +++ b/.github/workflows/build-corsika7.yml @@ -4,6 +4,7 @@ name: build-corsika7 # The no_opt (non-optimized) images are built as multi-platform images (x86_64 and arm64). # # The images are built and pushed when manually triggered (workflow_dispatch). +# on: workflow_dispatch: @@ -19,7 +20,6 @@ env: CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" jobs: - download-auxiliary-files: runs-on: ubuntu-latest @@ -44,13 +44,13 @@ jobs: strategy: matrix: version: - - {corsika_version: '78010', avx_instruction: 'no_opt'} - - {corsika_version: '78010', avx_instruction: 'avx2'} - - {corsika_version: '78010', avx_instruction: 'avx512f'} - - {corsika_version: '78010', avx_instruction: 'sse4'} - + - {corsika: '78010', avx_flag: 'no_opt', corsika_config: 'v0.1.0'} + - {corsika: '78010', avx_flag: 'avx2', corsika_config: 'v0.1.0'} + - {corsika: '78010', avx_flag: 'avx512f', corsika_config: 'v0.1.0'} + - {corsika: '78010', avx_flag: 'sse4', corsika_config: 'v0.1.0'} env: - BASE_LABELS: "corsika7-${{ matrix.version.corsika_version }}-${{ matrix.version.avx_instruction }}" + BASE_LABEL: "corsika7" + BASE_TAG: "${{ matrix.version.corsika }}-${{ matrix.version.avx_flag }}" steps: - name: Checkout repository @@ -73,34 +73,30 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata - id: meta-base + id: meta uses: docker/metadata-action@v5 with: tags: | - type=ref,event=pr - type=semver,pattern={{major}}.{{minor}}.{{patch}} - type=schedule,pattern={{date 'YYYYMMDD'}} - type=raw,value={{date 'YYYYMMDD-HHmmss'}} - images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }} - flavor: latest=true + type=ref,event=pr,suffix=-${{ env.BASE_TAG }} + type=semver,pattern={{major}}.{{minor}}.{{patch}}-${{ env.BASE_TAG }} + type=schedule,pattern={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + type=raw,value={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} + flavor: latest=false - name: Build and push uses: docker/build-push-action@v6 with: context: . # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific - platforms: ${{ matrix.version.avx_instruction == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} build-args: | - AVX_FLAG=${{ matrix.version.avx_instruction }} - CORSIKA_VERSION=${{ matrix.version.corsika_version }} + AVX_FLAG=${{ matrix.version.avx_flag }} + CORSIKA_VERSION=${{ matrix.version.corsika }} + CORSIKA_CONFIG_VERSION=${{ matrix.version.corsika_config }} secrets: | corsika7_token=${{ secrets.CORSIKA7_KIT_GITLAB }} push: ${{ github.event_name == 'workflow_dispatch' }} file: ./docker/Dockerfile-corsika7 - tags: ${{ steps.meta-base.outputs.tags }} - labels: ${{ steps.meta-base.outputs.labels }} - - - name: Output image information - run: | - echo "Built base image: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }}:latest" - echo "Image tags: ${{ steps.meta-base.outputs.tags }}" + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-sim_telarray.yml b/.github/workflows/build-sim_telarray.yml index bdd7596bdb..e4121b8e84 100644 --- a/.github/workflows/build-sim_telarray.yml +++ b/.github/workflows/build-sim_telarray.yml @@ -19,7 +19,7 @@ env: CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" jobs: - build-base-images: + build-sim_telarray: runs-on: ubuntu-latest permissions: contents: read @@ -28,11 +28,9 @@ jobs: matrix: version: - {simtel_version: 'master', hessio_version: 'master', stdtools_version: 'master'} - production: - - {build_opt: 'prod6-baseline', extra_def: '-DMAXIMUM_TELESCOPES=99'} - env: - BASE_LABELS: "simtel-${{ matrix.version.simtel_version }}-${{ matrix.production.build_opt }}" + BASE_LABEL: "sim_telarray" + BASE_TAG: "${{ matrix.version.simtel_version }}" steps: - name: Checkout repository @@ -48,19 +46,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata for sim_telarray image (tags, labels) - id: meta-base + - name: Extract metadata + id: meta uses: docker/metadata-action@v5 with: tags: | - type=ref,event=pr - type=semver,pattern={{major}}.{{minor}}.{{patch}} - type=schedule,pattern={{date 'YYYYMMDD'}} - type=raw,value={{date 'YYYYMMDD-HHmmss'}} - images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }} + type=ref,event=pr,suffix=-${{ env.BASE_TAG }} + type=semver,pattern={{major}}.{{minor}}.{{patch}}-${{ env.BASE_TAG }} + type=schedule,pattern={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + type=raw,value={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} flavor: latest=true - - name: Build /sim_telarray image + - name: Build and push uses: docker/build-push-action@v6 with: context: . @@ -69,17 +67,11 @@ jobs: SIMTEL_VERSION=${{ matrix.version.simtel_version }} HESSIO_VERSION=${{ matrix.version.hessio_version }} STDTOOLS_VERSION=${{ matrix.version.stdtools_version }} - EXTRA_DEF=${{ matrix.production.extra_def }} secrets: | - sim_telarray_token=${{ secrets.SIMTELARRAY_CTAO_GITLAB}} - hessio_token=${{ secrets.HESSIO_CTAO_GITLAB}} - stdtools_token=${{ secrets.STDTOOLS_CTAO_GITLAB}} + sim_telarray_token=${{ secrets.SIMTELARRAY_CTAO_GITLAB }} + hessio_token=${{ secrets.HESSIO_CTAO_GITLAB }} + stdtools_token=${{ secrets.STDTOOLS_CTAO_GITLAB }} push: ${{ github.event_name == 'workflow_dispatch' }} file: ./docker/Dockerfile-simtel_array - tags: ${{ steps.meta-base.outputs.tags }} - labels: ${{ steps.meta-base.outputs.labels }} - - - name: Output image information - run: | - echo "Built base image: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }}:latest" - echo "Image tags: ${{ steps.meta-base.outputs.tags }}" + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-simtools-dev.yml b/.github/workflows/build-simtools-dev.yml index 47f33d7409..b21beb3e21 100644 --- a/.github/workflows/build-simtools-dev.yml +++ b/.github/workflows/build-simtools-dev.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: pull_request: paths: - - './docker/Dockerfile-simtools-dev' + - docker/Dockerfile-simtools-dev release: types: [published] schedule: @@ -15,7 +15,6 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - TEST_TAG: user/app:test CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" jobs: @@ -34,15 +33,14 @@ jobs: qgsdat-III.bz2 retention-days: 1 - build-simtools-container: + build-simtools-dev: runs-on: ubuntu-latest needs: [download-qgsjet-tables] permissions: contents: read packages: write - strategy: - matrix: - type: ['dev'] + env: + BASE_LABEL: "simtools-dev" steps: - name: Checkout repository @@ -54,17 +52,14 @@ jobs: name: upload-qgsjet-tables path: . - - name: Set build arguments (correct branch) + - name: Set build branch run: | if [[ "${{ github.event_name }}" == 'pull_request' ]]; then - echo "BUILD_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" + echo "SIMTOOLS_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" else - echo "BUILD_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" + echo "SIMTOOLS_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" fi - echo "BUILD BRANCH ${{ env.BUILD_BRANCH }}" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + echo "BUILD BRANCH ${{ env.SIMTOOLS_BRANCH }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -76,7 +71,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker + - name: Extract metadata id: meta uses: docker/metadata-action@v5 with: @@ -84,26 +79,17 @@ jobs: type=ref,event=pr type=semver,pattern={{major}}.{{minor}}.{{patch}} type=schedule,pattern={{date 'YYYYMMDD'}} - type=raw,value={{date 'YYYYMMDD-HHmmss'}} - images: ${{ env.REGISTRY }}/gammasim/simtools-${{ matrix.type }} + type=raw,value={{date 'YYYYMMDD'}} + images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} flavor: latest=true - - name: Build Docker image - uses: docker/build-push-action@v6 - with: - context: . - build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }} - load: true - file: ./docker/Dockerfile-simtools-${{ matrix.type }} - tags: ${{ env.TEST_TAG }} - - - name: Build and push image + - name: Build and push uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64/v8 - build-args: BUILD_BRANCH=${{ env.BUILD_BRANCH }} + build-args: SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} - file: ./docker/Dockerfile-simtools-${{ matrix.type }} + file: ./docker/Dockerfile-simtools-dev tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }}-${{ matrix.type }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-simtools-legacy.yml b/.github/workflows/build-simtools-legacy.yml index 91f71ec900..4d3fe76665 100644 --- a/.github/workflows/build-simtools-legacy.yml +++ b/.github/workflows/build-simtools-legacy.yml @@ -1,5 +1,6 @@ --- name: build-simtools-legacy +# **legacy build** - will be deprecated in near future # Build and push simtools images using CORSIKA / sim_telarray base images. # Integration tests are running on the non-optimized ('no_opt') images. @@ -30,23 +31,23 @@ jobs: matrix: version: # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'avx2'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'avx512f'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'sse4'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'no_opt'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'avx512f'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'sse4'} + - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx512f'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'sse4'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'no_opt'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx512f'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'sse4'} production: # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used - {build_opt: 'prod6-baseline', extra_def: ''} env: - BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_instruction }}" - SIMTOOLS_LABELS: "simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_instruction }}" + BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" + SIMTOOLS_LABELS: "simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" steps: - name: Checkout repository @@ -55,9 +56,9 @@ jobs: - name: Set build branch run: | if [[ "${{ github.event_name }}" == 'pull_request' ]]; then - echo "BUILD_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" + echo "SIMTOOLS_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" else - echo "BUILD_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" + echo "SIMTOOLS_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" fi - name: Set up Docker Buildx @@ -89,7 +90,7 @@ jobs: platforms: linux/amd64 build-args: | BASE_IMAGE=${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }}:latest - BUILD_BRANCH=${{ env.BUILD_BRANCH }} + SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} push: ${{ github.event_name == 'release' || github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }} file: ./docker/Dockerfile-simtools-legacy tags: ${{ steps.meta-simtools.outputs.tags }} @@ -102,11 +103,11 @@ jobs: matrix: version: # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'no_opt'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_instruction: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'no_opt'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_instruction: 'avx2'} + - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'no_opt'} + - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'no_opt'} + - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} production: # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used @@ -114,7 +115,7 @@ jobs: model_version: - "6.0.2" container: - image: ghcr.io/gammasim/simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_instruction }} + image: ghcr.io/gammasim/simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }} options: --user 0 services: diff --git a/.github/workflows/build-simtools-prod.yml b/.github/workflows/build-simtools-prod.yml new file mode 100644 index 0000000000..c1c6857ea3 --- /dev/null +++ b/.github/workflows/build-simtools-prod.yml @@ -0,0 +1,85 @@ +--- +name: build-simtools-prod +# Build simtools production images based on CORSIKA / sim_telarray images + +on: + workflow_dispatch: + pull_request: + paths: + - docker/Dockerfile-simtools-prod + release: + types: [published] + schedule: + - cron: '0 0 * * 0' # Every Sunday at 00:00 UTC (build only) + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-simtools-prod: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + version: + - {corsika: '20251217-78010', avx_flag: 'no_opt', simtel_image: 'latest'} + - {corsika: '20251217-78010', avx_flag: 'avx2', simtel_image: 'latest'} + - {corsika: '20251217-78010', avx_flag: 'avx512f', simtel_image: 'latest'} + - {corsika: '20251217-78010', avx_flag: 'sse4', simtel_image: 'latest'} + env: + BASE_LABEL: "simtools-prod" + BASE_TAG: "corsika-${{ matrix.version.corsika }}-${{ matrix.version.avx_flag }}-simtel-${{ matrix.version.simtel_image }}" + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set build branch + run: | + if [[ "${{ github.event_name }}" == 'pull_request' ]]; then + echo "SIMTOOLS_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" + else + echo "SIMTOOLS_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" + fi + echo "BUILD BRANCH ${{ env.SIMTOOLS_BRANCH }}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=ref,event=pr,suffix=-${{ env.BASE_TAG }} + type=semver,pattern={{major}}.{{minor}}.{{patch}}-${{ env.BASE_TAG }} + type=schedule,pattern={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + type=raw,value={{date 'YYYYMMDD'}}-${{ env.BASE_TAG }} + images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }} + flavor: latest=false + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific + platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }} + build-args: | + CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }} + SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }} + AVX_FLAG=${{ matrix.version.avx_flag }} + SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} + push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + file: ./docker/Dockerfile-simtools-prod + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/docker/Dockerfile-corsika7 b/docker/Dockerfile-corsika7 index 701d71c35c..733374339d 100644 --- a/docker/Dockerfile-corsika7 +++ b/docker/Dockerfile-corsika7 @@ -12,7 +12,7 @@ ARG CORSIKA_VERSION="78010" ARG CORSIKA_OPT_PATCH_VERSION="v1.1.0" -ARG CORSIKA_CONFIG_VERSION="78010" +ARG CORSIKA_CONFIG_VERSION="v0.1.0" ARG AVX_FLAG="avx2" # hadolint global ignore=DL3013,DL3033,DL3041,DL4006,SC1091,SC2086 @@ -39,7 +39,7 @@ WORKDIR /src # Clone CORSIKA configuration files from CTAO gitlab RUN git config --global advice.detachedHead false && \ - git clone --depth 1 --branch "${CORSIKA_VERSION}" "http://${CORSIKA_CONFIG_REPO}" && \ + git clone --depth 1 --branch "${CORSIKA_CONFIG_VERSION}" "https://${CORSIKA_CONFIG_REPO}" && \ cp -v corsika7-config/config/${CORSIKA_VERSION}/*.h . && \ rm -rf corsika7-config @@ -47,7 +47,7 @@ RUN git config --global advice.detachedHead false && \ RUN --mount=type=secret,id=corsika7_token \ CORSIKA7_TAG="v$(echo "${CORSIKA_VERSION}" | cut -c1).$(echo "${CORSIKA_VERSION}" | cut -c2-)" && \ git config --global advice.detachedHead false && \ - git clone --depth 1 --branch "${CORSIKA7_TAG}" "https://oauth2:$(cat /run/secrets/corsika7_token)@${CORSIKA_REPO}" + GIT_SSL_NO_VERIFY=1 git clone --depth 1 --branch "${CORSIKA7_TAG}" "https://oauth2:$(cat /run/secrets/corsika7_token)@${CORSIKA_REPO}" # Clone CORSIKA optimization patches from CTAO gitlab RUN git config --global advice.detachedHead false && \ diff --git a/docker/Dockerfile-simtel_array b/docker/Dockerfile-simtel_array index d91131f48b..532732ad9f 100644 --- a/docker/Dockerfile-simtel_array +++ b/docker/Dockerfile-simtel_array @@ -4,7 +4,7 @@ # Image with sim_telarray and tools. # -ARG EXTRA_DEFINES="-DMAXIMUM_TELESCOPES=99" +ARG EXTRA_DEFINES="-DCTA_PROD6 -DMAXIMUM_TELESCOPES=99" ARG SIMTEL_VERSION="master" ARG HESSIO_VERSION="master" ARG STDTOOLS_VERSION="master" diff --git a/docker/Dockerfile-simtools-dev b/docker/Dockerfile-simtools-dev index 2a57b14109..9842a92373 100644 --- a/docker/Dockerfile-simtools-dev +++ b/docker/Dockerfile-simtools-dev @@ -1,6 +1,6 @@ # Dockerfile for simtools development. # -# Uses CORSIKA (no vector optimization) and sim_telarray images. +# Uses CORSIKA (no vector optimization) and sim_telarray pre-built images. # # Minimal CORSIKA version is 7.8x (otherwise QGSJet-II tables need to be added) # @@ -11,26 +11,19 @@ # hadolint global ignore=DL3013,DL3041 # - DL3013, DL3041: ignore warnings about using latest -ARG CORSIKA_VERSION="78010" -ARG CORSIKA_IMAGE_VERSION="latest" -ARG SIMTEL_VERSION="251124" -ARG SIMTEL_BUILD_OPT="prod6-baseline" +ARG CORSIKA_IMAGE_VERSION="20251217-78010" ARG SIMTEL_IMAGE_VERSION="latest" -ARG AVX_INSTRUCTION="no_opt" +ARG AVX_FLAG="no_opt" ARG PYTHON_VERSION="3.12" -ARG SIMTOOLS_REQUIREMENT="https://github.com/gammasim/simtools.git" -FROM ghcr.io/gammasim/corsika7-${CORSIKA_VERSION}-${AVX_INSTRUCTION}:${CORSIKA_IMAGE_VERSION} AS corsika_img -FROM ghcr.io/gammasim/simtel-${SIMTEL_VERSION}-${SIMTEL_BUILD_OPT}:${SIMTEL_IMAGE_VERSION} AS simtel_img +FROM ghcr.io/gammasim/corsika7:${CORSIKA_IMAGE_VERSION}-${AVX_FLAG} AS corsika_img +FROM ghcr.io/gammasim/sim_telarray:${SIMTEL_IMAGE_VERSION} AS simtel_img FROM almalinux:9.5-minimal ARG PYTHON_VERSION -ARG SIMTOOLS_REQUIREMENT +ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" -# Copy CORSIKA from corsika image COPY --from=corsika_img /workdir/simulation_software/corsika7 /workdir/simulation_software/corsika7 - -# Copy sim_telarray components from simtel image COPY --from=simtel_img /workdir/simulation_software/sim_telarray /workdir/simulation_software/sim_telarray COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulation_software/hessioxxx COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools diff --git a/docker/Dockerfile-simtools-legacy b/docker/Dockerfile-simtools-legacy index 24ab0cb4fc..bbf6187a72 100644 --- a/docker/Dockerfile-simtools-legacy +++ b/docker/Dockerfile-simtools-legacy @@ -10,13 +10,13 @@ # - SC1091: ignore warning about not being able to source the bashrc ARG BASE_IMAGE -ARG BUILD_BRANCH=main -ARG SIMTOOLS_REQUIREMENT="https://github.com/gammasim/simtools.git" +ARG SIMTOOLS_BRANCH=main +ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" ARG PYTHON_VERSION=3.12 FROM ${BASE_IMAGE} -ARG BUILD_BRANCH -ARG SIMTOOLS_REQUIREMENT +ARG SIMTOOLS_BRANCH +ARG SIMTOOLS_REPO ARG PYTHON_VERSION SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -28,12 +28,12 @@ RUN microdnf update -y && microdnf install -y \ rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python -RUN BUILD_BRANCH=${BUILD_BRANCH//refs\/tags\//} \ - && git clone -b "$BUILD_BRANCH" "${SIMTOOLS_REQUIREMENT}" --depth 1 \ +RUN SIMTOOLS_BRANCH=${SIMTOOLS_BRANCH//refs\/tags\//} \ + && git clone -b "$SIMTOOLS_BRANCH" "${SIMTOOLS_REPO}" --depth 1 \ && python${PYTHON_VERSION} -m venv env \ && . env/bin/activate \ && pip install --no-cache-dir -U pip \ - && pip install --no-cache-dir "git+${SIMTOOLS_REQUIREMENT}@${BUILD_BRANCH}" \ + && pip install --no-cache-dir "git+${SIMTOOLS_REPO}@${SIMTOOLS_BRANCH}" \ && pip cache purge && rm -rf /root/.cache \ && echo ". /workdir/env/bin/activate" >> ~/.bashrc diff --git a/docker/Dockerfile-simtools-prod b/docker/Dockerfile-simtools-prod index 435ff830f8..7498bed833 100644 --- a/docker/Dockerfile-simtools-prod +++ b/docker/Dockerfile-simtools-prod @@ -1,35 +1,25 @@ -# Dockerfile for simtools development. +# Dockerfile for simtools production. # -# Uses CORSIKA (no vector optimization) and sim_telarray images. -# -# Minimal CORSIKA version is 7.8x (otherwise QGSJet-II tables need to be added) -# -# All dependencies for simtools are installed, but not simtools itself. -# It is expected that simtools is installed in an external directory -# mounted in the container. +# Uses CORSIKA and sim_telarray pre-built images; CORSIKA AVX optimization +# (no_opt, avx2, avx512f, sse4) is selected via the AVX_FLAG build argument. +# No QGSJet tables are included. # # hadolint global ignore=DL3013,DL3041 # - DL3013, DL3041: ignore warnings about using latest -ARG CORSIKA_VERSION="78010" -ARG CORSIKA_IMAGE_VERSION="latest" -ARG SIMTEL_VERSION="251124" -ARG SIMTEL_BUILD_OPT="prod6-baseline" +ARG CORSIKA_IMAGE_VERSION="20251217-78010" ARG SIMTEL_IMAGE_VERSION="latest" -ARG AVX_INSTRUCTION="no_opt" +ARG AVX_FLAG="no_opt" ARG PYTHON_VERSION="3.12" -ARG BUILD_BRANCH=main -ARG SIMTOOLS_REQUIREMENT="https://github.com/gammasim/simtools.git" +ARG SIMTOOLS_BRANCH=main -ARG CORSIKA_VERSION ARG CORSIKA_IMAGE_VERSION ARG AVX_INSTRUCTION -FROM ghcr.io/gammasim/corsika7-${CORSIKA_VERSION}-${AVX_INSTRUCTION}:${CORSIKA_IMAGE_VERSION} AS corsika_img -ARG SIMTEL_VERSION ARG SIMTEL_IMAGE_VERSION ARG SIMTEL_BUILD_OPT -FROM ghcr.io/gammasim/simtel-${SIMTEL_VERSION}-${SIMTEL_BUILD_OPT}:${SIMTEL_IMAGE_VERSION} AS simtel_img +FROM ghcr.io/gammasim/corsika7:${CORSIKA_IMAGE_VERSION}-${AVX_FLAG} AS corsika_img +FROM ghcr.io/gammasim/sim_telarray:${SIMTEL_IMAGE_VERSION} AS simtel_img FROM almalinux:9.5-minimal -ARG BUILD_BRANCH -ARG SIMTOOLS_REQUIREMENT ARG PYTHON_VERSION +ARG SIMTOOLS_BRANCH +ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" COPY --from=corsika_img /workdir/simulation_software/corsika7 /workdir/simulation_software/corsika7 COPY --from=simtel_img /workdir/simulation_software/sim_telarray /workdir/simulation_software/sim_telarray @@ -37,7 +27,7 @@ COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulatio COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools RUN microdnf update -y && microdnf install -y \ - bc findutils git gsl-devel libgfortran procps zstd \ + bc findutils gcc-c++ git gsl libgfortran procps wget zstd \ python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ microdnf clean all && \ rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ @@ -46,12 +36,12 @@ RUN microdnf update -y && microdnf install -y \ WORKDIR /workdir # hadolint ignore=DL4006 -RUN BUILD_BRANCH="$(printf '%s' "$BUILD_BRANCH" | sed 's#refs/tags/##')" \ - && git clone -b "$BUILD_BRANCH" "${SIMTOOLS_REQUIREMENT}" --depth 1 \ +RUN SIMTOOLS_BRANCH="$(printf '%s' "$SIMTOOLS_BRANCH" | sed 's#refs/tags/##')" \ + && git clone -b "$SIMTOOLS_BRANCH" "${SIMTOOLS_REPO}" --depth 1 \ && python${PYTHON_VERSION} -m venv env \ && . env/bin/activate \ && pip install --no-cache-dir -U pip \ - && pip install --no-cache-dir "git+${SIMTOOLS_REQUIREMENT}@${BUILD_BRANCH}" \ + && pip install --no-cache-dir "git+${SIMTOOLS_REPO}@${SIMTOOLS_BRANCH}" \ && pip cache purge && rm -rf /root/.cache \ && echo ". /workdir/env/bin/activate" >> ~/.bashrc diff --git a/docs/changes/1954.api.md b/docs/changes/1954.api.md new file mode 100644 index 0000000000..193cb86151 --- /dev/null +++ b/docs/changes/1954.api.md @@ -0,0 +1,6 @@ +Change in naming of simtools docker images: + +- [CORSIKA image](https://github.com/gammasim/simtools/pkgs/container/corsika7) is called e.g. `ghcr.io/gammasim/corsika7:0.26.0-78010-avx512f` +- [sim_telarray image](https://github.com/gammasim/simtools/pkgs/container/sim_telarray) is called e.g. `ghcr.io/gammasim/sim_telarray:0.26.0-` +- [simtools-dev image](https://github.com/gammasim/simtools/pkgs/container/simtools-dev) is called e.g. `ghcr.io/gammasim/simtools-dev:latest` (safe to use the latest tag here) +- [simtools-prod image](https://github.com/gammasim/simtools/pkgs/container/simtools-prod) is called e.g. `ghcr.io/gammasim/simtools-prod:0.26.0-corsika--simtel` diff --git a/docs/changes/1954.feature.md b/docs/changes/1954.feature.md new file mode 100644 index 0000000000..c4efde1f0c --- /dev/null +++ b/docs/changes/1954.feature.md @@ -0,0 +1,3 @@ +Add building workflow for simtools-prod image. +Update docker files and building workflows for CORSIKA, sim_telarray, and simtools. +Simplify package naming.