diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..ad28562 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,213 @@ +name: Build / Test / Push + +on: + push: + branches: + - '**' + workflow_call: + workflow_dispatch: + +env: + BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }} + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + build: + runs-on: ${{ matrix.runner }} + outputs: + build-image-arm: ${{ steps.gen-output.outputs.image-arm64 }} + build-image-x64: ${{ steps.gen-output.outputs.image-x64 }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: build-meta + name: Produce the build image tag + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha,suffix=${{ env.BUILD_SUFFIX }} + + # Build cache is shared among all builds of the same architecture + - id: cache-meta + name: Fetch build cache metadata + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=raw,value=buildcache-${{ runner.arch }} + + - id: get-registry + name: Get the sanitized registry name + run: | + echo "registry=$(echo '${{ steps.build-meta.outputs.tags }}' | cut -f1 -d:)" | tee -a "$GITHUB_OUTPUT" + + - id: set_build_url + name: Set BUILD_URL + run: | + echo "build_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | tee -a "$GITHUB_OUTPUT" + + - id: build + name: Build/push the arch-specific image + uses: docker/build-push-action@v6 + with: + platforms: ${{ matrix.platform }} + build-args: | + BUILD_TIMESTAMP=${{ github.event.repository.updated_at }} + BUILD_URL=${{ steps.set_build_url.outputs.build_url }} + GIT_REF_NAME=${{ github.ref_name }} + GIT_SHA=${{ github.sha }} + GIT_REPOSITORY_URL=${{ github.repositoryUrl }} + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + labels: ${{ steps.build-meta.outputs.labels }} + provenance: mode=max + sbom: true + tags: ${{ steps.get-registry.outputs.registry }} + outputs: type=image,push-by-digest=true,push=true + + - id: gen-output + name: Write arch-specific image digest to outputs + run: | + echo "image-${RUNNER_ARCH,,}=${{ steps.get-registry.outputs.registry }}@${{ steps.build.outputs.digest }}" | tee -a "$GITHUB_OUTPUT" + + merge: + runs-on: ubuntu-latest + needs: + - build + env: + DOCKER_APP_IMAGE_ARM64: ${{ needs.build.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.build.outputs.build-image-x64 }} + outputs: + build-image: ${{ steps.meta.outputs.tags }} + build-image-arm: ${{ needs.build.outputs.build-image-arm }} + build-image-x64: ${{ needs.build.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }} + + - name: Push the multi-platform image + run: | + docker buildx imagetools create \ + --tag "$DOCKER_METADATA_OUTPUT_TAGS" \ + "$DOCKER_APP_IMAGE_ARM64" "$DOCKER_APP_IMAGE_X64" + + test: + runs-on: ubuntu-24.04 + needs: merge + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup the stack + run: | + docker compose build --quiet + docker compose pull --quiet + docker compose up --wait + docker compose exec -u root app chown -R bfs:bfs artifacts + + - name: Run RSpec + if: ${{ always() }} + run: | + docker compose exec app rspec --format progress --format html --out artifacts/rspec.html + + - name: Copy out artifacts + if: ${{ always() }} + run: | + docker compose cp app:/opt/app/artifacts ./ + docker compose logs > artifacts/docker-compose-services.log + docker compose config > artifacts/docker-compose.merged.yml + + - name: Upload the test report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: BFS Build Report (${{ github.run_id }}_${{ github.run_attempt }}) + path: artifacts/* + if-no-files-found: error + + push: + runs-on: ubuntu-24.04 + needs: + - merge + - test + env: + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + DOCKER_APP_IMAGE_ARM64: ${{ needs.merge.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.merge.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Produce permanent image tags + id: branch-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Retag and push the image + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $DOCKER_APP_IMAGE_ARM64 $DOCKER_APP_IMAGE_X64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b9a9c62 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,61 @@ +name: Push Release Tags + +on: + push: + tags: + - '**' + workflow_call: + workflow_dispatch: + +env: + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + retag: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine the sha-based image tag to retag + id: get-base-image + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha + + - name: Verify that the image was previously built + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker manifest inspect "$BASE_IMAGE" + + - name: Produce release tags + id: tag-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=false + tags: | + type=ref,event=tag + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + + - name: Retag the pulled image + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + "$(echo "$BASE_IMAGE" | cut -f1 -d:)" diff --git a/.gitignore b/.gitignore index b8a2827..300400f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ data/invoicing/pay/processed/* ssh_tests +# Build/test artifacts +artifacts/* + vendor .bundle diff --git a/.ruby-version b/.ruby-version index ef538c2..23887f6 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.1.2 +3.1.7 diff --git a/Dockerfile b/Dockerfile index 9f69c98..98ca861 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM registry.access.redhat.com/ubi8/ruby-31 - USER root # Configure users and groups @@ -7,12 +6,17 @@ RUN groupadd -g 40054 alma && \ useradd -r -s /sbin/nologin -M -u 40054 -g alma alma && \ groupadd -g 40061 bfs && \ usermod -u 40061 -g bfs -G alma -l bfs default && \ - find / -user 1001 -exec chown -h bfs {} \; || true + find / -user 1001 -exec chown -h bfs {} \; || true && \ + mkdir -p /opt/app && \ + chown -R bfs:bfs /opt/app +WORKDIR /opt/app COPY --chown=bfs Gemfile* .ruby-version ./ -RUN bundle install --system +RUN bundle config set force_ruby_platform true +RUN bundle config set system 'true' +RUN bundle install COPY --chown=bfs . . USER bfs -ENTRYPOINT ["/opt/app-root/src/bin/bfs"] +ENTRYPOINT ["/opt/app/bin/bfs"] CMD ["help"] diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 2f65418..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1 +0,0 @@ -dockerComposePipeline() diff --git a/README.md b/README.md index 4f73756..fbe1c6b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A command-line tool for processing BFS .xml files. Input files can be mounted an ## Building the app ```sh -docker-compose build +docker compose build ``` ## Running it @@ -13,38 +13,38 @@ docker-compose build View the CLI tool help/description: ```sh -docker-compose run --rm bfs help +docker compose run --rm bfs help ``` Adds test data to the default watch directory: ```sh -docker-compose run --rm bfs seed +docker compose run --rm bfs seed ``` Run the app in the background. It will continue running, monitoring for .xml files to process every 10s. ```sh -docker-compose up -d -docker-compose logs -f # view processing logs in real time +docker compose up -d +docker compose logs -f # view processing logs in real time ``` Watch a non-standard directory: ```sh -docker-compose run --rm bfs watch /path/in/container # absolute path -docker-compose run --rm bfs watch data/somedir # path relative to /opt/app-root/src +docker compose run --rm bfs watch /path/in/container # absolute path +docker compose run --rm bfs watch data/somedir # path relative to /opt/app-root/src ``` Process a specific file: ```sh -docker-compose run --rm bfs process /abs/path/to/myfile.xml # absolute path -docker-compose run --rm bfs process data/invoicing/pay/somefile.xml # relative path +docker compose run --rm bfs process /abs/path/to/myfile.xml # absolute path +docker compose run --rm bfs process data/invoicing/pay/somefile.xml # relative path ``` Delete previously processed files and error logs: ```sh -docker-compose run --rm bfs clear +docker compose run --rm bfs clear ``` diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..cd9bd27 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,14 @@ +services: + app: + build: !reset + image: ${DOCKER_APP_IMAGE} + environment: !override + SKIP_SFTP: "skip_sftp" + volumes: !override + - artifacts:/opt/app/artifacts + secrets: !reset + +volumes: + artifacts: + +secrets: !reset diff --git a/docker-compose.yml b/docker-compose.yml index 1f0051b..6677b7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,18 @@ services: - bfs: + app: build: . command: watch --interval 10 - image: containers.lib.berkeley.edu/lap/bfs/development:latest init: true environment: COA_APP_ID: "${COA_APP_ID}" COA_APP_KEY: "${COA_APP_KEY}" + BFS_SFTP_USER: "${BFS_SFTP_USER}" volumes: - - ./:/opt/app-root/src:rw + - ./:/opt/app:rw - ./secrets:/run/secrets:ro secrets: - source: SSH_KEY - target: /opt/app/src/.ssh/id_rsa + target: /opt/app/.ssh/id_rsa uid: "40061" gid: "40061" mode: 0400 @@ -20,5 +20,3 @@ services: secrets: SSH_KEY: file: secrets/SSH_KEY - -version: "3.8" diff --git a/lib/sftp_bfs.rb b/lib/sftp_bfs.rb index b6ad4f5..757bd78 100644 --- a/lib/sftp_bfs.rb +++ b/lib/sftp_bfs.rb @@ -12,7 +12,7 @@ def self.sftp_bfs() begin sftp = Net::SFTP.start( 'ucopmft-in.ucop.edu', - 'cUCB100_library', + ENV['BFS_SFTP_USER'], { append_all_supported_algorithms: true } ) diff --git a/spec/bfs_spec.rb b/spec/bfs_spec.rb index e0d8970..037ecbc 100644 --- a/spec/bfs_spec.rb +++ b/spec/bfs_spec.rb @@ -30,9 +30,4 @@ expect(Pathname.new(bfs_file)).to_not exist end -# it 'refreshes fixture data from incoming' do -# GOBI.refresh! -# expect(Pathname.new(incoming_file)).to exist -# end - end