From a873e8914abf0c32a676dda1738cbf8a6c571e26 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 10:32:48 +0800 Subject: [PATCH 01/40] =?UTF-8?q?=E8=A3=9C=E6=8E=92=E7=89=88=E5=99=A8?= =?UTF-8?q?=E6=AA=A2=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 28 +++++++++++++++++++++++++--- shellcheck.sh | 9 +++++++++ tox.ini | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 shellcheck.sh create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index 3092055..b76d272 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ branches: only: - main before_install: + - pip install --upgrade pip setuptools - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - sudo ./aws/install @@ -22,12 +23,33 @@ before_install: jobs: include: - name: Check YAML format + install: + - pip install tox + script: + - tox -e yamllint + - name: Check Markdown Format + install: + - pip install tox + script: + - tox -e pymarkdown + - name: Check Gherkin format + language: node_js + node_js: 22 before_install: skip install: - - pip install --upgrade pip setuptools - - pip install yamllint + - npm install gherkin-lint + script: + - npx gherkin-lint + - name: Check Python Format + install: + - pip install tox + script: + - tox -e flake8 + - name: Check Bash format + install: + - pip install tox script: - - yamllint . + - tox -e shellcheck - name: backup once install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach diff --git a/shellcheck.sh b/shellcheck.sh new file mode 100644 index 0000000..fced2ed --- /dev/null +++ b/shellcheck.sh @@ -0,0 +1,9 @@ +#!/bin/bash +exit_code=0 +while IFS= read -r -d '' file +do + shellcheck --severity=info "$file"; + tsitkai="$?" + exit_code=$(( tsitkai != 0 ? tsitkai : exit_code)) +done < <(find . -type f -name '*.sh' -not -path './venv/*' -print0) +exit $(( exit_code == 0 ? 0 : 1)) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..559d66c --- /dev/null +++ b/tox.ini @@ -0,0 +1,36 @@ +[tox] +skipsdist = True + +[flake8] +max-line-length = 79 +exclude = + .git + .tox + venv + +[testenv:yamllint] +deps = + yamllint +commands = + yamllint . + +[testenv:flake8] +deps = + flake8 +commands = + flake8 . --show-source --count + +[testenv:pymarkdown] +deps = + pymarkdownlnt +commands = + pymarkdown \ + --strict-config \ + --disable-rules md013,md029 \ + scan . + +[testenv:shellcheck] +allowlist_externals = + bash +commands = + bash shellcheck.sh From 90d6f3b2154a1427d8d458dbfd9bccddf09462af Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 10:37:17 +0800 Subject: [PATCH 02/40] pymarkdown --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 79bd7ef..732500a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Backup databases from dockerized PostgresSQL to any S3-compatible storage with a ## Quick Start -``` +```yaml version: '3' services: postgres: @@ -46,7 +46,7 @@ services: ### Backup Multiple Databases Simultaneously -To backup multiple databases simultaneously, you can label the database containers that require backup with the `backup.postgres=true` label. The backup script will then identify all containers with this label and execute the backup command for each of them. +To backup multiple databases simultaneously, you can label the database containers that require backup with the `backup.postgres=true` label. The backup script will then identify all containers with this label and execute the backup command for each of them. ### Easy Configuration @@ -75,6 +75,7 @@ The codebase undergoes automatic testing using Travis CI, which covers backup sc ## Configuration ### S3 Storage Configurations + - `S3_ENDPOINT_URL` (required): The S3 endpoint URL in the form of `http:///` or `https:/// `. Note that the scheme should be included. - `S3_REGION`: The name of the S3 region (eg. `eu-west-1`). This may be optional depending on your storage vendor. @@ -86,7 +87,6 @@ The codebase undergoes automatic testing using Travis CI, which covers backup sc - `SCHEDULE`: The backup schedule specified in a string following [crontab syntax](https://www.man7.org/linux/man-pages/man5/crontab.5.html) where the five fields are minute, hour, day of month, month and day of week. If set to a blank string, the script will perform a instant backup and exit. The default value is a blank string. - ### GPG Key - `GPG_PUBLIC_KEY`: Base64-encoded GPG public key used in the encryption process. If not set, backup files will be uploaded and saved un-encrypted. @@ -95,11 +95,14 @@ The codebase undergoes automatic testing using Travis CI, which covers backup sc 1. [Generate a new GPG key](https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key) if there is not any existing GPG key. 2. Encode GPG public key in base64 format and write it into the `.env` file. + ```bash GPG_PUBLIC_KEY=`gpg --armor --export | base64 --wrap 0` echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" > .env ``` + 3. Export the private key and store it securely. The private key is needed when decrypting a backup file. + ```bash gpg --export-secret-keys --armor > ``` @@ -107,10 +110,13 @@ gpg --export-secret-keys --armor > #### Decrypt a Backup File 1. Import the gpg private key if it hasn't been imported yet. + ```bash gpg --import ``` + 2. Decrypt the backup file to get the original SQL. + ```bash gpg --decrypt | zcat ``` From 8f9a148aba278807397fb175ab040b86016ad532 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 10:39:18 +0800 Subject: [PATCH 03/40] =?UTF-8?q?=E5=8A=A0gherkin=E8=A6=8F=E5=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gherkin-lintrc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .gherkin-lintrc diff --git a/.gherkin-lintrc b/.gherkin-lintrc new file mode 100644 index 0000000..9b12b0b --- /dev/null +++ b/.gherkin-lintrc @@ -0,0 +1,26 @@ +{ + "no-unnamed-features": "on", + "no-unnamed-scenarios": "on", + "no-unused-variables": "on", + "no-unused-variables": "on", + "no-dupe-feature-names": "on", + "no-dupe-scenario-names": "on", + "no-empty-background": "on", + "no-empty-file": "on", + "no-examples-in-scenarios": "on", + "no-scenario-outlines-without-examples": "on", + "no-multiple-empty-lines": "on", + "use-and": "on", + "indentation": [ + "on", {} + ], + "no-trailing-spaces": "on", + "new-line-at-eof": ["on", "yes"], + "one-space-between-tags": "on", + + "no-files-without-scenarios": "off", + "allowed-tags": [ + "on", {"tags": ["@caho還沒做", "@maherekayto做好了"]} + ] +} + From 5a2e5d777adc72261cb83c5780773f3f4ab5b8f3 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 10:45:56 +0800 Subject: [PATCH 04/40] =?UTF-8?q?=E6=8E=A7=E5=88=B6before=5Fintall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b76d272..b58e17a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,6 @@ branches: - main before_install: - pip install --upgrade pip setuptools - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip -q awscliv2.zip - - sudo ./aws/install - - pip install awscli-local jobs: include: - name: Check YAML format @@ -51,6 +47,12 @@ jobs: script: - tox -e shellcheck - name: backup once + before_install: &before_install_aws + - pip install --upgrade pip setuptools + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip -q awscliv2.zip + - sudo ./aws/install + - pip install awscli-local install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -65,6 +67,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: starting message + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -77,6 +81,8 @@ jobs: - cat backup.log | grep 'multiple-databases-backup is finished, exiting.' - cat backup.log - name: crontab + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -98,6 +104,8 @@ jobs: - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.log - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.error.log - name: backup once when erasing the SCHEDULE variable + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -108,6 +116,8 @@ jobs: - docker-compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -120,6 +130,8 @@ jobs: - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' - name: GPG encrypt with asymmetric key + before_install: + <<: *before_install_aws env: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase @@ -141,6 +153,8 @@ jobs: - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - gpg --decrypt --batch --pinentry-mode loopback --passphrase-file ${GPG_PRIVATE_KEY_PASSPHRASE_PATH} postgres15.sql.gz.gpg | zcat - name: backup strategy for keeping backup in 72 hours(3 days) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -159,6 +173,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup exceeding 72 hours(3 days) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -183,6 +199,8 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -201,6 +219,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backups if they are backuped in different day. + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -225,6 +245,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping day backup only + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -249,6 +271,8 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -273,6 +297,8 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -291,6 +317,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backups if they are backuped in different month. + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -315,6 +343,8 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping month backup only + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -339,6 +369,8 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) + before_install: + <<: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach From 092148ce6ee01dfb21c7b8d3f74baf88ca0931c1 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 10:57:22 +0800 Subject: [PATCH 05/40] shellcheck --- scripts/backup.sh | 2 +- scripts/cleanup.sh | 4 ++-- scripts/start.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/backup.sh b/scripts/backup.sh index 64eabd8..15a09a1 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -23,7 +23,7 @@ all_containers=`docker container list \ for postgres_container_name in $all_containers do >&2 echo "Backuping ${postgres_container_name} is starting." - FILE_PATH=$(filepath ${postgres_container_name} 'now') + FILE_PATH=$(filepath "${postgres_container_name}" 'now') docker exec "${postgres_container_name}" pg_dump -U postgres \ | gzip \ | ${ENCRYPT_COMMAND} \ diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index ebff041..8f11e07 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -5,7 +5,7 @@ source /app/filepath.sh CONTAINER_NAME=$1 -FILE_PATH=$(filepath ${CONTAINER_NAME} "${MAX_PERIOD_IN_HOURS_TO_KEEP_EVERY_BACKUPS} hours ago") +FILE_PATH=$(filepath "${CONTAINER_NAME}" "${MAX_PERIOD_IN_HOURS_TO_KEEP_EVERY_BACKUPS} hours ago") temp_dir=$(mktemp -d) ALL_FILES="${temp_dir}/all.list" @@ -43,7 +43,7 @@ done for month in $(seq 1 "${MAX_PERIOD_IN_MONTHS_TO_KEEP_MONTHLY_BACKUPS}") do TARGET_DAY=`date "+%Y-%m-01" --date "${month} months ago"` - FILE_PATH=$(filepath ${CONTAINER_NAME} "${TARGET_DAY}") + FILE_PATH=$(filepath "${CONTAINER_NAME}" "${TARGET_DAY}") aws s3api list-objects-v2 \ --endpoint-url "${S3_ENDPOINT_URL}" \ --bucket "${S3_BUCKET}" \ diff --git a/scripts/start.sh b/scripts/start.sh index ad3b5bb..daf05e0 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -31,7 +31,7 @@ if [ -z "${GPG_PUBLIC_KEY}" ]; then >&2 echo 'There is not a GPG_PUBLIC_KEY, all backup files will not be encrypted.' else >&2 echo 'There is the GPG_PUBLIC_KEY, all backup files will be encrypted.' - echo ${GPG_PUBLIC_KEY} | base64 -d > ${GPG_PUBLIC_KEY_PATH} + echo "${GPG_PUBLIC_KEY}" | base64 -d > ${GPG_PUBLIC_KEY_PATH} fi if [ -z "${SCHEDULE}" ]; then >&2 echo "multiple-databases-backup is starting." From 0f846636cedf0f295bc9753bedea62a536ccbbd3 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 11:04:59 +0800 Subject: [PATCH 06/40] =?UTF-8?q?=E4=BF=AE.travis.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index b58e17a..539b228 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,8 +67,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: starting message - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -81,8 +80,7 @@ jobs: - cat backup.log | grep 'multiple-databases-backup is finished, exiting.' - cat backup.log - name: crontab - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -104,8 +102,7 @@ jobs: - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.log - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.error.log - name: backup once when erasing the SCHEDULE variable - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -116,8 +113,7 @@ jobs: - docker-compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -130,8 +126,7 @@ jobs: - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' - name: GPG encrypt with asymmetric key - before_install: - <<: *before_install_aws + before_install: *before_install_aws env: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase @@ -153,8 +148,7 @@ jobs: - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - gpg --decrypt --batch --pinentry-mode loopback --passphrase-file ${GPG_PRIVATE_KEY_PASSPHRASE_PATH} postgres15.sql.gz.gpg | zcat - name: backup strategy for keeping backup in 72 hours(3 days) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -173,8 +167,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup exceeding 72 hours(3 days) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -199,8 +192,7 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -219,8 +211,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backups if they are backuped in different day. - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -245,8 +236,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping day backup only - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -271,8 +261,7 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -297,8 +286,7 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -317,8 +305,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backups if they are backuped in different month. - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -343,8 +330,7 @@ jobs: - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping month backup only - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach @@ -369,8 +355,7 @@ jobs: - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) - before_install: - <<: *before_install_aws + before_install: *before_install_aws install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach From 00017a81e74c728b1336cc712ed4f182eb4538a5 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 11:12:40 +0800 Subject: [PATCH 07/40] =?UTF-8?q?=E4=BF=AESC1091?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 6 +++--- scripts/backup.sh | 2 +- scripts/cleanup.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index d095583..f7b55f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,6 @@ RUN apt install -y unzip && \ /aws_build/aws/install && \ rm -rf /aws_build/ -COPY scripts/ /app/ - -CMD bash /app/start.sh +WORKDIR /app/ +COPY scripts/ ./ +CMD bash start.sh diff --git a/scripts/backup.sh b/scripts/backup.sh index 15a09a1..c422fa0 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -5,7 +5,7 @@ export AWS_DEFAULT_REGION="${S3_REGION}" export AWS_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY}" -source /app/filepath.sh +source filepath.sh if [ -z "${GPG_PUBLIC_KEY}" ]; then ENCRYPT_COMMAND="cat" diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 8f11e07..5c03b0d 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -source /app/filepath.sh +source filepath.sh CONTAINER_NAME=$1 @@ -29,7 +29,7 @@ aws s3api list-objects-v2 \ for day in $(seq 1 "${MAX_PERIOD_IN_DAYS_TO_KEEP_DAILY_BACKUPS}") do TARGET_DAY=`date "+%Y-%m-%d" --date "${day} days ago"` - FILE_PATH=$(filepath ${CONTAINER_NAME} "${TARGET_DAY}") + FILE_PATH=$(filepath "${CONTAINER_NAME}" "${TARGET_DAY}") aws s3api list-objects-v2 \ --endpoint-url "${S3_ENDPOINT_URL}" \ --bucket "${S3_BUCKET}" \ @@ -61,7 +61,7 @@ do aws s3api delete-object \ --endpoint-url "${S3_ENDPOINT_URL}" \ --bucket "${S3_BUCKET}" \ - --key ${filename} + --key "${filename}" done rm -rf "${temp_dir}" From f707e220e67c5e1f1a90381a29883fcad8280ef6 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 11:30:36 +0800 Subject: [PATCH 08/40] SC2155 --- scripts/filepath.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/filepath.sh b/scripts/filepath.sh index 5690506..0f06e02 100644 --- a/scripts/filepath.sh +++ b/scripts/filepath.sh @@ -3,9 +3,13 @@ set -euo pipefail function filepath() { - local container_name="$1" - local target_time="$2" - local DATE=`date "+%Y-%m-%d" --date "${target_time}"` - local TIME=`date "+%Y%m%dT%H%M" --date "${target_time}"` + local container_name + local target_time + local DATE + local TIME + container_name="$1" + target_time="$2" + DATE=`date "+%Y-%m-%d" --date "${target_time}"` + TIME=`date "+%Y%m%dT%H%M" --date "${target_time}"` echo "${container_name}/${DATE}/${container_name}_${TIME}" } From 6460c486c3e2a8a7b6615f3a65f7d5f80d2889a0 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 11:35:33 +0800 Subject: [PATCH 09/40] SC2013 --- scripts/cleanup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 5c03b0d..3532c70 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -54,9 +54,10 @@ do >> "${PRESERVE_FILES}" done -for filename in `cat "${ALL_FILES}" \ +cat "${ALL_FILES}" \ | grep --invert-match --line-regexp --file "${PRESERVE_FILES}" \ - | sed 's/^"\(.*\)"$/\1/g'` + | sed 's/^"\(.*\)"$/\1/g' \ + | while IFS= read -r filename do aws s3api delete-object \ --endpoint-url "${S3_ENDPOINT_URL}" \ From 3d1b0168a1c3ce522960c9ee44dfc3513aa013b5 Mon Sep 17 00:00:00 2001 From: Asing i Mac M1 16G Date: Thu, 3 Jul 2025 11:43:52 +0800 Subject: [PATCH 10/40] =?UTF-8?q?SC1091=E6=94=8F=E6=94=B9=E5=81=9Ascripts/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 ++-- scripts/backup.sh | 2 +- scripts/cleanup.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f7b55f6..170abf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,5 +25,5 @@ RUN apt install -y unzip && \ rm -rf /aws_build/ WORKDIR /app/ -COPY scripts/ ./ -CMD bash start.sh +COPY scripts/ ./scripts/ +CMD bash scripts/start.sh diff --git a/scripts/backup.sh b/scripts/backup.sh index c422fa0..b0a12df 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -5,7 +5,7 @@ export AWS_DEFAULT_REGION="${S3_REGION}" export AWS_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY}" -source filepath.sh +source scripts/filepath.sh if [ -z "${GPG_PUBLIC_KEY}" ]; then ENCRYPT_COMMAND="cat" diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 3532c70..79edb0a 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -source filepath.sh +source scripts/filepath.sh CONTAINER_NAME=$1 From ec192c55517b4253c5297f6c04eacdfffbabde50 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Thu, 3 Jul 2025 16:39:00 +0800 Subject: [PATCH 11/40] =?UTF-8?q?=E4=BF=AESC1091?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shellcheck.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shellcheck.sh b/shellcheck.sh index fced2ed..7542ba1 100644 --- a/shellcheck.sh +++ b/shellcheck.sh @@ -1,9 +1,12 @@ #!/bin/bash exit_code=0 +shellcheck --severity=info scripts/*; +tsitkai="$?" +exit_code=$(( tsitkai != 0 ? tsitkai : exit_code)) while IFS= read -r -d '' file do shellcheck --severity=info "$file"; tsitkai="$?" exit_code=$(( tsitkai != 0 ? tsitkai : exit_code)) -done < <(find . -type f -name '*.sh' -not -path './venv/*' -print0) +done < <(find . -type f -name '*.sh' -not -path './venv/*' -not -path './scripts/*' -print0) exit $(( exit_code == 0 ? 0 : 1)) From 9a270f889e774927f0337c0acb9b05af57985984 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Thu, 3 Jul 2025 17:07:40 +0800 Subject: [PATCH 12/40] =?UTF-8?q?=E5=9B=BA=E5=AE=9Aawscli=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 8 ++++---- scripts/backup.sh | 2 +- scripts/cleanup.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 170abf3..2905d2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive @@ -19,11 +19,11 @@ RUN apt update && \ RUN apt install -y unzip && \ mkdir /aws_build/ && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/aws_build/awscliv2.zip" && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.18.12.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ WORKDIR /app/ -COPY scripts/ ./scripts/ -CMD bash scripts/start.sh +COPY scripts/ /app/ +CMD bash /app/start.sh diff --git a/scripts/backup.sh b/scripts/backup.sh index b0a12df..15a09a1 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -5,7 +5,7 @@ export AWS_DEFAULT_REGION="${S3_REGION}" export AWS_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY}" -source scripts/filepath.sh +source /app/filepath.sh if [ -z "${GPG_PUBLIC_KEY}" ]; then ENCRYPT_COMMAND="cat" diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 79edb0a..3ab39ca 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -source scripts/filepath.sh +source /app/filepath.sh CONTAINER_NAME=$1 From 9118e6de7e1a183da8a38ab06ed4dbb57c9ba563 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Thu, 3 Jul 2025 17:18:46 +0800 Subject: [PATCH 13/40] =?UTF-8?q?Travis=20CI=E7=94=A8pip=E9=AC=A5awscli?= =?UTF-8?q?=E8=BC=83khin-kh=C3=B3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 5 +---- requirements_travisci.in | 4 ++++ requirements_travisci.txt | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 requirements_travisci.in create mode 100644 requirements_travisci.txt diff --git a/.travis.yml b/.travis.yml index 539b228..2ca6ba1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,10 +49,7 @@ jobs: - name: backup once before_install: &before_install_aws - pip install --upgrade pip setuptools - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip -q awscliv2.zip - - sudo ./aws/install - - pip install awscli-local + - pip install -r requirements_travisci.txt install: - docker-compose --file tests/postgres15/docker-compose.yml up --detach - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach diff --git a/requirements_travisci.in b/requirements_travisci.in new file mode 100644 index 0000000..d13a290 --- /dev/null +++ b/requirements_travisci.in @@ -0,0 +1,4 @@ +awscli +boto3~=1.35.0 + +awscli-local diff --git a/requirements_travisci.txt b/requirements_travisci.txt new file mode 100644 index 0000000..5f37f7e --- /dev/null +++ b/requirements_travisci.txt @@ -0,0 +1,45 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements_travisci.in +# +awscli==1.36.40 + # via -r requirements_travisci.in +awscli-local==0.22.0 + # via -r requirements_travisci.in +boto3==1.35.99 + # via + # -r requirements_travisci.in + # localstack-client +botocore==1.35.99 + # via + # awscli + # boto3 + # s3transfer +colorama==0.4.6 + # via awscli +docutils==0.16 + # via awscli +jmespath==1.0.1 + # via + # boto3 + # botocore +localstack-client==2.10 + # via awscli-local +pyasn1==0.6.1 + # via rsa +python-dateutil==2.9.0.post0 + # via botocore +pyyaml==6.0.2 + # via awscli +rsa==4.7.2 + # via awscli +s3transfer==0.10.4 + # via + # awscli + # boto3 +six==1.17.0 + # via python-dateutil +urllib3==2.5.0 + # via botocore From bc879d107b5f8251accedb057422aacf81a35be2 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Thu, 3 Jul 2025 17:26:15 +0800 Subject: [PATCH 14/40] =?UTF-8?q?docker=E5=85=A7=C3=AA=20path=EF=BC=8Cshel?= =?UTF-8?q?lcheck=E7=84=A1=E6=B3=95=E5=BA=A6=E6=AA=A2=E6=9F=A5=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .shellcheckrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 .shellcheckrc diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..0182ace --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +disable=SC1091 From a53221591942afebf5dbe0227876d02933f08d82 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Thu, 3 Jul 2025 17:43:20 +0800 Subject: [PATCH 15/40] =?UTF-8?q?=E5=81=87=E8=A8=AD=E6=8F=9B=E5=81=9Adocke?= =?UTF-8?q?r=20compose=20plugin=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 172 ++++++++++++++++++++++++++-------------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ca6ba1..4e12501 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,74 +51,74 @@ jobs: - pip install --upgrade pip setuptools - pip install -r requirements_travisci.txt install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker-compose --file tests/docker-compose-backup.yml up --detach + - docker compose --file tests/docker compose-backup.yml up --detach - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - sleep 10 - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" postgres15.sql.gz + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: starting message before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker-compose --file tests/docker-compose-backup.yml run --rm backup | tee backup.log + - docker compose --file tests/docker compose-backup.yml run --rm backup | tee backup.log - cat backup.log | grep 'multiple-databases-backup is starting.' - cat backup.log | grep 'multiple-databases-backup is finished, exiting.' - cat backup.log - name: crontab before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker-compose --file tests/docker-compose-backup-minute.yml up --detach + - docker compose --file tests/docker compose-backup-minute.yml up --detach - sleep 1m - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" postgres15.sql.gz + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - sleep 1m - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" postgres15.sql.gz + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.log - - docker-compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.error.log + - docker compose --file tests/docker compose-backup-minute.yml exec backup cat /var/log/cron.log + - docker compose --file tests/docker compose-backup-minute.yml exec backup cat /var/log/cron.error.log - name: backup once when erasing the SCHEDULE variable before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker-compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log + - docker compose --file tests/docker compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker-compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." | tee error.log) + - (docker compose --file tests/docker compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' @@ -128,9 +128,9 @@ jobs: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-encrypt.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-encrypt.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -139,235 +139,235 @@ jobs: - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" | tee tests/.env - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - - docker-compose --file tests/docker-compose-backup-encrypt.yml run --rm backup + - docker compose --file tests/docker compose-backup-encrypt.yml run --rm backup - sleep 10 - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz.gpg" postgres15.sql.gz.gpg + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz.gpg" postgres15.sql.gz.gpg - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - gpg --decrypt --batch --pinentry-mode loopback --passphrase-file ${GPG_PRIVATE_KEY_PASSPHRASE_PATH} postgres15.sql.gz.gpg | zcat - name: backup strategy for keeping backup in 72 hours(3 days) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-%d %H:%M" --date '3 days ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup exceeding 72 hours(3 days) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-%d %H:%M" --date '3 days ago 1 hour ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" - DAY_TARGET_TIME=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - DAY_DATE=`date "+%Y-%m-%d" --date "${DAY_TARGET_TIME}"` - DAY_TIME=`date "+%Y%m%dT%H%M" --date "${DAY_TARGET_TIME}"` - - DAY_BACKUP="postgres15_postgres_1/${DAY_DATE}/postgres15_postgres_1_${DAY_TIME}.sql.gz" + - DAY_BACKUP="postgres15-postgres-1/${DAY_DATE}/postgres15-postgres-1_${DAY_TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-01" --date '3 months ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backups if they are backuped in different day. before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET1_TIME=`date "+%Y-%m-01" --date '2 months ago'` - DATE1=`date "+%Y-%m-%d" --date "${TARGET1_TIME}"` - TIME1=`date "+%Y%m%dT%H%M" --date "${TARGET1_TIME}"` - - DAY1_BACKUP="postgres15_postgres_1/${DATE1}/postgres15_postgres_1_${TIME1}.sql.gz" + - DAY1_BACKUP="postgres15-postgres-1/${DATE1}/postgres15-postgres-1_${TIME1}.sql.gz" - TARGET2_TIME=`date "+%Y-%m-02" --date '2 months ago'` - DATE2=`date "+%Y-%m-%d" --date "${TARGET2_TIME}"` - TIME2=`date "+%Y%m%dT%H%M" --date "${TARGET2_TIME}"` - - DAY2_BACKUP="postgres15_postgres_1/${DATE2}/postgres15_postgres_1_${TIME2}.sql.gz" + - DAY2_BACKUP="postgres15-postgres-1/${DATE2}/postgres15-postgres-1_${TIME2}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY1_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY1_BACKUP}" postgres15-1.sql.gz - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping day backup only before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET1_TIME="`date "+%Y-%m-01 09:00" --date '3 months ago'`" - DATE1=`date "+%Y-%m-%d" --date "${TARGET1_TIME}"` - TIME1=`date "+%Y%m%dT%H%M" --date "${TARGET1_TIME}"` - - DAY_BACKUP="postgres15_postgres_1/${DATE1}/postgres15_postgres_1_${TIME1}.sql.gz" + - DAY_BACKUP="postgres15-postgres-1/${DATE1}/postgres15-postgres-1_${TIME1}.sql.gz" - TARGET2_TIME="`date "+%Y-%m-01 10:00" --date '3 months ago'`" - DATE2=`date "+%Y-%m-%d" --date "${TARGET2_TIME}"` - TIME2=`date "+%Y%m%dT%H%M" --date "${TARGET2_TIME}"` - - NOT_DAY_BACKUP="postgres15_postgres_1/${DATE2}/postgres15_postgres_1_${TIME2}.sql.gz" + - NOT_DAY_BACKUP="postgres15-postgres-1/${DATE2}/postgres15-postgres-1_${TIME2}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-10" --date '5 months ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" - MONTH_TARGET_TIME=`date "+%Y-%m-01" --date "${TARGET_TIME}"` - MONTH_DATE=`date "+%Y-%m-%d" --date "${MONTH_TARGET_TIME}"` - MONTH_TIME=`date "+%Y%m%dT%H%M" --date "${MONTH_TARGET_TIME}"` - - MONTH_BACKUP="postgres15_postgres_1/${MONTH_DATE}/postgres15_postgres_1_${MONTH_TIME}.sql.gz" + - MONTH_BACKUP="postgres15-postgres-1/${MONTH_DATE}/postgres15-postgres-1_${MONTH_TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-01" --date '3 years ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backups if they are backuped in different month. before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET1_TIME=`date "+%Y-01-01" --date '1 years ago'` - DATE1=`date "+%Y-%m-%d" --date "${TARGET1_TIME}"` - TIME1=`date "+%Y%m%dT%H%M" --date "${TARGET1_TIME}"` - - MONTH1_BACKUP="postgres15_postgres_1/${DATE1}/postgres15_postgres_1_${TIME1}.sql.gz" + - MONTH1_BACKUP="postgres15-postgres-1/${DATE1}/postgres15-postgres-1_${TIME1}.sql.gz" - TARGET2_TIME=`date "+%Y-02-02" --date '1 years ago'` - DATE2=`date "+%Y-%m-%d" --date "${TARGET2_TIME}"` - TIME2=`date "+%Y%m%dT%H%M" --date "${TARGET2_TIME}"` - - MONTH2_BACKUP="postgres15_postgres_1/${DATE2}/postgres15_postgres_1_${TIME2}.sql.gz" + - MONTH2_BACKUP="postgres15-postgres-1/${DATE2}/postgres15-postgres-1_${TIME2}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH1_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH1_BACKUP}" postgres15-1.sql.gz - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping month backup only before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET1_TIME="`date "+%Y-%m-01 09:00" --date '10 months ago'`" - DATE1=`date "+%Y-%m-%d" --date "${TARGET1_TIME}"` - TIME1=`date "+%Y%m%dT%H%M" --date "${TARGET1_TIME}"` - - MONTH_BACKUP="postgres15_postgres_1/${DATE1}/postgres15_postgres_1_${TIME1}.sql.gz" + - MONTH_BACKUP="postgres15-postgres-1/${DATE1}/postgres15-postgres-1_${TIME1}.sql.gz" - TARGET2_TIME="`date "+%Y-%m-20 09:00" --date '10 months ago'`" - DATE2=`date "+%Y-%m-%d" --date "${TARGET2_TIME}"` - TIME2=`date "+%Y%m%dT%H%M" --date "${TARGET2_TIME}"` - - NOT_MONTH_BACKUP="postgres15_postgres_1/${DATE2}/postgres15_postgres_1_${TIME2}.sql.gz" + - NOT_MONTH_BACKUP="postgres15-postgres-1/${DATE2}/postgres15-postgres-1_${TIME2}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) before_install: *before_install_aws install: - - docker-compose --file tests/postgres15/docker-compose.yml up --detach - - docker-compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker-compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker compose.yml up --detach + - docker compose --file tests/s3/docker compose-localstack.yml up --detach + - docker compose --file tests/docker compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: - TARGET_TIME=`date "+%Y-%m-01" --date '3 years ago 1 months ago'` - DATE=`date "+%Y-%m-%d" --date "${TARGET_TIME}"` - TIME=`date "+%Y%m%dT%H%M" --date "${TARGET_TIME}"` - - OLD_BACKUP="postgres15_postgres_1/${DATE}/postgres15_postgres_1_${TIME}.sql.gz" + - OLD_BACKUP="postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker-compose --file tests/docker-compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" From 89681a2de01b458c8da589b688acdbc084a15c1b Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:05:45 +0800 Subject: [PATCH 16/40] =?UTF-8?q?=E9=AC=A5compose=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4e12501..460684e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,6 +50,10 @@ jobs: before_install: &before_install_aws - pip install --upgrade pip setuptools - pip install -r requirements_travisci.txt + - DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} + - mkdir -p $DOCKER_CONFIG/cli-plugins + - curl -SL https://github.com/docker/compose/releases/download/v2.38.1/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose + - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose install: - docker compose --file tests/postgres15/docker compose.yml up --detach - docker compose --file tests/s3/docker compose-localstack.yml up --detach From 498566a94554057c64b374afb928728401909c0f Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:26:08 +0800 Subject: [PATCH 17/40] Siu yaml mia --- .travis.yml | 132 ++++++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.travis.yml b/.travis.yml index 460684e..a6c1376 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,13 +55,13 @@ jobs: - curl -SL https://github.com/docker/compose/releases/download/v2.38.1/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker compose-backup.yml up --detach + - docker compose --file tests/docker-compose-backup.yml up --detach - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - sleep 10 @@ -70,26 +70,26 @@ jobs: - name: starting message before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker compose-backup.yml run --rm backup | tee backup.log + - docker compose --file tests/docker-compose-backup.yml run --rm backup | tee backup.log - cat backup.log | grep 'multiple-databases-backup is starting.' - cat backup.log | grep 'multiple-databases-backup is finished, exiting.' - cat backup.log - name: crontab before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker compose-backup-minute.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml up --detach - sleep 1m - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` @@ -100,29 +100,29 @@ jobs: - TIME=`date "+%Y%m%dT%H%M"` - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - - docker compose --file tests/docker compose-backup-minute.yml exec backup cat /var/log/cron.log - - docker compose --file tests/docker compose-backup-minute.yml exec backup cat /var/log/cron.error.log + - docker compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.log + - docker compose --file tests/docker-compose-backup-minute.yml exec backup cat /var/log/cron.error.log - name: backup once when erasing the SCHEDULE variable before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log + - docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' @@ -132,9 +132,9 @@ jobs: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-encrypt.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-encrypt.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -143,7 +143,7 @@ jobs: - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" | tee tests/.env - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - - docker compose --file tests/docker compose-backup-encrypt.yml run --rm backup + - docker compose --file tests/docker-compose-backup-encrypt.yml run --rm backup - sleep 10 - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "postgres15-postgres-1/${DATE}/postgres15-postgres-1_${TIME}.sql.gz.gpg" postgres15.sql.gz.gpg - awslocal s3api list-objects --bucket "${BUCKET_NAME}" @@ -151,9 +151,9 @@ jobs: - name: backup strategy for keeping backup in 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -164,15 +164,15 @@ jobs: script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup exceeding 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -188,16 +188,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -208,15 +208,15 @@ jobs: script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backups if they are backuped in different day. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -232,16 +232,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY1_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY1_BACKUP}" postgres15-1.sql.gz - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${DAY2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping day backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -257,16 +257,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -282,16 +282,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -302,15 +302,15 @@ jobs: script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backups if they are backuped in different month. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -326,16 +326,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH1_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH1_BACKUP}" postgres15-1.sql.gz - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${MONTH2_BACKUP}" postgres15-2.sql.gz - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting backup because of keeping month backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -351,16 +351,16 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker compose.yml up --detach - - docker compose --file tests/s3/docker compose-localstack.yml up --detach - - docker compose --file tests/docker compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach + - docker compose --file tests/docker-compose-backup-strategy.yml build - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -371,7 +371,7 @@ jobs: script: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - - docker compose --file tests/docker compose-backup-strategy.yml run --rm backup + - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" From 6e0cf5116f68291da2feecbd393d6c5324e0ab81 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:35:48 +0800 Subject: [PATCH 18/40] =?UTF-8?q?=E7=94=A8=E4=B8=8A=E6=96=B0=E7=89=88cli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2905d2d..f03dbfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,9 +17,10 @@ RUN apt update && \ apt-get update && \ apt install -y docker-ce-cli cron gnupg jq +# https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 RUN apt install -y unzip && \ mkdir /aws_build/ && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.18.12.zip" -o "/aws_build/awscliv2.zip" && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.27.49.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ From 4c3e1f4a3345832d1a840d1f104d86809045a3bb Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:38:41 +0800 Subject: [PATCH 19/40] =?UTF-8?q?=E7=94=A8ubuntu24.04?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f03dbfe..1dc8645 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive From 70bbcae814f5d7d3c855d2e10a20438b1a716739 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:46:27 +0800 Subject: [PATCH 20/40] =?UTF-8?q?=E4=BF=AECI=20pipe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 18 +++++++++--------- Dockerfile | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6c1376..7204d48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup.yml run --rm backup | tee backup.log + - docker compose --file tests/docker-compose-backup.yml run --rm backup 2>&1 | tee backup.log - cat backup.log | grep 'multiple-databases-backup is starting.' - cat backup.log | grep 'multiple-databases-backup is finished, exiting.' - cat backup.log @@ -111,7 +111,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup | tee backup.log + - docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup 2>&1 | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format before_install: *before_install_aws @@ -122,7 +122,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." 2>&1 | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' @@ -140,7 +140,7 @@ jobs: script: - gpg --import --batch ${GPG_PRIVATE_KEY_PATH} - GPG_PUBLIC_KEY=`gpg --armor --export ithuan+multiple-databases-backup@ithuan.tw | base64 --wrap 0` - - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" | tee tests/.env + - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" 2>&1 | tee tests/.env - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - docker compose --file tests/docker-compose-backup-encrypt.yml run --rm backup @@ -189,7 +189,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) @@ -258,7 +258,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) @@ -283,7 +283,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) @@ -352,7 +352,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) @@ -372,6 +372,6 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" diff --git a/Dockerfile b/Dockerfile index 1dc8645..e85d0c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:24.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive @@ -20,7 +20,7 @@ RUN apt update && \ # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 RUN apt install -y unzip && \ mkdir /aws_build/ && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.27.49.zip" -o "/aws_build/awscliv2.zip" && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.18.12.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ From c6d4d1992eb4a101d2019569e767dad665a34624 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:53:56 +0800 Subject: [PATCH 21/40] =?UTF-8?q?=E6=9B=B4=E6=96=B0awscli=E7=89=88?= =?UTF-8?q?=E6=9C=AC2.22.35=20koh=20=C4=93-t=C3=A0ng=E8=B5=B0=EF=BC=8C2.23?= =?UTF-8?q?.0=E5=B0=B1=E7=84=A1=E6=B3=95=E5=BA=A6--ah=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e85d0c6..5b4893a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN apt update && \ # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 RUN apt install -y unzip && \ mkdir /aws_build/ && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.18.12.zip" -o "/aws_build/awscliv2.zip" && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ From f981112753cf22bdb8a053f1bc05bb75952b3b8b Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 08:54:22 +0800 Subject: [PATCH 22/40] =?UTF-8?q?=E6=8F=9B=E6=96=B0=E7=89=88compose?= =?UTF-8?q?=E8=A6=8F=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/docker-compose-backup-encrypt.yml | 1 - tests/docker-compose-backup-minute.yml | 1 - tests/docker-compose-backup-strategy.yml | 1 - tests/docker-compose-backup.yml | 1 - tests/postgres15/docker-compose.yml | 1 - tests/s3/docker-compose-localstack.yml | 1 - 6 files changed, 6 deletions(-) diff --git a/tests/docker-compose-backup-encrypt.yml b/tests/docker-compose-backup-encrypt.yml index 72cc381..1b019a0 100644 --- a/tests/docker-compose-backup-encrypt.yml +++ b/tests/docker-compose-backup-encrypt.yml @@ -1,5 +1,4 @@ --- -version: '3' services: backup: build: ../ diff --git a/tests/docker-compose-backup-minute.yml b/tests/docker-compose-backup-minute.yml index 57cf0ca..70c2f41 100644 --- a/tests/docker-compose-backup-minute.yml +++ b/tests/docker-compose-backup-minute.yml @@ -1,5 +1,4 @@ --- -version: '3' services: backup: build: ../ diff --git a/tests/docker-compose-backup-strategy.yml b/tests/docker-compose-backup-strategy.yml index 538c2ec..2dc45fc 100644 --- a/tests/docker-compose-backup-strategy.yml +++ b/tests/docker-compose-backup-strategy.yml @@ -1,5 +1,4 @@ --- -version: '3' services: backup: build: ../ diff --git a/tests/docker-compose-backup.yml b/tests/docker-compose-backup.yml index e23213a..c88b876 100644 --- a/tests/docker-compose-backup.yml +++ b/tests/docker-compose-backup.yml @@ -1,5 +1,4 @@ --- -version: '3' services: backup: build: ../ diff --git a/tests/postgres15/docker-compose.yml b/tests/postgres15/docker-compose.yml index 3a2393f..b8587c5 100644 --- a/tests/postgres15/docker-compose.yml +++ b/tests/postgres15/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: '3' services: postgres: image: postgres:15 diff --git a/tests/s3/docker-compose-localstack.yml b/tests/s3/docker-compose-localstack.yml index c7cde06..4c04a9e 100644 --- a/tests/s3/docker-compose-localstack.yml +++ b/tests/s3/docker-compose-localstack.yml @@ -1,5 +1,4 @@ --- -version: '3' services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" From 66fe2075796c1483a8e32ca867a1c71f62b959dc Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 09:00:32 +0800 Subject: [PATCH 23/40] ubuntu24 kah nonroot --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5b4893a..c754d88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive +RUN addgroup --gid 1000 nonroot && \ + adduser --uid 1000 --disabled-password --ingroup nonroot --quiet nonroot + RUN apt update && \ apt install -y \ ca-certificates \ @@ -27,4 +30,6 @@ RUN apt install -y unzip && \ WORKDIR /app/ COPY scripts/ /app/ + +USER nonroot CMD bash /app/start.sh From 2c8f3a6ecbee9777917c9e88a8565061d46cd17c Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 09:43:55 +0800 Subject: [PATCH 24/40] nonroot --- Dockerfile | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c754d88..863c532 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,23 @@ FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive - -RUN addgroup --gid 1000 nonroot && \ - adduser --uid 1000 --disabled-password --ingroup nonroot --quiet nonroot - +RUN groupadd --system --gid 138 docker RUN apt update && \ - apt install -y \ - ca-certificates \ - curl \ - gnupg \ - lsb-release && \ - mkdir -m 0755 -p /etc/apt/keyrings && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ - | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ - apt-get update && \ - apt install -y docker-ce-cli cron gnupg jq + apt install -y docker.io cron gnupg jq # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 -RUN apt install -y unzip && \ +RUN apt install -y curl unzip && \ mkdir /aws_build/ && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ +RUN useradd --uid 1001 nonroot --user-group # --groups docker +RUN usermod -aG docker nonroot +RUN touch /etc/environment +RUN chown nonroot:nonroot /etc/environment + WORKDIR /app/ COPY scripts/ /app/ From 8081163149c994da697e3d4740f03bbe1e0fe4ba Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:10:43 +0800 Subject: [PATCH 25/40] =?UTF-8?q?grep=20-v=E6=9C=89=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E5=81=9Acomm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/cleanup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 3ab39ca..4378f24 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -54,8 +54,11 @@ do >> "${PRESERVE_FILES}" done -cat "${ALL_FILES}" \ - | grep --invert-match --line-regexp --file "${PRESERVE_FILES}" \ +sort -u "${ALL_FILES}" > "${temp_dir}/tmp.list" +mv "${temp_dir}/tmp.list" "${ALL_FILES}" +sort -u "${PRESERVE_FILES}" > "${temp_dir}/tmp.list" +mv "${temp_dir}/tmp.list" "${PRESERVE_FILES}" +comm -1 "${ALL_FILES}" "${PRESERVE_FILES}" \ | sed 's/^"\(.*\)"$/\1/g' \ | while IFS= read -r filename do From 0dab285d09403d0d59222ef2ebdc07c14be68036 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:10:43 +0800 Subject: [PATCH 26/40] =?UTF-8?q?grep=20-v=E6=9C=89=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E5=81=9Acomm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/cleanup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 3ab39ca..4378f24 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -54,8 +54,11 @@ do >> "${PRESERVE_FILES}" done -cat "${ALL_FILES}" \ - | grep --invert-match --line-regexp --file "${PRESERVE_FILES}" \ +sort -u "${ALL_FILES}" > "${temp_dir}/tmp.list" +mv "${temp_dir}/tmp.list" "${ALL_FILES}" +sort -u "${PRESERVE_FILES}" > "${temp_dir}/tmp.list" +mv "${temp_dir}/tmp.list" "${PRESERVE_FILES}" +comm -1 "${ALL_FILES}" "${PRESERVE_FILES}" \ | sed 's/^"\(.*\)"$/\1/g' \ | while IFS= read -r filename do From d5f5c9b4f3ff515a606dc36f21fdf9f2b2885b6a Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:12:39 +0800 Subject: [PATCH 27/40] =?UTF-8?q?docker=20build=E9=81=8E=E7=A8=8Bm=C3=A0i?= =?UTF-8?q?=E5=8D=B0=EF=BC=8CCI=E8=BC=83=E7=84=A1=E8=B2=A0=E6=93=94?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 100 ++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7204d48..bc0e1c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,13 +55,13 @@ jobs: - curl -SL https://github.com/docker/compose/releases/download/v2.38.1/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup.yml up --detach + - docker compose --file tests/docker-compose-backup.yml up --detach --quiet-pull - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - sleep 10 @@ -70,9 +70,9 @@ jobs: - name: starting message before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -83,13 +83,13 @@ jobs: - name: crontab before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup-minute.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml up --detach --quiet-pull - sleep 1m - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` @@ -105,9 +105,9 @@ jobs: - name: backup once when erasing the SCHEDULE variable before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -116,9 +116,9 @@ jobs: - name: exit with SCHEDULE wrong format before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -132,9 +132,9 @@ jobs: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-encrypt.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-encrypt.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -151,9 +151,9 @@ jobs: - name: backup strategy for keeping backup in 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -170,9 +170,9 @@ jobs: - name: backup strategy for deleting backup exceeding 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -195,9 +195,9 @@ jobs: - name: backup strategy for keeping day backup in 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -214,9 +214,9 @@ jobs: - name: backup strategy for keeping day backups if they are backuped in different day. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -239,9 +239,9 @@ jobs: - name: backup strategy for deleting backup because of keeping day backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -264,9 +264,9 @@ jobs: - name: backup strategy for deleting day backup exceeding 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -289,9 +289,9 @@ jobs: - name: backup strategy for keeping month backup in 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -308,9 +308,9 @@ jobs: - name: backup strategy for keeping month backups if they are backuped in different month. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -333,9 +333,9 @@ jobs: - name: backup strategy for deleting backup because of keeping month backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -358,9 +358,9 @@ jobs: - name: backup strategy for deleting month backup exceeding 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: From 8220414fd6758d8f883f4bbc708a8cf5dc8d3ec0 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:12:39 +0800 Subject: [PATCH 28/40] =?UTF-8?q?docker=20build=E9=81=8E=E7=A8=8Bm=C3=A0i?= =?UTF-8?q?=E5=8D=B0=EF=BC=8CCI=E8=BC=83=E7=84=A1=E8=B2=A0=E6=93=94?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 100 ++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7204d48..bc0e1c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,13 +55,13 @@ jobs: - curl -SL https://github.com/docker/compose/releases/download/v2.38.1/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup.yml up --detach + - docker compose --file tests/docker-compose-backup.yml up --detach --quiet-pull - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - sleep 10 @@ -70,9 +70,9 @@ jobs: - name: starting message before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -83,13 +83,13 @@ jobs: - name: crontab before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup-minute.yml up --detach + - docker compose --file tests/docker-compose-backup-minute.yml up --detach --quiet-pull - sleep 1m - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` @@ -105,9 +105,9 @@ jobs: - name: backup once when erasing the SCHEDULE variable before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -116,9 +116,9 @@ jobs: - name: exit with SCHEDULE wrong format before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-minute.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-minute.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -132,9 +132,9 @@ jobs: - GPG_PRIVATE_KEY_PATH=tests/gpg-key/ithuan.tw.asc - GPG_PRIVATE_KEY_PASSPHRASE_PATH=tests/gpg-key/ithuan.tw.passphrase install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-encrypt.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-encrypt.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: @@ -151,9 +151,9 @@ jobs: - name: backup strategy for keeping backup in 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -170,9 +170,9 @@ jobs: - name: backup strategy for deleting backup exceeding 72 hours(3 days) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -195,9 +195,9 @@ jobs: - name: backup strategy for keeping day backup in 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -214,9 +214,9 @@ jobs: - name: backup strategy for keeping day backups if they are backuped in different day. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -239,9 +239,9 @@ jobs: - name: backup strategy for deleting backup because of keeping day backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -264,9 +264,9 @@ jobs: - name: backup strategy for deleting day backup exceeding 90 days(3 months) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -289,9 +289,9 @@ jobs: - name: backup strategy for keeping month backup in 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -308,9 +308,9 @@ jobs: - name: backup strategy for keeping month backups if they are backuped in different month. before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -333,9 +333,9 @@ jobs: - name: backup strategy for deleting backup because of keeping month backup only before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: @@ -358,9 +358,9 @@ jobs: - name: backup strategy for deleting month backup exceeding 36 months(3 years) before_install: *before_install_aws install: - - docker compose --file tests/postgres15/docker-compose.yml up --detach - - docker compose --file tests/s3/docker-compose-localstack.yml up --detach - - docker compose --file tests/docker-compose-backup-strategy.yml build + - docker compose --file tests/postgres15/docker-compose.yml up --detach --quiet-pull + - docker compose --file tests/s3/docker-compose-localstack.yml up --detach --quiet-pull + - docker compose --file tests/docker-compose-backup-strategy.yml build --quiet - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" before_script: From 845eab72236d65f4aa83fcff751aa761e5804644 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:49:22 +0800 Subject: [PATCH 29/40] =?UTF-8?q?=E8=99=95=E7=90=86comm=20tab?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 4378f24..db29458 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -59,7 +59,7 @@ mv "${temp_dir}/tmp.list" "${ALL_FILES}" sort -u "${PRESERVE_FILES}" > "${temp_dir}/tmp.list" mv "${temp_dir}/tmp.list" "${PRESERVE_FILES}" comm -1 "${ALL_FILES}" "${PRESERVE_FILES}" \ - | sed 's/^"\(.*\)"$/\1/g' \ + | sed 's/^[[:space:]]"\(.*\)"$/\1/g' \ | while IFS= read -r filename do aws s3api delete-object \ From 352dacb23d17d8629953109f6705b51d3cf37b13 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 10:50:47 +0800 Subject: [PATCH 30/40] =?UTF-8?q?=E4=BF=AEpipe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc0e1c1..a9889e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,7 +122,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup > backup.log) || (echo "The exit code is $?." 2>&1 | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." 2>&1 | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' @@ -189,7 +189,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping day backup in 90 days(3 months) @@ -258,7 +258,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${DAY_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_DAY_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting day backup exceeding 90 days(3 months) @@ -283,7 +283,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for keeping month backup in 36 months(3 years) @@ -352,7 +352,7 @@ jobs: - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${MONTH_BACKUP}" --body test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${NOT_MONTH_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" - name: backup strategy for deleting month backup exceeding 36 months(3 years) @@ -372,6 +372,6 @@ jobs: - echo '# SQL' | gzip > test.sql.gz - awslocal s3api put-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" --body test.sql.gz - docker compose --file tests/docker-compose-backup-strategy.yml run --rm backup - - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' 2>&1 | tee error.log) + - awslocal s3api get-object --bucket "${BUCKET_NAME}" --key "${OLD_BACKUP}" postgres15.sql.gz || (echo 'Not found.' | tee error.log) - cat error.log | grep 'Not found.' - awslocal s3api list-objects --bucket "${BUCKET_NAME}" From 503a902cf585db28d958e4f7505592293ad282d8 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 11:30:43 +0800 Subject: [PATCH 31/40] =?UTF-8?q?=E4=BF=AEpipe=20redirect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a9889e4..d2edb0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,7 +122,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." 2>&1 | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' @@ -140,7 +140,7 @@ jobs: script: - gpg --import --batch ${GPG_PRIVATE_KEY_PATH} - GPG_PUBLIC_KEY=`gpg --armor --export ithuan+multiple-databases-backup@ithuan.tw | base64 --wrap 0` - - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" 2>&1 | tee tests/.env + - echo "GPG_PUBLIC_KEY=${GPG_PUBLIC_KEY}" | tee tests/.env - DATE=`date "+%Y-%m-%d"` - TIME=`date "+%Y%m%dT%H%M"` - docker compose --file tests/docker-compose-backup-encrypt.yml run --rm backup diff --git a/Dockerfile b/Dockerfile index e85d0c6..162772b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,4 +27,4 @@ RUN apt install -y unzip && \ WORKDIR /app/ COPY scripts/ /app/ -CMD bash /app/start.sh +CMD ["bash", "/app/start.sh"] From bb444ded22d4208e985be32444e2069d3f86a6f3 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 11:32:52 +0800 Subject: [PATCH 32/40] =?UTF-8?q?=E6=94=8F=E6=84=9B--rm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2edb0f..72d7fca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,7 +111,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE= backup 2>&1 | tee backup.log + - docker compose --file tests/docker-compose-backup-minute.yml run --rm -e SCHEDULE= backup 2>&1 | tee backup.log - cat backup.log | grep 'There is not SCHEDULE variable' - name: exit with SCHEDULE wrong format before_install: *before_install_aws @@ -122,7 +122,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker-compose-backup-minute.yml run -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run --rm -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' From 69353b4e78c027cf53821c52332464c0f37d9b7e Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 11:41:50 +0800 Subject: [PATCH 33/40] =?UTF-8?q?=E4=BF=AE=E6=AD=A3cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/cleanup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index db29458..f9a3901 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -58,8 +58,8 @@ sort -u "${ALL_FILES}" > "${temp_dir}/tmp.list" mv "${temp_dir}/tmp.list" "${ALL_FILES}" sort -u "${PRESERVE_FILES}" > "${temp_dir}/tmp.list" mv "${temp_dir}/tmp.list" "${PRESERVE_FILES}" -comm -1 "${ALL_FILES}" "${PRESERVE_FILES}" \ - | sed 's/^[[:space:]]"\(.*\)"$/\1/g' \ +comm -23 "${ALL_FILES}" "${PRESERVE_FILES}" \ + | sed 's/^[[:space:]]*"\(.*\)"$/\1/g' \ | while IFS= read -r filename do aws s3api delete-object \ From 05b39370faa4f161c0028bc855c590be038a3e9c Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 11:55:57 +0800 Subject: [PATCH 34/40] =?UTF-8?q?=E4=BF=AEpipe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 72d7fca..42672a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,7 +122,7 @@ jobs: - sleep 10 - awslocal s3api create-bucket --bucket "${BUCKET_NAME}" script: - - (docker compose --file tests/docker-compose-backup-minute.yml run --rm -e SCHEDULE="0 * *" backup 2> backup.log) || (echo "The exit code is $?." | tee error.log) + - (docker compose --file tests/docker-compose-backup-minute.yml run --rm -e SCHEDULE="0 * *" backup > backup.log 2>&1) || (echo "The exit code is $?." | tee error.log) - cat backup.log - cat backup.log | grep 'The SCHEDULE variable should contain five fields exactly.' - cat error.log | grep 'The exit code is 1.' From 7a2614fde774e37289135e1ab8c73e8480eb8902 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 13:49:24 +0800 Subject: [PATCH 35/40] sonarcloud --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 029eedb..14af5c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,14 @@ FROM ubuntu:24.04 +# https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 + ARG DEBIAN_FRONTEND=noninteractive RUN groupadd --system --gid 138 docker RUN apt update && \ - apt install -y docker.io cron gnupg jq - -# https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 -RUN apt install -y curl unzip && \ + apt install --no-install-recommends -y \ + docker.io cron gnupg jq \ + curl unzip && \ + apt-get clean && \ mkdir /aws_build/ && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ From 79f737affa52c245bf4eb9c42451f99becf27605 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 13:52:12 +0800 Subject: [PATCH 36/40] =?UTF-8?q?=E6=95=B4=E7=90=86dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14af5c3..5aa7696 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,8 @@ FROM ubuntu:24.04 - # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst?plain=1 -ARG DEBIAN_FRONTEND=noninteractive RUN groupadd --system --gid 138 docker +ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install --no-install-recommends -y \ docker.io cron gnupg jq \ @@ -15,10 +14,10 @@ RUN apt update && \ /aws_build/aws/install && \ rm -rf /aws_build/ -RUN useradd --uid 1001 nonroot --user-group # --groups docker -RUN usermod -aG docker nonroot -RUN touch /etc/environment -RUN chown nonroot:nonroot /etc/environment +RUN useradd --uid 1001 nonroot --user-group && \ + usermod -aG docker nonroot && \ + touch /etc/environment && \ + chown nonroot:nonroot /etc/environment WORKDIR /app/ COPY scripts/ /app/ From 09384e797e2c166fe8f958b426054e48e4777835 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 13:55:18 +0800 Subject: [PATCH 37/40] sonarcloud --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5aa7696..c82a919 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,9 @@ RUN apt update && \ curl unzip && \ apt-get clean && \ mkdir /aws_build/ && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ + curl --proto "=https" --tlsv1.2 -sSf \ + -L "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" \ + -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ From 4904543319a6201ec8a48aa06e2f7dfe8b82a2c8 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 14:01:18 +0800 Subject: [PATCH 38/40] =?UTF-8?q?=E5=8A=A0sonarcloud=E5=BB=BA=E8=AD=B0?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E5=8F=83=E6=95=B8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c82a919..f98756e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,7 @@ RUN apt update && \ curl unzip && \ apt-get clean && \ mkdir /aws_build/ && \ - curl --proto "=https" --tlsv1.2 -sSf \ - -L "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" \ - -o "/aws_build/awscliv2.zip" && \ + curl -sSf "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ rm -rf /aws_build/ From af92831702b7ae9b8337f1265213c537e63a297d Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 14:11:16 +0800 Subject: [PATCH 39/40] =?UTF-8?q?=E8=99=95=E7=90=86curl=20ca-certificates?= =?UTF-8?q?=E5=95=8F=E9=A1=8C=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f98756e..08e6e66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,10 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install --no-install-recommends -y \ docker.io cron gnupg jq \ - curl unzip && \ + curl ca-certificates unzip && \ apt-get clean && \ mkdir /aws_build/ && \ + update-ca-certificates && \ curl -sSf "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "/aws_build/awscliv2.zip" && \ unzip -q /aws_build/awscliv2.zip -d /aws_build/ && \ /aws_build/aws/install && \ From 1acbafb8eee8ccdc20814ec1ea16547ea68d1279 Mon Sep 17 00:00:00 2001 From: Tia i X570-ACE Date: Fri, 4 Jul 2025 15:48:25 +0800 Subject: [PATCH 40/40] =?UTF-8?q?=E7=B9=BC=E7=BA=8C=E8=A9=A6nonroot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 ++- scripts/crontab.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08e6e66..81224dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,8 @@ RUN apt update && \ RUN useradd --uid 1001 nonroot --user-group && \ usermod -aG docker nonroot && \ touch /etc/environment && \ - chown nonroot:nonroot /etc/environment + chown nonroot:nonroot /etc/environment && \ + chmod u+s /usr/sbin/cron WORKDIR /app/ COPY scripts/ /app/ diff --git a/scripts/crontab.sh b/scripts/crontab.sh index dd67cf5..abaedb4 100644 --- a/scripts/crontab.sh +++ b/scripts/crontab.sh @@ -2,7 +2,7 @@ set -euo pipefail echo "PATH=${PATH} -${SCHEDULE} bash /app/backup.sh >> /var/log/cron.log 2>> /var/log/cron.error.log" \ +${SCHEDULE} bash /app/backup.sh" \ | crontab - cron -f -L 15