From 6a0e4ba0d8e608dba8ab560dc10ddf1066ac7c2f Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Mon, 19 Jul 2021 09:54:13 +1000 Subject: [PATCH 01/30] feat(logging): make elastic stack optional When ENABLE_ELASTIC_STACK == true the containers will be added to docker-compose.yml --- .env | 3 +- compose-files/elastic-stack.yml | 69 +++++++++++++++++++++++++++++++++ compose-files/metricbeat.yml | 1 + scripts/elastic-stack | 7 ++++ 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 compose-files/elastic-stack.yml create mode 100755 scripts/elastic-stack diff --git a/.env b/.env index 69828b9..af43f9d 100644 --- a/.env +++ b/.env @@ -66,8 +66,9 @@ RETHINKDB_VERSION=2.4 POSTGRES_USER=placeos POSTGRES_PASSWORD=development -# Monitor Node variables +# Logging variables +ENABLE_ELASTIC_STACK=false LOGSTASH_HOST=logstash LOGSTASH_PORT=12201 KIBANA_PORT=443 diff --git a/compose-files/elastic-stack.yml b/compose-files/elastic-stack.yml new file mode 100644 index 0000000..0a89848 --- /dev/null +++ b/compose-files/elastic-stack.yml @@ -0,0 +1,69 @@ + + # Logging + + # Aggregates logs and forwards them to Elasticsearch. + logstash: + image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} + restart: always + container_name: logstash + hostname: logstash + expose: + - 12201/udp + - 5044 + networks: + placeos: + volumes: + - ./config/logstash/config:/config + - ./config/logstash/patterns:/opt/logstash/extra_patterns + restart: always + command: logstash -f /config + << : *logging-env + + # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. + validate-logstash-config: + container_name: validate-logstash + image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} + volumes: + - ./config/logstash/config:/config + command: logstash -t -f /config + + # Sends all container json-file logs to logstash + logspout: + image: vincit/logspout-gelf + hostname: $MONITOR_HOSTNAME + container_name: logspout + networks: + placeos: + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} + restart: unless-stopped + << : *logging-env + + kibana: + image: docker.elastic.co/kibana/kibana-oss:${ELASTIC_VERSION:-7.6} + restart: always + container_name: kibana + hostname: kibana + expose: + - 5601 + networks: + placeos: + environment: + - NODE_OPTIONS=--max-old-space-size=200 # fixes memory leak (https://github.com/elastic/kibana/issues/5170) + - HTTPS_METHOD=nohttp + - ELASTICSEARCH_HOSTS="http://${ELASTIC_HOST}:${ELASTIC_PORT}" + - SERVER_BASEPATH="/${PLACE_METRICS_ROUTE}" + - SERVER_REWRITEBASEPATH=true + - SERVER_PUBLICBASEURL="https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" + << : *logging-env + + # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. + # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. + curator: + container_name: curator + image: bobrik/curator:5.7.6 + volumes: + - ./config/curator/action-file.yml:/action-file.yml + - ./config/curator/config.yml:/config.yml + << : *logging-env diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml index e7ac66a..5e8a811 100644 --- a/compose-files/metricbeat.yml +++ b/compose-files/metricbeat.yml @@ -1,3 +1,4 @@ + # Gets metrics from host machine and send to elastic metricbeat: image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} diff --git a/scripts/elastic-stack b/scripts/elastic-stack new file mode 100755 index 0000000..91525ed --- /dev/null +++ b/scripts/elastic-stack @@ -0,0 +1,7 @@ +#! /usr/bin/env bash + +echo "Enabling elastic stack containers..." +if [[ $(grep "container_name: kibana" docker-compose.yml -c) == "0" ]] +then + cat compose-files/elastic-stack.yml >> docker-compose.yml +fi From 9d6d60e103626c9608a143a96be17554259ee590 Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Mon, 16 Aug 2021 16:10:25 +1000 Subject: [PATCH 02/30] feat(logging): add argument to `placeos start` to enable elastic stack if disabled in .env --- placeos | 17 +++++++++++++++++ scripts/start-services | 3 --- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/placeos b/placeos index 02ce0a4..7a0f311 100755 --- a/placeos +++ b/placeos @@ -99,6 +99,7 @@ Arguments: --domain DOMAIN Domain to configure. [default: localhost:8443] --application APP Application to configure. [default: backoffice] -s, --sentry Set-up Sentry. + -k, --kibana Set-up Kibana and Elastic stack. -v, --verbose Write logs to STDOUT in addition to the log file. -h, --help Display this message. EOF @@ -108,6 +109,7 @@ start_environment() { SERVICES=('') hard_reset=false setup_sentry=false + setup_kibana=false email_argument="" password_argument="" domain_argument="" @@ -147,6 +149,9 @@ start_environment() { setup_sentry=true SERVICES+=('-s') ;; + -k | --kibana) + setup_kibana=true + ;; -v | --verbose) VERBOSE="true" ;; @@ -258,6 +263,18 @@ start_environment() { rm -r "${base_path}/.htpasswd-kibana" fi + if [[ $ENABLE_ELASTIC_STACK == "true" ]] || [[ $setup_kibana == "true" ]]; then + run_or_abort \ + "${base_path}/scripts/elastic-stack" \ + "Enabling elastic stack services..." \ + "Failed to enable elastic stack services." + + run_or_abort \ + "${base_path}/scripts/metricbeat" \ + "Enabling metricbeat depending on OS..." \ + "Failed to enable metricbeat or check OS." + fi + run_or_abort \ "${base_path}/scripts/generate-secrets" \ "Generating secrets..." \ diff --git a/scripts/start-services b/scripts/start-services index cb86c49..a6bd2b3 100755 --- a/scripts/start-services +++ b/scripts/start-services @@ -6,9 +6,6 @@ COMPOSE_FILES=('-f ./docker-compose.yml') for arg in "$@"; do case $arg in - -e | --elk) - COMPOSE_FILES+=('-f ./compose-files/elk/docker-compose.yml') - ;; -s | --sentry) COMPOSE_FILES+=('-f ./compose-files/sentry/docker-compose.yml') ;; From eb363c619d9b5e5427af37e66991551a54217e56 Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Mon, 16 Aug 2021 16:16:50 +1000 Subject: [PATCH 03/30] fix(volumes): use ${pwd} instead of . --- compose-files/elastic-stack.yml | 10 +++++----- compose-files/metricbeat.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compose-files/elastic-stack.yml b/compose-files/elastic-stack.yml index 0a89848..d6b84fd 100644 --- a/compose-files/elastic-stack.yml +++ b/compose-files/elastic-stack.yml @@ -13,8 +13,8 @@ networks: placeos: volumes: - - ./config/logstash/config:/config - - ./config/logstash/patterns:/opt/logstash/extra_patterns + - ${pwd}/config/logstash/config:/config + - ${pwd}/config/logstash/patterns:/opt/logstash/extra_patterns restart: always command: logstash -f /config << : *logging-env @@ -24,7 +24,7 @@ container_name: validate-logstash image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} volumes: - - ./config/logstash/config:/config + - ${pwd}/config/logstash/config:/config command: logstash -t -f /config # Sends all container json-file logs to logstash @@ -64,6 +64,6 @@ container_name: curator image: bobrik/curator:5.7.6 volumes: - - ./config/curator/action-file.yml:/action-file.yml - - ./config/curator/config.yml:/config.yml + - ${pwd}/config/curator/action-file.yml:/action-file.yml + - ${pwd}/config/curator/config.yml:/config.yml << : *logging-env diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml index 5e8a811..c098952 100644 --- a/compose-files/metricbeat.yml +++ b/compose-files/metricbeat.yml @@ -12,7 +12,7 @@ - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro - /:/hostfs:ro - /var/run/docker.sock:/var/run/docker.sock:ro - - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml + - ${pwd}/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml cap_add: - SYS_PTRACE - DAC_READ_SEARCH From fc0487e632501adfc47293c3ae99f1264fe2f2b9 Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Fri, 27 Aug 2021 10:19:53 +1000 Subject: [PATCH 04/30] wip(profiles): add docker-compose profiles kibana & metricbeat --- compose-files/elastic-stack.yml | 69 --------------------- compose-files/metricbeat.yml | 21 ------- docker-compose.yml | 102 +++++++++++++++++++++++++++++++- placeos | 25 +++----- 4 files changed, 109 insertions(+), 108 deletions(-) delete mode 100644 compose-files/elastic-stack.yml delete mode 100644 compose-files/metricbeat.yml diff --git a/compose-files/elastic-stack.yml b/compose-files/elastic-stack.yml deleted file mode 100644 index d6b84fd..0000000 --- a/compose-files/elastic-stack.yml +++ /dev/null @@ -1,69 +0,0 @@ - - # Logging - - # Aggregates logs and forwards them to Elasticsearch. - logstash: - image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} - restart: always - container_name: logstash - hostname: logstash - expose: - - 12201/udp - - 5044 - networks: - placeos: - volumes: - - ${pwd}/config/logstash/config:/config - - ${pwd}/config/logstash/patterns:/opt/logstash/extra_patterns - restart: always - command: logstash -f /config - << : *logging-env - - # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. - validate-logstash-config: - container_name: validate-logstash - image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} - volumes: - - ${pwd}/config/logstash/config:/config - command: logstash -t -f /config - - # Sends all container json-file logs to logstash - logspout: - image: vincit/logspout-gelf - hostname: $MONITOR_HOSTNAME - container_name: logspout - networks: - placeos: - volumes: - - /var/run/docker.sock:/var/run/docker.sock - command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} - restart: unless-stopped - << : *logging-env - - kibana: - image: docker.elastic.co/kibana/kibana-oss:${ELASTIC_VERSION:-7.6} - restart: always - container_name: kibana - hostname: kibana - expose: - - 5601 - networks: - placeos: - environment: - - NODE_OPTIONS=--max-old-space-size=200 # fixes memory leak (https://github.com/elastic/kibana/issues/5170) - - HTTPS_METHOD=nohttp - - ELASTICSEARCH_HOSTS="http://${ELASTIC_HOST}:${ELASTIC_PORT}" - - SERVER_BASEPATH="/${PLACE_METRICS_ROUTE}" - - SERVER_REWRITEBASEPATH=true - - SERVER_PUBLICBASEURL="https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" - << : *logging-env - - # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. - # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. - curator: - container_name: curator - image: bobrik/curator:5.7.6 - volumes: - - ${pwd}/config/curator/action-file.yml:/action-file.yml - - ${pwd}/config/curator/config.yml:/config.yml - << : *logging-env diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml deleted file mode 100644 index c098952..0000000 --- a/compose-files/metricbeat.yml +++ /dev/null @@ -1,21 +0,0 @@ - - # Gets metrics from host machine and send to elastic - metricbeat: - image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} - hostname: $MONITOR_HOSTNAME - container_name: metricbeat - user: root - networks: - placeos: - volumes: - - /proc:/hostfs/proc:ro - - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro - - /:/hostfs:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ${pwd}/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml - cap_add: - - SYS_PTRACE - - DAC_READ_SEARCH - command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] - restart: unless-stopped - << : *logging-env diff --git a/docker-compose.yml b/docker-compose.yml index 84d44d1..bd34f03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.7" +version: "3.9" networks: placeos: @@ -439,3 +439,103 @@ services: target: /data/rethinkdb_data environment: TZ: $TZ + + # Aggregates logs and forwards them to Elasticsearch. + logstash: + profiles: + - kibana + image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} + restart: always + container_name: logstash + hostname: logstash + expose: + - 12201/udp + - 5044 + networks: + placeos: + volumes: + - ${PWD}/config/logstash/config:/config + - ${PWD}/config/logstash/patterns:/opt/logstash/extra_patterns + restart: always + command: logstash -f /config + << : *logging-env + + # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. + validate-logstash-config: + profiles: + - kibana + container_name: validate-logstash + image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} + volumes: + - ${pwd}/config/logstash/config:/config + command: logstash -t -f /config + + # Sends all container json-file logs to logstash + logspout: + profiles: + - kibana + image: vincit/logspout-gelf + hostname: $MONITOR_HOSTNAME + container_name: logspout + networks: + placeos: + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} + restart: unless-stopped + << : *logging-env + + kibana: + profiles: + - kibana + image: docker.elastic.co/kibana/kibana-oss:${ELASTIC_VERSION:-7.6} + restart: always + container_name: kibana + hostname: kibana + expose: + - 5601 + networks: + placeos: + environment: + - NODE_OPTIONS=--max-old-space-size=200 # fixes memory leak (https://github.com/elastic/kibana/issues/5170) + - HTTPS_METHOD=nohttp + - ELASTICSEARCH_HOSTS="http://${ELASTIC_HOST}:${ELASTIC_PORT}" + - SERVER_BASEPATH="/${PLACE_METRICS_ROUTE}" + - SERVER_REWRITEBASEPATH=true + - SERVER_PUBLICBASEURL="https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" + << : *logging-env + + # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. + # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. + curator: + profiles: + - kibana + container_name: curator + image: bobrik/curator:5.7.6 + volumes: + - ${pwd}/config/curator/action-file.yml:/action-file.yml + - ${pwd}/config/curator/config.yml:/config.yml + << : *logging-env + + # Gets metrics from host machine and send to elastic + metricbeat: + profiles: + - metricbeat + image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} + hostname: $MONITOR_HOSTNAME + container_name: metricbeat + user: root + networks: + placeos: + volumes: + - /proc:/hostfs/proc:ro + - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro + - /:/hostfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ${PWD}/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml + cap_add: + - SYS_PTRACE + - DAC_READ_SEARCH + command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] + restart: unless-stopped + << : *logging-env diff --git a/placeos b/placeos index 7a0f311..8668f87 100755 --- a/placeos +++ b/placeos @@ -109,7 +109,8 @@ start_environment() { SERVICES=('') hard_reset=false setup_sentry=false - setup_kibana=false + enable_kibana=false + enable_metricbeat=false email_argument="" password_argument="" domain_argument="" @@ -150,7 +151,8 @@ start_environment() { SERVICES+=('-s') ;; -k | --kibana) - setup_kibana=true + enable_kibana=true + SERVICES+=('-k') ;; -v | --verbose) VERBOSE="true" @@ -248,11 +250,6 @@ start_environment() { echo "" done - # run_or_abort \ - # "${base_path}/scripts/metricbeat" \ - # "Checking Host OS..." \ - # "Error occurred while checking Host OS." - # Write the email so as to not prompt the user again. echo "PLACE_EMAIL=${PLACE_EMAIL}" >"${EMAIL_ENV}" # TODO: use init check instead of writing the password. @@ -263,16 +260,10 @@ start_environment() { rm -r "${base_path}/.htpasswd-kibana" fi - if [[ $ENABLE_ELASTIC_STACK == "true" ]] || [[ $setup_kibana == "true" ]]; then - run_or_abort \ - "${base_path}/scripts/elastic-stack" \ - "Enabling elastic stack services..." \ - "Failed to enable elastic stack services." - - run_or_abort \ - "${base_path}/scripts/metricbeat" \ - "Enabling metricbeat depending on OS..." \ - "Failed to enable metricbeat or check OS." + if [[ $ENABLE_ELASTIC_STACK == "true" ]] || [[ $enable_kibana == "true" ]]; then + if [[ $(uname) == "Linux" ]]; then + enable_metricbeat="true" + fi fi run_or_abort \ From 1c4b2a77a1e5830c619d2eeddcc2f6cedc57c23e Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Fri, 27 Aug 2021 10:30:10 +1000 Subject: [PATCH 05/30] fix(placeos): run docker-compose from run_or_abort --- docker-compose.yml | 6 +++--- placeos | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bd34f03..1e96a7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -467,7 +467,7 @@ services: container_name: validate-logstash image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} volumes: - - ${pwd}/config/logstash/config:/config + - ${PWD}/config/logstash/config:/config command: logstash -t -f /config # Sends all container json-file logs to logstash @@ -513,8 +513,8 @@ services: container_name: curator image: bobrik/curator:5.7.6 volumes: - - ${pwd}/config/curator/action-file.yml:/action-file.yml - - ${pwd}/config/curator/config.yml:/config.yml + - ${PWD}/config/curator/action-file.yml:/action-file.yml + - ${PWD}/config/curator/config.yml:/config.yml << : *logging-env # Gets metrics from host machine and send to elastic diff --git a/placeos b/placeos index 8668f87..7d52e8d 100755 --- a/placeos +++ b/placeos @@ -106,7 +106,7 @@ EOF } start_environment() { - SERVICES=('') + PROFILES="" hard_reset=false setup_sentry=false enable_kibana=false @@ -152,7 +152,7 @@ start_environment() { ;; -k | --kibana) enable_kibana=true - SERVICES+=('-k') + PROFILES+=" --profile kibana" ;; -v | --verbose) VERBOSE="true" @@ -262,7 +262,7 @@ start_environment() { if [[ $ENABLE_ELASTIC_STACK == "true" ]] || [[ $enable_kibana == "true" ]]; then if [[ $(uname) == "Linux" ]]; then - enable_metricbeat="true" + PROFILES+=" --profile metricbeat" fi fi @@ -272,7 +272,12 @@ start_environment() { "Failed to generate secrets." run_or_abort \ - "${base_path}/scripts/start-services ${SERVICES[@]}" \ + "docker-compose ${PROFILES[@]} pull -q" \ + "Pulling service images..." \ + "Failed to pull images." + + run_or_abort \ + "docker-compose ${PROFILES[@]} up -d" \ "Bringing up services..." \ "Failed to start services." From 0a475506a770c21dccfbb973dcf8ad05165df1ca Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Fri, 27 Aug 2021 11:22:44 +1000 Subject: [PATCH 06/30] chore(placeos): adjust output --- placeos | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/placeos b/placeos index 7d52e8d..0d38449 100755 --- a/placeos +++ b/placeos @@ -110,7 +110,6 @@ start_environment() { hard_reset=false setup_sentry=false enable_kibana=false - enable_metricbeat=false email_argument="" password_argument="" domain_argument="" @@ -272,12 +271,12 @@ start_environment() { "Failed to generate secrets." run_or_abort \ - "docker-compose ${PROFILES[@]} pull -q" \ - "Pulling service images..." \ + "docker-compose ${PROFILES} pull -q" \ + "Pulling images..." \ "Failed to pull images." run_or_abort \ - "docker-compose ${PROFILES[@]} up -d" \ + "docker-compose ${PROFILES} up -d" \ "Bringing up services..." \ "Failed to start services." From c5f271c41883436b954b063d036c6519ba2737f5 Mon Sep 17 00:00:00 2001 From: Viv Briffa Date: Fri, 27 Aug 2021 11:25:37 +1000 Subject: [PATCH 07/30] chore(env): rename ENABLE_ELASTIC_STACK to ENABLE_KIBANA --- .env | 2 +- placeos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index af43f9d..439a4d7 100644 --- a/.env +++ b/.env @@ -68,7 +68,7 @@ POSTGRES_PASSWORD=development # Logging variables -ENABLE_ELASTIC_STACK=false +ENABLE_KIBANA=true LOGSTASH_HOST=logstash LOGSTASH_PORT=12201 KIBANA_PORT=443 diff --git a/placeos b/placeos index 0d38449..05a212f 100755 --- a/placeos +++ b/placeos @@ -259,7 +259,7 @@ start_environment() { rm -r "${base_path}/.htpasswd-kibana" fi - if [[ $ENABLE_ELASTIC_STACK == "true" ]] || [[ $enable_kibana == "true" ]]; then + if [[ $ENABLE_KIBANA == "true" ]] || [[ $enable_kibana == "true" ]]; then if [[ $(uname) == "Linux" ]]; then PROFILES+=" --profile metricbeat" fi From 221ba8df9c20a9b62131bbcc79bc9af506589dac Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 12:43:05 +1000 Subject: [PATCH 08/30] chore: remove redundant scripts --- scripts/elastic-stack | 7 ------- scripts/metricbeat | 10 ---------- scripts/start-services | 17 ----------------- 3 files changed, 34 deletions(-) delete mode 100755 scripts/elastic-stack delete mode 100755 scripts/metricbeat delete mode 100755 scripts/start-services diff --git a/scripts/elastic-stack b/scripts/elastic-stack deleted file mode 100755 index 91525ed..0000000 --- a/scripts/elastic-stack +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash - -echo "Enabling elastic stack containers..." -if [[ $(grep "container_name: kibana" docker-compose.yml -c) == "0" ]] -then - cat compose-files/elastic-stack.yml >> docker-compose.yml -fi diff --git a/scripts/metricbeat b/scripts/metricbeat deleted file mode 100755 index c2250be..0000000 --- a/scripts/metricbeat +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/env bash - -if [[ $(uname) == "Linux" ]] -then - echo "Host is Linux, enabling metricbeat" - if [[ $(grep "metricbeat:" docker-compose.yml -c) == "0" ]] - then - cat compose-files/metricbeat.yml >> docker-compose.yml - fi -fi diff --git a/scripts/start-services b/scripts/start-services deleted file mode 100755 index a6bd2b3..0000000 --- a/scripts/start-services +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -COMPOSE_FILES=('-f ./docker-compose.yml') - -for arg in "$@"; do - case $arg in - -s | --sentry) - COMPOSE_FILES+=('-f ./compose-files/sentry/docker-compose.yml') - ;; - esac -done - -docker-compose ${COMPOSE_FILES[@]} pull -q - -docker-compose ${COMPOSE_FILES[@]} up -d From baf421ed04057993e53e6fc6aa27423eb876f9bf Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:11:28 +1000 Subject: [PATCH 09/30] refactor(placeos): add kibana profiles in one place --- placeos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/placeos b/placeos index 05a212f..c745b3e 100755 --- a/placeos +++ b/placeos @@ -151,7 +151,6 @@ start_environment() { ;; -k | --kibana) enable_kibana=true - PROFILES+=" --profile kibana" ;; -v | --verbose) VERBOSE="true" @@ -260,6 +259,7 @@ start_environment() { fi if [[ $ENABLE_KIBANA == "true" ]] || [[ $enable_kibana == "true" ]]; then + PROFILES+=" --profile kibana" if [[ $(uname) == "Linux" ]]; then PROFILES+=" --profile metricbeat" fi @@ -344,7 +344,7 @@ stop_environment() { done run_or_abort \ - "docker-compose down" \ + "docker-compose down --remove-orphans" \ "Tearing down PlaceOS" \ "Failed to teardown PlaceOS" } From 67b61672271c8bc84f16fbc80248ced42d9ee5cf Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:26:31 +1000 Subject: [PATCH 10/30] fix(docker-compose.yml): remove duplicate `restart` key --- docker-compose.yml | 48 +++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e96a7b..aa3aed5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,10 +114,10 @@ services: hostname: auth networks: placeos: + <<: *logging-env healthcheck: # TODO: Remove after placeos-1.2109.0 test: wget --no-verbose -q --spider http://0.0.0.0:8080/auth/authority?health=true - <<: *logging-env depends_on: - redis - rethink @@ -328,11 +328,11 @@ services: restart: always container_name: elastic hostname: elastic - healthcheck: - test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health networks: placeos: <<: *logging-env + healthcheck: + test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health volumes: - type: volume source: elastic-data @@ -348,23 +348,23 @@ services: restart: always container_name: etcd hostname: etcd - healthcheck: - test: etcdctl endpoint health networks: placeos: <<: *logging-env + healthcheck: + test: etcdctl endpoint health environment: ALLOW_NONE_AUTHENTICATION: "yes" TZ: $TZ influxdb: image: quay.io/influxdb/influxdb:${INFLUXDB_IMAGE_TAG:-v2.0.7} - container_name: influx restart: always + container_name: influx + hostname: influx networks: placeos: <<: *logging-env - hostname: influx healthcheck: test: influx bucket list volumes: @@ -410,11 +410,11 @@ services: restart: always container_name: redis hostname: redis - healthcheck: - test: redis-cli ping networks: placeos: <<: *logging-env + healthcheck: + test: redis-cli ping volumes: - type: volume source: redis-data @@ -427,12 +427,12 @@ services: restart: always container_name: rethink hostname: rethink - healthcheck: - # Check if the DB's port is open - test: "bash -c ': &>/dev/null /dev/null Date: Sat, 28 Aug 2021 13:29:37 +1000 Subject: [PATCH 11/30] build(docker-compose): use alpine logstash --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index aa3aed5..49277b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -444,7 +444,7 @@ services: logstash: profiles: - kibana - image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} + image: blacktop/logstash:${ELASTIC_VERSION:-7.6} restart: always container_name: logstash hostname: logstash From 20b92bfafef0bcdc306a94b21a80738bed68e9cb Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:30:07 +1000 Subject: [PATCH 12/30] build(docker-compose): use open distro for elasticsearch --- .env | 1 + docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 439a4d7..9cf71fa 100644 --- a/.env +++ b/.env @@ -41,6 +41,7 @@ PLACE_SERVER_SECRET=development ELASTIC_HOST=elastic ELASTIC_PORT=9200 ELASTIC_VERSION=7.6.2 +ELASTIC_OD_VERSION=-1.13.2 ETCD_HOST=etcd ETCD_PORT=2379 diff --git a/docker-compose.yml b/docker-compose.yml index 49277b8..76381a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -324,7 +324,7 @@ services: # Resources elastic: - image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.6} + image: amazon/opendistro-for-elasticsearch:${ELASTIC_OD_VERSION:-1.13.2} restart: always container_name: elastic hostname: elastic @@ -484,7 +484,7 @@ services: kibana: profiles: - kibana - image: docker.elastic.co/kibana/kibana-oss:${ELASTIC_VERSION:-7.6} + image: amazon/opendistro-for-elasticsearch-kibana:${ELASTIC_OD_VERSION:-1.13.2} restart: always container_name: kibana hostname: kibana From 088b65146aef2c57d08433f0e0f98c2fd15f7f7d Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:35:19 +1000 Subject: [PATCH 13/30] build(docker-compose): set tag on logspout-gelf --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 76381a0..d265c98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -470,7 +470,7 @@ services: logspout: profiles: - kibana - image: vincit/logspout-gelf + image: vincit/logspout-gelf:3.2.6-alpine restart: unless-stopped hostname: $MONITOR_HOSTNAME container_name: logspout From 37748d7f18a78548f44e0268c11f94293634ff96 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:45:01 +1000 Subject: [PATCH 14/30] refactor(docker-compose): use dockerhub hosted where possible This saves extra allowlist entries --- docker-compose.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d265c98..61bb6c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -189,7 +189,7 @@ services: <<: *deployment-env source: - image: docker.io/placeos/source:${PLACE_SOURCE_TAG:-latest} + image: placeos/source:${PLACE_SOURCE_TAG:-latest} restart: always container_name: source networks: @@ -234,6 +234,7 @@ services: postgres: # Database used by Staff API image: postgres:${POSTGRES_VERSION:-13-alpine} container_name: postgres + hostname: postgres restart: unless-stopped networks: placeos: @@ -248,6 +249,7 @@ services: staff: # Staff API image: placeos/staff-api:${PLACE_STAFF_API_TAG:-latest} container_name: staff + hostname: staff restart: unless-stopped networks: placeos: @@ -358,7 +360,7 @@ services: TZ: $TZ influxdb: - image: quay.io/influxdb/influxdb:${INFLUXDB_IMAGE_TAG:-v2.0.7} + image: influxdb:${INFLUXDB_IMAGE_TAG:-v2.0.8}-alpine restart: always container_name: influx hostname: influx @@ -458,19 +460,19 @@ services: # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. validate-logstash-config: + image: blacktop/logstash:${ELASTIC_VERSION:-7.6} profiles: - kibana container_name: validate-logstash - image: docker.elastic.co/logstash/logstash-oss:${ELASTIC_VERSION:-7.6} volumes: - ${PWD}/config/logstash/config:/config command: logstash -t -f /config # Sends all container json-file logs to logstash logspout: + image: vincit/logspout-gelf:3.2.6-alpine profiles: - kibana - image: vincit/logspout-gelf:3.2.6-alpine restart: unless-stopped hostname: $MONITOR_HOSTNAME container_name: logspout @@ -502,9 +504,9 @@ services: # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. curator: + image: bobrik/curator:5.7.6 profiles: - kibana - image: bobrik/curator:5.7.6 container_name: curator hostname: curator << : *logging-env @@ -514,9 +516,9 @@ services: # Gets metrics from host machine and send to elastic metricbeat: + image: elastic/metricbeat:${ELASTIC_VERSION:-7.6} profiles: - metricbeat - image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} restart: unless-stopped hostname: $MONITOR_HOSTNAME container_name: metricbeat From d8c3ceb7a79a1031f1e704e19d64fd18336c0be5 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:49:57 +1000 Subject: [PATCH 15/30] build(docker-compose): dep between logstash validator and logstash --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 61bb6c4..6af54f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -453,6 +453,8 @@ services: networks: placeos: << : *logging-env + depends_on: + - validate-logstash-config volumes: - ${PWD}/config/logstash/config:/config - ${PWD}/config/logstash/patterns:/opt/logstash/extra_patterns @@ -461,6 +463,7 @@ services: # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. validate-logstash-config: image: blacktop/logstash:${ELASTIC_VERSION:-7.6} + restart: "no" profiles: - kibana container_name: validate-logstash From 6d12df9c970154f1171b83daaea66b276a1ea0f5 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 13:53:15 +1000 Subject: [PATCH 16/30] fix(docker-compose): correct influx tag --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6af54f4..da6ca8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -360,7 +360,7 @@ services: TZ: $TZ influxdb: - image: influxdb:${INFLUXDB_IMAGE_TAG:-v2.0.8}-alpine + image: influxdb:${INFLUXDB_IMAGE_TAG:-2.0.8}-alpine restart: always container_name: influx hostname: influx From 4b0523042cd0fbccb812b8c07d599f4c15b26af1 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sat, 28 Aug 2021 14:01:51 +1000 Subject: [PATCH 17/30] fix(.env): typo for opendistro elastic version --- .env | 2 +- docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 9cf71fa..e29cfdd 100644 --- a/.env +++ b/.env @@ -41,7 +41,7 @@ PLACE_SERVER_SECRET=development ELASTIC_HOST=elastic ELASTIC_PORT=9200 ELASTIC_VERSION=7.6.2 -ELASTIC_OD_VERSION=-1.13.2 +ELASTIC_OD_VERSION=1.13.2 ETCD_HOST=etcd ETCD_PORT=2379 diff --git a/docker-compose.yml b/docker-compose.yml index da6ca8c..ff4d9e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -444,9 +444,9 @@ services: # Aggregates logs and forwards them to Elasticsearch. logstash: + image: blacktop/logstash:${ELASTIC_VERSION:-7.6} profiles: - kibana - image: blacktop/logstash:${ELASTIC_VERSION:-7.6} restart: always container_name: logstash hostname: logstash @@ -463,9 +463,9 @@ services: # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. validate-logstash-config: image: blacktop/logstash:${ELASTIC_VERSION:-7.6} - restart: "no" profiles: - kibana + restart: "no" container_name: validate-logstash volumes: - ${PWD}/config/logstash/config:/config @@ -487,9 +487,9 @@ services: command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} kibana: + image: amazon/opendistro-for-elasticsearch-kibana:${ELASTIC_OD_VERSION:-1.13.2} profiles: - kibana - image: amazon/opendistro-for-elasticsearch-kibana:${ELASTIC_OD_VERSION:-1.13.2} restart: always container_name: kibana hostname: kibana From b6c0f308ce765384678e55529fa470a8df05433a Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Sun, 29 Aug 2021 00:06:44 +1000 Subject: [PATCH 18/30] refactor(placeos): DRY argument handling --- placeos | 51 +++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/placeos b/placeos index c745b3e..e927af4 100755 --- a/placeos +++ b/placeos @@ -39,6 +39,17 @@ logfile="${base_path}/.logs/$(date +"%Y%m%d%H%M").log" COMPOSE_PROJECT_NAME=placeos +# Helpers +################################################################################################### + +unknown_argument() { + unknown_arg="${1}" + usage="${2}" + [ -n "${unknown_arg}" ] && echo -e "░░░ ${red}Unknown option:${reset} ${unknown_arg}" + eval "${usage}" + exit 1 +} + abort() { echo -e "░░░ ${red}${1}${reset}" echo "░░░ Logs can be found in ${logfile}" @@ -66,6 +77,9 @@ run_or_abort() { fi } +# Start +################################################################################################### + hard_reset() { # TODO: drop influxdb tables # TODO: clear redis @@ -160,12 +174,7 @@ start_environment() { exit 0 ;; *) - if [ -n "${command}" ]; then - echo -e "░░░ ${red}Unknown option:${reset} ${command}" - else - start_environment__usage - exit 1 - fi + unknown_argument "${command}" "start_environment__usage" ;; esac done @@ -308,6 +317,9 @@ start_environment() { echo "░░░ $PLACE_EMAIL:$PLACE_PASSWORD" } +# Stop +################################################################################################### + stop_environment__usage() { cat < Date: Sun, 29 Aug 2021 00:07:02 +1000 Subject: [PATCH 19/30] style: update chars --- scripts/init-influxdb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/init-influxdb b/scripts/init-influxdb index 312dfe3..34a8224 100755 --- a/scripts/init-influxdb +++ b/scripts/init-influxdb @@ -19,7 +19,7 @@ org="${INFLUX_ORG:-PlaceOS}" bucket="${INFLUX_BUCKET:-place}" retention="${INFLUX_RETENTION:-4w}" -echo "=== Initialising InfluxDB API" +echo "░░░ Initialising InfluxDB API" # Wait for the service to be available wait=0 max_wait=60 @@ -27,7 +27,7 @@ until [ $wait -eq $max_wait ] || docker exec $instance influx ping > /dev/null; sleep $(( wait++ )) done if [ $wait -eq $max_wait ]; then - echo "=== Timeout waiting for InfluxDB to be ready" + echo "░░░ Timeout waiting for InfluxDB to be ready" exit 1 fi From 67e176800fe3e865998bb87cd5b92abf538d843d Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Thu, 2 Sep 2021 12:40:52 +1000 Subject: [PATCH 20/30] build: remove MONITOR_HOSTNAME --- docker-compose.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ff4d9e1..f2df715 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ x-jwt-public: &jwt-public .env.public_key x-jwt-secret: &jwt-secret .env.secret_key x-elastic-client-env: &elastic-client-env + ELASTIC_HOST: ${ELASTIC_HOST:-elastic} + ELASTIC_PORT: ${ELASTIC_PORT:-9200} ES_HOST: ${ELASTIC_HOST:-elastic} ES_PORT: ${ELASTIC_PORT:-9200} @@ -467,6 +469,8 @@ services: - kibana restart: "no" container_name: validate-logstash + networks: + placeos: volumes: - ${PWD}/config/logstash/config:/config command: logstash -t -f /config @@ -477,8 +481,8 @@ services: profiles: - kibana restart: unless-stopped - hostname: $MONITOR_HOSTNAME container_name: logspout + hostname: logspout networks: placeos: << : *logging-env @@ -497,12 +501,13 @@ services: placeos: << : *logging-env environment: - - NODE_OPTIONS=--max-old-space-size=200 # fixes memory leak (https://github.com/elastic/kibana/issues/5170) - - HTTPS_METHOD=nohttp - - ELASTICSEARCH_HOSTS="http://${ELASTIC_HOST}:${ELASTIC_PORT}" - - SERVER_BASEPATH="/${PLACE_METRICS_ROUTE}" - - SERVER_REWRITEBASEPATH=true - - SERVER_PUBLICBASEURL="https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" + <<: *elastic-client-env + NODE_OPTIONS: "--max-old-space-size=200" # fixes memory leak (https://github.com/elastic/kibana/issues/5170) + HTTPS_METHOD: "nohttp" + ELASTICSEARCH_HOSTS: "http://${ELASTIC_HOST}:${ELASTIC_PORT}" + SERVER_BASEPATH: "/${PLACE_METRICS_ROUTE}" + SERVER_REWRITEBASEPATH: "true" + SERVER_PUBLICBASEURL: "https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. @@ -512,6 +517,8 @@ services: - kibana container_name: curator hostname: curator + networks: + placeos: << : *logging-env volumes: - ${PWD}/config/curator/action-file.yml:/action-file.yml @@ -523,13 +530,14 @@ services: profiles: - metricbeat restart: unless-stopped - hostname: $MONITOR_HOSTNAME container_name: metricbeat hostname: metricbeat user: root networks: placeos: << : *logging-env + environment: + <<: *elastic-client-env volumes: - /proc:/hostfs/proc:ro - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro From ef8b416baf9ca7f2ce0003b6a9346debc923f2e7 Mon Sep 17 00:00:00 2001 From: Kim Burgess Date: Tue, 7 Sep 2021 13:59:28 +1000 Subject: [PATCH 21/30] placeos-1.2109.0 (#72) --- .env | 6 ++--- .github/workflows/ci.yml | 1 + docker-compose.yml | 51 +++++++++++++++++++--------------------- placeos | 22 ++++++++++------- 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/.env b/.env index e29cfdd..b0f96a0 100644 --- a/.env +++ b/.env @@ -6,13 +6,13 @@ SG_ENV=development # PlaceOS Service Image Tags -PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2108.4} +PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2109.0} PLACE_AUTH_TAG=${PLACEOS_TAG} PLACE_CORE_TAG=${PLACEOS_TAG} PLACE_DISPATCH_TAG=${PLACEOS_TAG} PLACE_EDGE_TAG=${PLACEOS_TAG} -PLACE_FRONTENDS_TAG=${PLACEOS_TAG} +PLACE_FRONTEND_LOADER_TAG=${PLACEOS_TAG} PLACE_INIT_TAG=${PLACEOS_TAG} PLACE_REST_API_TAG=${PLACEOS_TAG} PLACE_RUBBER_SOUL_TAG=${PLACEOS_TAG} @@ -24,7 +24,7 @@ PLACE_NGINX_TAG=${PLACEOS_TAG} # PlaceOS variables PLACE_AUTH_HOST=auth:8080 -PLACE_LOADER_URI=http://frontends:3000 +PLACE_LOADER_URI=http://frontend-loader:3000 RUBBER_SOUL_URI=http://rubber-soul:3000 # PLACE_EMAIL=support@place.tech diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 760b9b7..6b04bb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: - placeos-1.2108.2 - placeos-1.2108.3 - placeos-1.2108.4 + - placeos-1.2109.0 include: - version: nightly stable: false diff --git a/docker-compose.yml b/docker-compose.yml index f2df715..bf22505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,7 +51,7 @@ x-influxdb-client-env: &influxdb-client-env INFLUX_ORG: ${INFLUX_ORG:-PlaceOS} x-place-loader-client-env: &place-loader-client-env - PLACE_LOADER_URI: ${PLACE_LOADER_URI:-http://frontends:3000} + PLACE_LOADER_URI: ${PLACE_LOADER_URI:-http://frontend-loader:3000} x-redis-client-env: &redis-client-env REDIS_URL: ${REDIS_URL:-redis://redis:6379} @@ -71,7 +71,7 @@ x-smtp-client-env: &smtp-client-env SMTP_PASS: ${SMTP_PASS:-} # password if required SMTP_SECURE: ${SMTP_SECURE:-} # blank for unsecure, `SMTPS` for TLS, `STARTTLS` for negotiating TLS on unsecure connection -x-logging-env: &logging-env +x-logging: &std-logging logging: driver: json-file options: @@ -87,7 +87,7 @@ services: hostname: api networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - auth - elastic @@ -116,10 +116,7 @@ services: hostname: auth networks: placeos: - <<: *logging-env - healthcheck: - # TODO: Remove after placeos-1.2109.0 - test: wget --no-verbose -q --spider http://0.0.0.0:8080/auth/authority?health=true + <<: *std-logging depends_on: - redis - rethink @@ -138,7 +135,7 @@ services: hostname: core networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - etcd - redis @@ -163,21 +160,21 @@ services: hostname: edge networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - core environment: <<: *edge-env <<: *deployment-env - frontends: # Frontend deployment service - image: placeos/frontends:${PLACE_FRONTENDS_TAG:-latest} + frontend-loader: # Frontend deployment service + image: placeos/${PLACE_FRONTEND_LOADER_IMAGE:-frontend-loader}:${PLACE_FRONTEND_LOADER_TAG:-latest} restart: always - container_name: frontends - hostname: frontends + container_name: frontend-loader + hostname: frontend-loader networks: placeos: - <<: *logging-env + <<: *std-logging volumes: - type: volume source: www @@ -196,7 +193,7 @@ services: container_name: source networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - influxdb - redis @@ -218,7 +215,7 @@ services: hostname: triggers networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - core - etcd @@ -240,7 +237,7 @@ services: restart: unless-stopped networks: placeos: - <<: *logging-env + <<: *std-logging volumes: - postgres-data:/var/lib/postgresql/data environment: @@ -255,7 +252,7 @@ services: restart: unless-stopped networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - api - postgres @@ -277,7 +274,7 @@ services: restart: on-failure networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - rethink - rubber-soul @@ -303,7 +300,7 @@ services: hostname: rubber-soul networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - elastic - rethink @@ -321,7 +318,7 @@ services: hostname: dispatch networks: placeos: - <<: *logging-env + <<: *std-logging environment: SERVER_SECRET: ${PLACE_SERVER_SECRET:-development} @@ -334,9 +331,9 @@ services: hostname: elastic networks: placeos: - <<: *logging-env healthcheck: test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health + <<: *std-logging volumes: - type: volume source: elastic-data @@ -354,9 +351,9 @@ services: hostname: etcd networks: placeos: - <<: *logging-env healthcheck: test: etcdctl endpoint health + <<: *std-logging environment: ALLOW_NONE_AUTHENTICATION: "yes" TZ: $TZ @@ -368,7 +365,7 @@ services: hostname: influx networks: placeos: - <<: *logging-env + <<: *std-logging healthcheck: test: influx bucket list volumes: @@ -387,7 +384,7 @@ services: - 8443:443 networks: placeos: - <<: *logging-env + <<: *std-logging depends_on: - api - auth @@ -416,9 +413,9 @@ services: hostname: redis networks: placeos: - <<: *logging-env healthcheck: test: redis-cli ping + <<: *std-logging volumes: - type: volume source: redis-data @@ -433,10 +430,10 @@ services: hostname: rethink networks: placeos: - <<: *logging-env healthcheck: # Check if the DB's port is open test: "bash -c ': &>/dev/null "${ext_env}" - - # Import .env set -o allexport - . "${base_path}/.env" - set +o allexport - # Overlay the original config and cleanup - . "${ext_env}" - rm "${ext_env}" - unset ext_env + # Load contents of .env + . "${base_path}/.env" + # Conditional config for backwards compat if [[ $PLACEOS_TAG < "placeos-1.2108.2" ]]; then PLACE_STAFF_API_TAG="nightly" PLACE_NGINX_TAG="staff-api" fi + if [[ $PLACEOS_TAG < "placeos-1.2109.0" ]]; then + PLACE_FRONTEND_LOADER_IMAGE="frontends" + fi + + # Overlay the existing environment + . "${ext_env}" + + # Cleanup + set +o allexport + rm "${ext_env}" + unset ext_env echo "░░░ Starting PlaceOS <${PLACEOS_TAG}>" [ $VERBOSE == "false" ] && echo "░░░ For detailed logging, run \`tail -f ${logfile}\`" From 18f69287a098d8a021d8710c2c2524e00a5469a9 Mon Sep 17 00:00:00 2001 From: Kim Burgess Date: Fri, 10 Sep 2021 15:38:20 +1000 Subject: [PATCH 22/30] fix: prevent compat overlays from applying to nightly --- .env | 2 +- placeos | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.env b/.env index b0f96a0..dd9a461 100644 --- a/.env +++ b/.env @@ -6,7 +6,7 @@ SG_ENV=development # PlaceOS Service Image Tags -PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2109.0} +PLACEOS_TAG=nightly PLACE_AUTH_TAG=${PLACEOS_TAG} PLACE_CORE_TAG=${PLACEOS_TAG} diff --git a/placeos b/placeos index 16b2353..571de9e 100755 --- a/placeos +++ b/placeos @@ -189,18 +189,24 @@ start_environment() { # Load contents of .env . "${base_path}/.env" - # Conditional config for backwards compat - if [[ $PLACEOS_TAG < "placeos-1.2108.2" ]]; then - PLACE_STAFF_API_TAG="nightly" - PLACE_NGINX_TAG="staff-api" - fi - if [[ $PLACEOS_TAG < "placeos-1.2109.0" ]]; then - PLACE_FRONTEND_LOADER_IMAGE="frontends" - fi # Overlay the existing environment . "${ext_env}" + # Conditional config for backwards compat + if [[ $PLACEOS_TAG =~ ^placeos-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + + if [[ $PLACEOS_TAG < "placeos-1.2108.2" ]]; then + PLACE_STAFF_API_TAG="nightly" + PLACE_NGINX_TAG="staff-api" + fi + + if [[ $PLACEOS_TAG < "placeos-1.2109.0" ]]; then + PLACE_FRONTEND_LOADER_IMAGE="frontends" + fi + + fi + # Cleanup set +o allexport rm "${ext_env}" From eaf6ffdb310ac3644cda76805ffd912a37c217d7 Mon Sep 17 00:00:00 2001 From: Kim Burgess Date: Fri, 10 Sep 2021 16:06:55 +1000 Subject: [PATCH 23/30] fix: remove unintended .env change --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index dd9a461..b0f96a0 100644 --- a/.env +++ b/.env @@ -6,7 +6,7 @@ SG_ENV=development # PlaceOS Service Image Tags -PLACEOS_TAG=nightly +PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2109.0} PLACE_AUTH_TAG=${PLACEOS_TAG} PLACE_CORE_TAG=${PLACEOS_TAG} From 09215866e251bf087d2f8d94a037584df999501f Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Fri, 10 Sep 2021 19:10:11 +1000 Subject: [PATCH 24/30] build: update to 1.2109.1 --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b0f96a0..638fa3e 100644 --- a/.env +++ b/.env @@ -6,7 +6,7 @@ SG_ENV=development # PlaceOS Service Image Tags -PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2109.0} +PLACEOS_TAG=${PLACEOS_TAG:-placeos-1.2109.1} PLACE_AUTH_TAG=${PLACEOS_TAG} PLACE_CORE_TAG=${PLACEOS_TAG} From fbf6d7f2ab1950d8d6e641a030493759af067a4f Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Fri, 10 Sep 2021 22:07:26 +1000 Subject: [PATCH 25/30] ci: add 1.2109.1 Also removes some intermediary patch versions < `1.2109.x` --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b04bb1..9e6deee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push jobs: test: - name: "Test Install - os: ${{ matrix.os }}, version: ${{ matrix.version }}, stable: ${{ matrix.stable }}" + name: "os: ${{ matrix.os }}, version: ${{ matrix.version }}, stable: ${{ matrix.stable }}" runs-on: ${{ matrix.os }} continue-on-error: ${{ !matrix.stable }} strategy: @@ -16,11 +16,9 @@ jobs: - "" # Test empty tag - placeos-1.2105.3 - placeos-1.2107.4 - - placeos-1.2108.1 - - placeos-1.2108.2 - - placeos-1.2108.3 - placeos-1.2108.4 - placeos-1.2109.0 + - placeos-1.2109.1 include: - version: nightly stable: false From 6b96708ae510f428beaba0515c480211ddf1be69 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Fri, 10 Sep 2021 22:13:14 +1000 Subject: [PATCH 26/30] ci: reorder values in test name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e6deee..f94b64f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push jobs: test: - name: "os: ${{ matrix.os }}, version: ${{ matrix.version }}, stable: ${{ matrix.stable }}" + name: "version: ${{ matrix.version }}, os: ${{ matrix.os }}, stable: ${{ matrix.stable }}" runs-on: ${{ matrix.os }} continue-on-error: ${{ !matrix.stable }} strategy: @@ -35,7 +35,7 @@ jobs: PLACE_EMAIL: robot@place.tech PLACE_PASSWORD: development - name: Test initialization, with unset PLACEOS_TAG - if: ${{ matrix.version == '' }} + if: ${{ matrix.version == '' }} # Testing an empty tag run: | docker-compose down --volumes &> /dev/null rm .env.* From 5952cee0167e3dac0d3a5ac394fb5c25d1d3b43b Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Fri, 10 Sep 2021 22:15:44 +1000 Subject: [PATCH 27/30] ci: render empty version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f94b64f..1e3e978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push jobs: test: - name: "version: ${{ matrix.version }}, os: ${{ matrix.os }}, stable: ${{ matrix.stable }}" + name: "version: ${{ matrix.version != '' && matrix.version || 'empty'}}, os: ${{ matrix.os }}, stable: ${{ matrix.stable }}" runs-on: ${{ matrix.os }} continue-on-error: ${{ !matrix.stable }} strategy: From 547a3772262e3dfa90fd749be9887402dcf55573 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Mon, 13 Sep 2021 10:40:45 +1000 Subject: [PATCH 28/30] refactor: remove sentry Remove `sentry` from the set of vendored services. Support for `SENTRY_DSN` remains on service and interfaces. --- README.md | 1 - compose-files/sentry/docker-compose.yml | 61 ----------------------- placeos | 18 ------- scripts/setup-sentry.sh | 65 ------------------------- 4 files changed, 145 deletions(-) delete mode 100644 compose-files/sentry/docker-compose.yml delete mode 100755 scripts/setup-sentry.sh diff --git a/README.md b/README.md index 29d5a11..43e21ac 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,6 @@ Arguments: --application APP_NAME Application to configure. [default: backoffice] --domain DOMAIN Domain to configure. [default: localhost:8443] --hard-reset Reset the environment to a default state. - -s, --sentry Set-up Sentry -v, --verbose Write logs to STDOUT in addition to the log file. -h, --help Display this message ``` diff --git a/compose-files/sentry/docker-compose.yml b/compose-files/sentry/docker-compose.yml deleted file mode 100644 index 237bd6e..0000000 --- a/compose-files/sentry/docker-compose.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: "3.7" -services: - - postgres: - image: postgres:alpine - restart: always - container_name: postgres - hostname: postgres - volumes: - - ./data/postgres-data:/var/lib/postgresql/data - environment: - - POSTGRES_USER=sentry - - POSTGRES_PASSWORD=test - - TZ=$TZ - - sentry: - image: sentry - restart: always - container_name: sentry - hostname: sentry - ports: - - 0.0.0.0:8989:9000 - environment: - - TZ=$TZ - - SENTRY_SECRET_KEY='dev' - - SENTRY_POSTGRES_HOST=postgres - - SENTRY_DB_USER=sentry - - SENTRY_DB_PASSWORD=test - - SENTRY_REDIS_HOST=redis - - sentry-cron: - image: sentry - restart: always - container_name: sentry-cron - hostname: sentry-cron - command: sentry run cron - depends_on: - - sentry - environment: - - TZ=$TZ - - SENTRY_SECRET_KEY='dev' - - SENTRY_POSTGRES_HOST=postgres - - SENTRY_DB_USER=sentry - - SENTRY_DB_PASSWORD=test - - SENTRY_REDIS_HOST=redis - - sentry-worker: - image: sentry - restart: always - container_name: sentry-worker - hostname: sentry-worker - command: sentry run worker - depends_on: - - sentry - environment: - - TZ=$TZ - - SENTRY_SECRET_KEY='dev' - - SENTRY_POSTGRES_HOST=postgres - - SENTRY_DB_USER=sentry - - SENTRY_DB_PASSWORD=test - - SENTRY_REDIS_HOST=redis diff --git a/placeos b/placeos index 571de9e..8f6fa7f 100755 --- a/placeos +++ b/placeos @@ -112,7 +112,6 @@ Arguments: --password PASSWORD Password for created admin account. [default: development] --domain DOMAIN Domain to configure. [default: localhost:8443] --application APP Application to configure. [default: backoffice] - -s, --sentry Set-up Sentry. -k, --kibana Set-up Kibana and Elastic stack. -v, --verbose Write logs to STDOUT in addition to the log file. -h, --help Display this message. @@ -122,7 +121,6 @@ EOF start_environment() { PROFILES="" hard_reset=false - setup_sentry=false enable_kibana=false email_argument="" password_argument="" @@ -159,10 +157,6 @@ start_environment() { application_arguement="${1}" shift ;; - -s | --sentry) - setup_sentry=true - SERVICES+=('-s') - ;; -k | --kibana) enable_kibana=true ;; @@ -306,18 +300,6 @@ start_environment() { "Configuring InfluxDB..." \ "Failed to configure InfluxDB." - if [[ $setup_sentry == "true" ]]; then - run_or_abort \ - "${base_path}/scripts/setup-sentry.sh" \ - "Setting up sentry..." \ - "Failed to setup sentry" - - echo "░░░ Sentry setup complete. Login to http://$PLACE_DOMAIN on port 8989 with..." - echo "░░░ ${PLACE_EMAIL}:${PLACE_PASSWORD}" - else - echo "░░░ To setup Sentry, run \`${base_path}/placeos start -s\`" - fi - [ ${hard_reset} == "true" ] && hard_reset run_or_abort \ diff --git a/scripts/setup-sentry.sh b/scripts/setup-sentry.sh deleted file mode 100755 index b188587..0000000 --- a/scripts/setup-sentry.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -printf "Removing Sentry containers\n" -printf -- "--------------------------\n\n" - -yes | docker-compose -f ./docker-compose.yml -f ./compose-files/sentry/docker-compose.yml \ - stop sentry sentry-cron sentry-worker 2> /dev/null -yes | docker-compose -f ./docker-compose.yml -f ./compose-files/sentry/docker-compose.yml \ - rm sentry sentry-cron sentry-worker 2> /dev/null - -# temp key in docker-compose for dev -#printf -- "--------------------------\n\n" -#printf "Generating Sentry secret key\n" -#printf -- "--------------------------\n\n" - -#SENTRY_SECRET_KEY=`docker run --rm sentry config generate-secret-key` -#echo "${SENTRY_SECRET_KEY}" -SENTRY_SECRET_KEY=dev - -# These docker runs can probbaly just exec in the running sentry container? (don't stop it ^) -printf -- "\n--------------------------\n\n" -printf "Setting up database\n" -printf -- "--------------------------\n\n" - -docker-compose -f ./docker-compose.yml -f ./compose-files/sentry/docker-compose.yml run \ - -e SENTRY_SECRET_KEY=$SENTRY_SECRET_KEY \ - -e SENTRY_POSTGRES_HOST=postgres \ - -e SENTRY_DB_USER=sentry \ - -e SENTRY_DB_PASSWORD=test \ - -e SENTRY_REDIS_HOST=redis \ - -e SENTRY_SINGLE_ORGANIZATION=true \ - sentry upgrade --noinput --lock - -printf -- "--------------------------\n\n" -printf "Creating admin user\n" -printf -- "--------------------------\n\n" - -docker-compose -f ./docker-compose.yml -f ./compose-files/sentry/docker-compose.yml run --rm \ - -e SENTRY_SECRET_KEY=$SENTRY_SECRET_KEY \ - -e SENTRY_POSTGRES_HOST=postgres \ - -e SENTRY_DB_USER=sentry \ - -e SENTRY_DB_PASSWORD=test \ - -e SENTRY_REDIS_HOST=redis \ - -e SENTRY_SINGLE_ORGANIZATION=true \ - sentry createuser --email=support@place.tech --password=test --superuser --no-input - -# Don't edit docker-compose - temp key for dev -#printf -- "--------------------------\n\n" -#printf "Replacing secret key in docker-compose.yml\n" -#printf -- "--------------------------\n\n" - -#SENTRY_SECRET_KEY=$(echo $SENTRY_SECRET_KEY | sed 's/\&/\\&/g') -#SENTRY_SECRET_KEY=$(echo $SENTRY_SECRET_KEY | sed 's/\^/\\^/g') -#sed -i "s/SENTRY_SECRET_KEY=temp/SENTRY_SECRET_KEY='${SENTRY_SECRET_KEY}'/" ./docker-compose.yml - -printf -- "--------------------------\n\n" -printf "Recreating Sentry containers\n" -printf -- "--------------------------\n\n" - -docker-compose -f docker-compose.yml -f compose-files/sentry/docker-compose.yml \ - up -d sentry sentry-cron sentry-worker - -printf -- "--------------------------\n\n" -printf "You should now be able to login to localhost:8989 as support@place.tech:test\n" -printf -- "--------------------------\n\n" From 82dedcfbbad63a56726f1feda63b7aedbf988e95 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Mon, 13 Sep 2021 11:15:26 +1000 Subject: [PATCH 29/30] build(docker-compose): update logging env yaml anchor --- .github/workflows/ci.yml | 2 +- docker-compose.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e3e978..229e453 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push jobs: test: - name: "version: ${{ matrix.version != '' && matrix.version || 'empty'}}, os: ${{ matrix.os }}, stable: ${{ matrix.stable }}" + name: "placeos: ${{ matrix.version != '' && matrix.version || 'empty'}}, os: ${{ matrix.os }}, stable: ${{ matrix.stable }}" runs-on: ${{ matrix.os }} continue-on-error: ${{ !matrix.stable }} strategy: diff --git a/docker-compose.yml b/docker-compose.yml index bf22505..b530448 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -451,7 +451,7 @@ services: hostname: logstash networks: placeos: - << : *logging-env + << : *std-logging depends_on: - validate-logstash-config volumes: @@ -482,7 +482,7 @@ services: hostname: logspout networks: placeos: - << : *logging-env + << : *std-logging volumes: - /var/run/docker.sock:/var/run/docker.sock command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} @@ -496,7 +496,7 @@ services: hostname: kibana networks: placeos: - << : *logging-env + << : *std-logging environment: <<: *elastic-client-env NODE_OPTIONS: "--max-old-space-size=200" # fixes memory leak (https://github.com/elastic/kibana/issues/5170) @@ -516,7 +516,7 @@ services: hostname: curator networks: placeos: - << : *logging-env + << : *std-logging volumes: - ${PWD}/config/curator/action-file.yml:/action-file.yml - ${PWD}/config/curator/config.yml:/config.yml @@ -532,7 +532,7 @@ services: user: root networks: placeos: - << : *logging-env + << : *std-logging environment: <<: *elastic-client-env volumes: From b1fef1178d0598f79e14ddf118616fa3091e68af Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Tue, 15 Mar 2022 11:21:02 +1100 Subject: [PATCH 30/30] feat(analytics): optional analytics stack --- .env | 22 +++--- compose-files/metricbeat.yml | 20 ------ docker-compose.yml | 127 +++++++++++++++++++++++++++++++++-- placeos | 91 ++++++++++++++++--------- scripts/metricbeat | 8 --- scripts/start-services | 17 ----- 6 files changed, 192 insertions(+), 93 deletions(-) delete mode 100644 compose-files/metricbeat.yml delete mode 100755 scripts/metricbeat delete mode 100755 scripts/start-services diff --git a/.env b/.env index e08cc4f..3629bcd 100644 --- a/.env +++ b/.env @@ -38,12 +38,22 @@ PLACE_METRICS_ROUTE=monitor ELASTIC_HOST=elastic ELASTIC_PORT=9200 -ELASTIC_VERSION=7.16.2 +ELASTIC_VERSION=7.10.2 ETCD_HOST=etcd ETCD_PORT=2379 ETCD_VERSION=3.5.1 +REDIS_URL=redis://redis:6379 + +RETHINKDB_DB=place_development +RETHINKDB_HOST=rethink +RETHINKDB_PORT=28015 +RETHINKDB_VERSION=2.4 + +# Analytics variables +ENABLE_ANALYTICS=true + # INFLUX_USER=placeos # INFLUX_PASSWORD=development @@ -52,20 +62,14 @@ INFLUX_HOST=http://influxdb:8086 INFLUX_ORG=PlaceOS INFLUX_RETENTION=4w -REDIS_URL=redis://redis:6379 - -RETHINKDB_DB=place_development -RETHINKDB_HOST=rethink -RETHINKDB_PORT=28015 -RETHINKDB_VERSION=2.4 - # Staff API variables POSTGRES_USER=placeos POSTGRES_PASSWORD=development -# Monitor Node variables +# Logging variables +ENABLE_KIBANA=true LOGSTASH_HOST=logstash LOGSTASH_PORT=12201 KIBANA_PORT=443 diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml deleted file mode 100644 index e7ac66a..0000000 --- a/compose-files/metricbeat.yml +++ /dev/null @@ -1,20 +0,0 @@ - # Gets metrics from host machine and send to elastic - metricbeat: - image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} - hostname: $MONITOR_HOSTNAME - container_name: metricbeat - user: root - networks: - placeos: - volumes: - - /proc:/hostfs/proc:ro - - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro - - /:/hostfs:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml - cap_add: - - SYS_PTRACE - - DAC_READ_SEARCH - command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] - restart: unless-stopped - << : *logging-env diff --git a/docker-compose.yml b/docker-compose.yml index 706950a..73dc207 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ x-jwt-public-key-env: &jwt-public-key-env .env.public_key x-secret-key-env: &secret-key-env .env.secret_key x-elastic-client-env: &elastic-client-env + ELASTIC_HOST: ${ELASTIC_HOST:-elastic} + ELASTIC_PORT: ${ELASTIC_PORT:-9200} ES_HOST: ${ELASTIC_HOST:-elastic} ES_PORT: ${ELASTIC_PORT:-9200} @@ -191,9 +193,12 @@ services: PLACE_LOADER_WWW: www source: - image: docker.io/placeos/source:${PLACE_SOURCE_TAG:-nightly} + image: placeos/source:${PLACE_SOURCE_TAG:-nightly} + profiles: + - analytics restart: always container_name: source + hostname: source <<: *std-network <<: *std-logging depends_on: @@ -235,6 +240,7 @@ services: postgres: # Database used by Staff API image: postgres:${POSTGRES_VERSION:-13-alpine} container_name: postgres + hostname: postgres restart: unless-stopped <<: *std-network <<: *std-logging @@ -248,6 +254,7 @@ services: staff: # Staff API image: placeos/staff-api:${PLACE_STAFF_API_TAG:-nightly} container_name: staff + hostname: staff restart: unless-stopped <<: *std-network <<: *std-logging @@ -323,12 +330,13 @@ services: # Resources elastic: - image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.9.1} + image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.16.2} restart: always container_name: elastic hostname: elastic healthcheck: test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health + start_period: 1m <<: *std-network <<: *std-logging volumes: @@ -356,11 +364,13 @@ services: influxdb: image: influxdb:${INFLUXDB_IMAGE_TAG:-2.0.8-alpine} - container_name: influx + profiles: + - analytics restart: always + container_name: influx + hostname: influx <<: *std-network <<: *std-logging - hostname: influx healthcheck: test: influx bucket list volumes: @@ -368,11 +378,14 @@ services: source: influx-data target: /root/.influxdbv2 command: "--reporting-disabled" - + chronograf: image: chronograf:${CHRONOGRAF_IMAGE_TAG:-1.9} - container_name: chronograf + profiles: + - analytics restart: always + container_name: chronograf + hostname: chronograf <<: *std-network <<: *std-logging env_file: @@ -394,6 +407,8 @@ services: mosquitto: image: iegomez/mosquitto-go-auth:${MOSQUITTO_IMAGE_TAG:-latest} + profiles: + - analytics restart: always container_name: mosquitto hostname: mosquitto @@ -466,3 +481,103 @@ services: target: /data/rethinkdb_data environment: TZ: $TZ + + # Aggregates logs and forwards them to Elasticsearch. + logstash: + image: blacktop/logstash:${ELASTIC_VERSION:-7.6} + profiles: + - kibana + restart: always + container_name: logstash + hostname: logstash + << : *std-network + << : *std-logging + depends_on: + - validate-logstash-config + volumes: + - ${PWD}/config/logstash/config:/config + - ${PWD}/config/logstash/patterns:/opt/logstash/extra_patterns + command: logstash -f /config + + # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. + validate-logstash-config: + image: blacktop/logstash:${ELASTIC_VERSION:-7.6} + profiles: + - kibana + restart: "no" + container_name: validate-logstash + << : *std-network + << : *std-logging + volumes: + - ${PWD}/config/logstash/config:/config + command: logstash -t -f /config + + # Sends all container json-file logs to logstash + logspout: + image: vincit/logspout-gelf:3.2.6-alpine + profiles: + - kibana + restart: unless-stopped + container_name: logspout + hostname: logspout + << : *std-network + << : *std-logging + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} + + kibana: + image: blacktop/kibana:${ELASTIC_VERSION:-7.6} + profiles: + - kibana + restart: always + container_name: kibana + hostname: kibana + << : *std-network + << : *std-logging + environment: + <<: *elastic-client-env + NODE_OPTIONS: "--max-old-space-size=200" # fixes memory leak (https://github.com/elastic/kibana/issues/5170) + HTTPS_METHOD: "nohttp" + ELASTICSEARCH_HOSTS: "http://${ELASTIC_HOST}:${ELASTIC_PORT}" + SERVER_BASEPATH: "/${PLACE_METRICS_ROUTE}" + SERVER_REWRITEBASEPATH: "true" + SERVER_PUBLICBASEURL: "https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" + + # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. + # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. + curator: + image: bobrik/curator:5.7.6 + profiles: + - kibana + container_name: curator + hostname: curator + << : *std-network + << : *std-logging + volumes: + - ${PWD}/config/curator/action-file.yml:/action-file.yml + - ${PWD}/config/curator/config.yml:/config.yml + + # Gets metrics from host machine and send to elastic + metricbeat: + image: elastic/metricbeat:${ELASTIC_VERSION:-7.6} + profiles: + - metricbeat + restart: unless-stopped + container_name: metricbeat + hostname: metricbeat + user: root + << : *std-network + << : *std-logging + environment: + <<: *elastic-client-env + volumes: + - /proc:/hostfs/proc:ro + - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro + - /:/hostfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ${PWD}/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml + cap_add: + - SYS_PTRACE + - DAC_READ_SEARCH + command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] diff --git a/placeos b/placeos index 09c666d..d5a6b8f 100755 --- a/placeos +++ b/placeos @@ -75,6 +75,17 @@ logfile="${base_path}/.logs/$(date +"%Y%m%d%H%M").log" COMPOSE_PROJECT_NAME=placeos +# Helpers +################################################################################################### + +unknown_argument() ( + unknown_arg="${1}" + usage="${2}" + [ -n "${unknown_arg}" ] && echo -e "░░░ ${red}Unknown option:${reset} ${unknown_arg}" + eval "${usage}" + exit 1 +) + abort() ( echo -e "░░░ ${red}${1}${reset}" echo "░░░ Logs can be found in ${logfile}" @@ -102,6 +113,9 @@ run_or_abort() { fi } +# Start +################################################################################################### + hard_reset() ( # TODO: drop influxdb tables # TODO: clear redis @@ -134,14 +148,18 @@ Arguments: --password PASSWORD Password for created admin account. [default: development] --domain DOMAIN Domain to configure. [default: localhost:8443] --application APP Application to configure. [default: backoffice] + --analytics Set-up analytics stack, including MQTT & InfluxDB & Chronograf + --kibana Set-up Kibana and Elastic stack. -v, --verbose Write logs to STDOUT in addition to the log file. -h, --help Display this message. EOF ) start_environment() ( - SERVICES=('') + PROFILES="" hard_reset=false + enable_analytics=false + enable_kibana=false email_argument="" password_argument="" domain_argument="" @@ -177,6 +195,12 @@ start_environment() ( application_arguement="${1}" shift ;; + --analytics) + enable_analytics=true + ;; + --kibana) + enable_kibana=true + ;; -v | --verbose) VERBOSE="true" ;; @@ -185,12 +209,7 @@ start_environment() ( exit 0 ;; *) - if [ -n "${command}" ]; then - echo -e "░░░ ${red}Unknown option:${reset} ${command}" - else - start_environment__usage - exit 1 - fi + unknown_argument "${command}" "start_environment__usage" ;; esac done @@ -199,8 +218,6 @@ start_environment() ( load_environment - # Override .env arguments with CLI arguments - echo "░░░ Starting PlaceOS <${PLACEOS_TAG}>" [ $VERBOSE == "false" ] && echo "░░░ For detailed logging, run \`tail -f ${logfile}\`" @@ -257,28 +274,39 @@ start_environment() ( echo "" done - # run_or_abort \ - # "${base_path}/scripts/metricbeat" \ - # "Checking Host OS..." \ - # "Error occurred while checking Host OS." - # Write the email so as to not prompt the user again. echo "PLACE_EMAIL=${PLACE_EMAIL}" >"${EMAIL_ENV}" # TODO: use init check instead of writing the password. echo "PLACE_PASSWORD=${PLACE_PASSWORD}" >>"${EMAIL_ENV}" + if [[ $ENABLE_ANALYTICS == "true" ]] || [[ $enable_analytics == "true" ]]; then + PROFILES+=" --profile analytics" + fi + if [[ -d "${base_path}/.htpasswd-kibana" ]]; then echo "░░░ Detected malformed auth file. Cleaning up" rm -r "${base_path}/.htpasswd-kibana" fi + if [[ $ENABLE_KIBANA == "true" ]] || [[ $enable_kibana == "true" ]]; then + PROFILES+=" --profile kibana" + if [[ $(uname) == "Linux" ]]; then + PROFILES+=" --profile metricbeat" + fi + fi + run_or_abort \ "${base_path}/scripts/generate-secrets" \ "Generating secrets..." \ "Failed to generate secrets." run_or_abort \ - "${base_path}/scripts/start-services ${SERVICES[@]}" \ + "docker-compose ${PROFILES} pull -q" \ + "Pulling images..." \ + "Failed to pull images." + + run_or_abort \ + "docker-compose ${PROFILES} up -d" \ "Bringing up services..." \ "Failed to start services." @@ -298,6 +326,9 @@ start_environment() ( echo "░░░ $PLACE_EMAIL:$PLACE_PASSWORD" ) +# Stop +################################################################################################### + stop_environment__usage() ( cat < [help|...] [arguments...] @@ -383,6 +412,9 @@ task() ( ./scripts/run-sam-task ${PARAMS} ) +# Update +################################################################################################### + update_environment__usage() ( cat <>docker-compose.yml - fi -fi diff --git a/scripts/start-services b/scripts/start-services deleted file mode 100755 index 3b57e68..0000000 --- a/scripts/start-services +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -COMPOSE_FILES=('-f ./docker-compose.yml') - -for arg in "$@"; do - case $arg in - -e | --elk) - COMPOSE_FILES+=('-f ./compose-files/elk/docker-compose.yml') - ;; - esac -done - -docker-compose ${COMPOSE_FILES[@]} pull --quiet - -docker-compose ${COMPOSE_FILES[@]} up --detach --remove-orphans