diff --git a/.env b/.env index e08cc4f..1b2790d 100644 --- a/.env +++ b/.env @@ -38,12 +38,23 @@ PLACE_METRICS_ROUTE=monitor ELASTIC_HOST=elastic ELASTIC_PORT=9200 -ELASTIC_VERSION=7.16.2 +ELASTIC_OD_VERSION=1.13.2 +ELASTIC_VERSION=7.10.2 ETCD_HOST=etcd ETCD_PORT=2379 ETCD_VERSION=3.5.1 +REDIS_URL=redis://redis:6379 + +RETHINKDB_DB=place_development +RETHINKDB_HOST=rethink +RETHINKDB_PORT=28015 +RETHINKDB_VERSION=2.4 + +# Analytics variables +ENABLE_ANALYTICS=true + # INFLUX_USER=placeos # INFLUX_PASSWORD=development @@ -52,20 +63,14 @@ INFLUX_HOST=http://influxdb:8086 INFLUX_ORG=PlaceOS INFLUX_RETENTION=4w -REDIS_URL=redis://redis:6379 - -RETHINKDB_DB=place_development -RETHINKDB_HOST=rethink -RETHINKDB_PORT=28015 -RETHINKDB_VERSION=2.4 - # Staff API variables POSTGRES_USER=placeos POSTGRES_PASSWORD=development -# Monitor Node variables +# Logging variables +ENABLE_KIBANA=true LOGSTASH_HOST=logstash LOGSTASH_PORT=12201 KIBANA_PORT=443 diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml deleted file mode 100644 index e7ac66a..0000000 --- a/compose-files/metricbeat.yml +++ /dev/null @@ -1,20 +0,0 @@ - # Gets metrics from host machine and send to elastic - metricbeat: - image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} - hostname: $MONITOR_HOSTNAME - container_name: metricbeat - user: root - networks: - placeos: - volumes: - - /proc:/hostfs/proc:ro - - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro - - /:/hostfs:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml - cap_add: - - SYS_PTRACE - - DAC_READ_SEARCH - command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] - restart: unless-stopped - << : *logging-env diff --git a/docker-compose.yml b/docker-compose.yml index 706950a..bcab8e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.7" +version: "3.9" networks: placeos: @@ -34,6 +34,8 @@ x-jwt-public-key-env: &jwt-public-key-env .env.public_key x-secret-key-env: &secret-key-env .env.secret_key x-elastic-client-env: &elastic-client-env + ELASTIC_HOST: ${ELASTIC_HOST:-elastic} + ELASTIC_PORT: ${ELASTIC_PORT:-9200} ES_HOST: ${ELASTIC_HOST:-elastic} ES_PORT: ${ELASTIC_PORT:-9200} @@ -191,9 +193,12 @@ services: PLACE_LOADER_WWW: www source: - image: docker.io/placeos/source:${PLACE_SOURCE_TAG:-nightly} + image: placeos/source:${PLACE_SOURCE_TAG:-nightly} + profiles: + - analytics restart: always container_name: source + hostname: source <<: *std-network <<: *std-logging depends_on: @@ -235,6 +240,7 @@ services: postgres: # Database used by Staff API image: postgres:${POSTGRES_VERSION:-13-alpine} container_name: postgres + hostname: postgres restart: unless-stopped <<: *std-network <<: *std-logging @@ -248,6 +254,7 @@ services: staff: # Staff API image: placeos/staff-api:${PLACE_STAFF_API_TAG:-nightly} container_name: staff + hostname: staff restart: unless-stopped <<: *std-network <<: *std-logging @@ -315,20 +322,23 @@ services: restart: always container_name: dispatch hostname: dispatch - env_file: - - *secret-key-env <<: *std-network <<: *std-logging + environment: + SERVER_SECRET: ${PLACE_SERVER_SECRET:-development} + env_file: + - *secret-key-env # Resources elastic: - image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.9.1} + image: amazon/opendistro-for-elasticsearch:${ELASTIC_OD_VERSION:-1.13.2} restart: always container_name: elastic hostname: elastic healthcheck: test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health + start_period: 1m <<: *std-network <<: *std-logging volumes: @@ -346,21 +356,23 @@ services: restart: always container_name: etcd hostname: etcd - healthcheck: - test: etcdctl endpoint health <<: *std-network <<: *std-logging + healthcheck: + test: etcdctl endpoint health environment: ALLOW_NONE_AUTHENTICATION: "yes" TZ: $TZ influxdb: image: influxdb:${INFLUXDB_IMAGE_TAG:-2.0.8-alpine} - container_name: influx + profiles: + - analytics restart: always + container_name: influx + hostname: influx <<: *std-network <<: *std-logging - hostname: influx healthcheck: test: influx bucket list volumes: @@ -368,11 +380,14 @@ services: source: influx-data target: /root/.influxdbv2 command: "--reporting-disabled" - + chronograf: image: chronograf:${CHRONOGRAF_IMAGE_TAG:-1.9} - container_name: chronograf + profiles: + - analytics restart: always + container_name: chronograf + hostname: chronograf <<: *std-network <<: *std-logging env_file: @@ -394,6 +409,8 @@ services: mosquitto: image: iegomez/mosquitto-go-auth:${MOSQUITTO_IMAGE_TAG:-latest} + profiles: + - analytics restart: always container_name: mosquitto hostname: mosquitto @@ -439,10 +456,10 @@ services: restart: always container_name: redis hostname: redis - healthcheck: - test: redis-cli ping <<: *std-network <<: *std-logging + healthcheck: + test: redis-cli ping volumes: - type: volume source: redis-data @@ -455,14 +472,114 @@ services: restart: always container_name: rethink hostname: rethink + <<: *std-network + <<: *std-logging healthcheck: # Check if the DB's port is open test: "bash -c ': &>/dev/null " [ $VERBOSE == "false" ] && echo "░░░ For detailed logging, run \`tail -f ${logfile}\`" @@ -257,28 +274,39 @@ start_environment() ( echo "" done - # run_or_abort \ - # "${base_path}/scripts/metricbeat" \ - # "Checking Host OS..." \ - # "Error occurred while checking Host OS." - # Write the email so as to not prompt the user again. echo "PLACE_EMAIL=${PLACE_EMAIL}" >"${EMAIL_ENV}" # TODO: use init check instead of writing the password. echo "PLACE_PASSWORD=${PLACE_PASSWORD}" >>"${EMAIL_ENV}" + if [[ $ENABLE_ANALYTICS == "true" ]] || [[ $enable_analytics == "true" ]]; then + PROFILES+=" --profile analytics" + fi + if [[ -d "${base_path}/.htpasswd-kibana" ]]; then echo "░░░ Detected malformed auth file. Cleaning up" rm -r "${base_path}/.htpasswd-kibana" fi + if [[ $ENABLE_KIBANA == "true" ]] || [[ $enable_kibana == "true" ]]; then + PROFILES+=" --profile kibana" + if [[ $(uname) == "Linux" ]]; then + PROFILES+=" --profile metricbeat" + fi + fi + run_or_abort \ "${base_path}/scripts/generate-secrets" \ "Generating secrets..." \ "Failed to generate secrets." run_or_abort \ - "${base_path}/scripts/start-services ${SERVICES[@]}" \ + "docker-compose ${PROFILES} pull -q" \ + "Pulling images..." \ + "Failed to pull images." + + run_or_abort \ + "docker-compose ${PROFILES} up -d" \ "Bringing up services..." \ "Failed to start services." @@ -298,6 +326,9 @@ start_environment() ( echo "░░░ $PLACE_EMAIL:$PLACE_PASSWORD" ) +# Stop +################################################################################################### + stop_environment__usage() ( cat < [help|...] [arguments...] @@ -383,6 +412,9 @@ task() ( ./scripts/run-sam-task ${PARAMS} ) +# Update +################################################################################################### + update_environment__usage() ( cat <>docker-compose.yml - fi -fi diff --git a/scripts/start-services b/scripts/start-services deleted file mode 100755 index 3b57e68..0000000 --- a/scripts/start-services +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -COMPOSE_FILES=('-f ./docker-compose.yml') - -for arg in "$@"; do - case $arg in - -e | --elk) - COMPOSE_FILES+=('-f ./compose-files/elk/docker-compose.yml') - ;; - esac -done - -docker-compose ${COMPOSE_FILES[@]} pull --quiet - -docker-compose ${COMPOSE_FILES[@]} up --detach --remove-orphans