feat(saas): Add custom strategy the fork a catalog item and add customization

ansible/playbooks/paas/main.yml

@@ -27,6 +27,7 @@
           - unzip
           - make
           - jq
+          - dnsutils
 
         state: present
         install_recommends: false

ansible/playbooks/paas/roles/nomad/defaults/main.yml

@@ -146,10 +146,10 @@ nomad_client_host_network_cluster:
   name: cluster
   interface: "{{ nomad_cluster_bridge }}"
 
-nomad_client_meta_list: >-
-  {"arch": "{{ architecture_map[ansible_facts.architecture] }}",
-   "location": "{{ fact_instance.location }}",
-   "instance": "{{ inventory_hostname }}"}
+nomad_client_meta_list:
+  arch: "{{ architecture_map[ansible_facts.architecture] }}"
+  location: "{{ fact_instance.location }}"
+  instance: "{{ inventory_hostname }}"
 
 nomad_server_join: >-
   "{% if nomad_mode == 'single' %}127.0.0.1{% else %}{{ (groups[nomad_deploy_cluster_name] |
@@ -195,8 +195,7 @@ nomad_tls_ca_provider: ownca
 nomad_tls_host_certificate_dir: /etc/ssl/simplestack
 
 nomad_tls_common_name: nomad
-# IP range for 192.168.0.0/24 (all 256 addresses)
-nomad_tls_ip_range: "{{ range(0,256) | map('regex_replace', '^', 'IP:192.168.0.') | list | join(',') }}"
+nomad_tls_ip_range: "{{ ((range(0,256) | map('regex_replace', '^', 'IP:192.168.0.')) + (range(0,256) | map('regex_replace', '^', 'IP:10.0.0.')) ) | list | join(',') }}"
 nomad_tls_check_delay: "+2w"
 
 # TLS Server

ansible/playbooks/paas/roles/nomad/tasks/04_tls_certs.yml

@@ -60,7 +60,7 @@
       notify: Nomad_restart
 
 - name: Nomad | Copy certificate on client nodes
-  when: nomad_node_role in ['client']
+  when: nomad_node_role in ['client', 'both']
   block:
     - name: "Nomad | Check if TLS cert exists for Client"
       ansible.builtin.stat:

ansible/playbooks/paas/roles/nomad/templates/client.hcl.j2

@@ -40,7 +40,7 @@ client {
     }
 {% endif %}
 
-{% if nomad_client_meta_list%}
+{% if nomad_client_meta_list %}
     meta = {
     {% for key, value in nomad_client_meta_list.items() %}
     "{{ key }}" = "{{ value }}"

ansible/playbooks/paas/roles/nomad/templates/nomad.hcl.j2

@@ -30,6 +30,8 @@ log_rotate_max_files = {{ nomad_log_rotate_max_files }}
 leave_on_terminate = {{ nomad_leave_on_terminate | lower }}
 leave_on_interrupt = {{ nomad_leave_on_interrupt | lower }}
 
+consul {}
+
 tls {
     http = true
     rpc = true

ansible/playbooks/paas/roles/prometheus/templates/config.j2

@@ -223,7 +223,7 @@ scrape_configs:
 
   - job_name: 'mimir_exporter'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -262,7 +262,7 @@ scrape_configs:
 
   - job_name: 'traefik'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -297,7 +297,7 @@ scrape_configs:
     metrics_path: /minio/v2/metrics/cluster
     scheme: http
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -324,7 +324,7 @@ scrape_configs:
 
   - job_name: 'caddy'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -356,7 +356,7 @@ scrape_configs:
 
   - job_name: 'vllm'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -388,7 +388,7 @@ scrape_configs:
 
   - job_name: 'mysql_exporter'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -419,7 +419,7 @@ scrape_configs:
 
   - job_name: 'nginx_exporter'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -450,7 +450,7 @@ scrape_configs:
 
   - job_name: 'phpfpm_exporter'
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true
@@ -484,7 +484,7 @@ scrape_configs:
     metrics_path: /api/prometheus
     scheme: http
     nomad_sd_configs:
-      - server: "https://{{ nomad_primary_master_address | default(inventory_hostname) }}:4646"
+      - server: "https://{{ nomad_primary_master_address | default(hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname].nomad_iface]['ipv4']['address']) }}:4646"
         region: "{{ fact_instance.region }}"
         tls_config:
           insecure_skip_verify: true

ansible/playbooks/paas/roles/script_exporter/vars/main.yml

@@ -22,3 +22,4 @@ script_exporter_scripts:
       armv7: armhf
       amd64: amd64
       x86_64: x86_64
+      aarch64: aarch64

ansible/playbooks/saas/image-forkable.yml

@@ -0,0 +1,107 @@
+---
+- name: Build a Docker image from a catalog id
+  hosts: "{{ hosts_limit | default('infrastructure') }}"
+  become: true
+  gather_facts: true
+  vars_prompt:
+    - name: catalog_id
+      prompt: "Catalog item ID"
+      private: false
+  vars:
+    architecture_map:
+      amd64: amd64
+      x86_64: amd64
+      armv7l: arm
+      aarch64: arm64
+      arm64: arm64
+    catalog: "{{ catalog_response.json.origin | default(catalog_response.json.name) }}"
+    build_work_dir: "/tmp/{{ catalog }}"
+    download_dir: "{{ build_work_dir }}/download"
+    arch_dir: "{{ build_work_dir }}/{{ upstream_default_arch }}"
+    upstream_default_arch: "{{ architecture_map[ansible_facts.architecture] }}"    
+    ui_url: "{{ lookup('ansible.builtin.env', 'SIMPLE_STACK_UI_URL') }}"
+    ui_user: "{{ lookup('ansible.builtin.env', 'SIMPLE_STACK_UI_USER') }}"
+    ui_password: "{{ lookup('ansible.builtin.env', 'SIMPLE_STACK_UI_PASSWORD') }}"
+
+  pre_tasks:
+    - name: Retrieve catalog item from UI
+      ansible.builtin.uri:
+        url: "{{ ui_url }}/api"
+        user: "{{ ui_user }}"
+        password: "{{ ui_password }}"
+        method: POST
+        body_format: json
+        body:
+          schema: "catalogs_read/{{ catalog_id }}"
+        force_basic_auth: true
+        status_code: 200
+      delegate_to: localhost
+      register: catalog_response
+      become: false
+
+    - name: Ensure temporary build directories exist
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        mode: "0755"
+      loop:
+        - "{{ download_dir }}"
+        - "{{ arch_dir }}"
+
+  tasks:
+    - name: Build assets
+      ansible.builtin.include_role:
+        name: "{{ catalog }}"
+        tasks_from: build
+      vars:
+        catalog_image_name: "{{ catalog_response.json.name }}"
+        dockerfile_root: "{{ catalog_response.json.dockerfile_root | default('') }}"
+        dockerfile_nonroot: "{{ catalog_response.json.dockerfile_nonroot | default('') }}"
+
+    - name: Build and push Docker image
+      community.docker.docker_image_build:
+        name: "{{ docker_private_registry.url }}/{% if docker_private_registry.project is defined %}{{ docker_private_registry.project }}/{% endif %}{{ catalog_response.json.name }}:{{ image_version }}"
+        tag: latest
+        path: "{{ build_work_dir }}"
+        dockerfile: Dockerfile
+        labels: "{{ image_definition.labels | default({}) }}"
+        rebuild: always
+        outputs:
+          - type: image
+            push: true
+      register: docker_build
+      when: image_definition.build
+      notify: Cleanup build directory
+
+    - name: Update catalog item version on UI
+      ansible.builtin.uri:
+        url: "{{ ui_url }}/api"
+        user: "{{ ui_user }}"
+        password: "{{ ui_password }}"
+        method: POST
+        body_format: json
+        body:
+          schema: catalogs_create
+          data:
+            name: "{{ catalog_response.json.name }}"
+            version: "{{ image_version }}"
+            forkable: "{{ image_forkable | default(false) }}"
+        force_basic_auth: true
+        status_code: 200
+      delegate_to: localhost
+      register: ui_update
+      failed_when: ui_update.status != 200
+      become: false
+
+  handlers:
+    - name: Cleanup build directory
+      ansible.builtin.file:
+        path: "{{ build_work_dir }}"
+        state: absent
+      listen: cleanup_build
+
+  post_tasks:
+    - name: Trigger cleanup on failure
+      ansible.builtin.meta: clear_host_errors
+      when: ansible_failed_result is defined
+      notify: Cleanup build directory

ansible/playbooks/saas/image.yml

@@ -16,7 +16,6 @@
       aarch64: arm64
       arm64: arm64
     upstream_default_arch: "{{ architecture_map[ansible_facts.architecture] }}"
-    path: "/tmp/{{ catalog }}"
 
   pre_tasks:
     - name: Create temporary build directory
@@ -34,32 +33,31 @@
       ansible.builtin.include_role:
         name: "{{ catalog }}"
         tasks_from: build
+      vars:
+        catalog_image_name: "{{ catalog }}"
+        dockerfile_root: ""
+        dockerfile_nonroot: ""
 
     - name: Debug latest version
       ansible.builtin.debug:
         msg: "{{ image_version }}"
 
     - name: Build
-      when: image_build
+      when: image_definition.build
       block:
         - name: Build and publish image
           community.docker.docker_image_build:
-            name: "{{ docker_private_registry.url }}/{% if docker_private_registry.project is defined %}{{ docker_private_registry.project }}/{% endif %}{{ image_name }}:{{ image_version }}"
+            name: "{{ docker_private_registry.url }}/{% if docker_private_registry.project is defined %}{{ docker_private_registry.project }}/{% endif %}{{ image_definition.name }}:{{ image_version }}"
             tag: latest
-            path: "/tmp/{{ catalog }}"
+            path: "{{ build_work_dir }}"
             dockerfile: Dockerfile
-            labels: "{{ image_labels }}"
+            labels: "{{ image_definition.labels }}"
             rebuild: always
             outputs:
               - type: image
                 push: true
           register: docker_image_build
 
-        - name: Debug
-          ansible.builtin.debug:
-            msg: "{{ docker_image_build }}"
-            verbosity: 1
-
     - name: Update catalog item version on UI
       ansible.builtin.uri:
         url: "{{ lookup('ansible.builtin.env', 'SIMPLE_STACK_UI_URL') }}/api"
@@ -70,7 +68,7 @@
         body:
           schema: catalogs_create
           data:
-            name: "{{ image_name }}"
+            name: "{{ image_definition.name }}"
             version: "{{ image_version }}"
         force_basic_auth: true
         status_code: 200

ansible/playbooks/saas/roles/adguard/tasks/build.yml

@@ -5,39 +5,17 @@
 - name: Set custom variables
   ansible.builtin.set_fact:
     image_version: "{{ latest_version }}"
-    image_name: "{{ image.name }}"
-    image_labels: "{{ image.labels }}"
-    image_build: "{{ image.build }}"
-
+    image_definition: "{{ image }}"
+    image_forkable: "{{ image.forkable }}"
+    upstream_file_url: "{{ upstream_file_url }}"
+    upstream_file_name: "{{ upstream_file_name }}"
+
 - name: End playbook if no new version
   ansible.builtin.meta: end_host
-  when: softwares[image.name] is defined and softwares[image.name].version == image_version
-
-- name: Download Github release
-  ansible.builtin.get_url:
-    url: "{{ upstream_file_url }}"
-    dest: "{{ build_work_dir }}/download/"
-    mode: '0644'
-    force: no
-  register: download_result
-
-- name: Unarchive GitHub release
-  ansible.builtin.unarchive:
-    src: "{{ build_work_dir }}/download/{{ upstream_file_name }}"
-    dest: "{{ build_work_dir }}/download"
-    remote_src: true
-  when: download_result.changed
-
-- name: Copy binary in build directory
-  ansible.builtin.copy:
-    src: "{{ path }}/download/AdGuardHome/{{ image.upstream.binary }}"
-    dest: "{{ path }}/{{ image.upstream.binary }}"
-    mode: '0755'
-    remote_src: true
-  when: download_result.changed
+  when: softwares[catalog_image_name] is defined and softwares[catalog_image_name].version == image_version
 
 - name: Copy dockerfile
   ansible.builtin.template:
     src: Dockerfile.j2
-    dest: "{{ path }}/Dockerfile"
+    dest: "{{ build_work_dir }}/Dockerfile"
     mode: '0644'

ansible/playbooks/saas/roles/adguard/templates/Dockerfile.j2

@@ -4,7 +4,13 @@ FROM {{ image.origin }}
 
 ARG TARGETARCH
 
-COPY {{ image.upstream.binary }} /usr/local/bin/{{ image.upstream.binary }}
+RUN apk add wget
+
+RUN wget -O {{ upstream_file_name }} {{ upstream_file_url }} && \
+    tar xzf {{ upstream_file_name }} && \
+    mv AdGuardHome/AdGuardHome /usr/local/bin/AdGuardHome
+
+{{ dockerfile_root | default('') }}
 
 USER root
 

ansible/playbooks/saas/roles/adguard/vars/main.yml

@@ -1,6 +1,7 @@
 ---
 image:
   build: true
+  forkable: false
   upstream:
     source: github
     user: AdguardTeam

ansible/playbooks/saas/roles/arangodb/tasks/build.yml

@@ -5,10 +5,8 @@
 - name: Set custom variables
   ansible.builtin.set_fact:
     image_version: "{{ latest_version }}"
-    image_name: "{{ image.name }}"
-    image_labels: "{{ image.labels }}"
-    image_build: "{{ image.build }}"
+    image_definition: "{{ image }}"
 
 - name: End playbook if no new version
   ansible.builtin.meta: end_host
-  when: softwares[image.name] is defined and softwares[image.name].version == image_version
+  when: softwares[catalog_image_name] is defined and softwares[catalog_image_name].version == image_version