Fixed bugs, added first documentation iteration.

Author: samoht9277

infra/ansible/README.md

@@ -0,0 +1,60 @@
+- name: Install Alert Manager
+  hosts: "{{ host }}"
+
+  tasks:
+    - name: Install Alert Manager
+      get_url:
+        url: https://github.com/prometheus/alertmanager/releases/download/v{{ alert_manager_version }}/alertmanager-{{ alert_manager_version }}.linux-amd64.tar.gz
+        dest: /tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz
+        mode: '0644'
+
+    - name: Install Alert Manager package
+      become: true
+      unarchive:
+        src: "/tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz"
+        dest: "/usr/local/bin/"
+        remote_src: yes
+        extra_opts:
+        - --strip-components=1
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Clean up Alert Manager tarball package
+      file:
+        path: /tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz
+        state: absent
+
+    - name: Make sure /etc/alertmanager directory exists
+      become: true
+      file:
+        path: /etc/alertmanager
+        state: directory
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Create Alert Manager config file
+      become: true
+      template:
+        src: alert_manager/alertmanager.yml.j2
+        dest: /etc/alertmanager/alertmanager.yml
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+        pagerduty_routing_key: "{{ lookup('ini', 'pagerduty_routing_key', file=ini_file) }}"
+
+    - name: Create Alert Manager systemd service
+      become: true
+      template:
+        src: services/alert_manager.service.j2
+        dest: /etc/systemd/system/alert_manager.service
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Start Alert Manager
+      become: true
+      systemd_service:
+        name: alert_manager
+        state: started
+        daemon_reload: true
+        enabled: true
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"

infra/ansible/playbooks/alert_manager.yaml

@@ -2,6 +2,17 @@
   hosts: "{{ host }}"
 
   tasks:
+  - name: Allow http/https traffic on UFW
+    become: true
+    ufw:
+      rule: allow
+      state: enabled
+      port: '{{ item }}'
+    loop:
+      - http
+      - https
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
 
   - name: Install dependencies for Caddy
     become: true

infra/ansible/playbooks/caddy.yaml

@@ -49,7 +49,6 @@
         deb: "{{ download_libssl.dest }}"
       when: libssl_check.rc != 0
 
-    ########## Install Erlang 26.2.1-1 ##########
     - name: Check if Erlang 26.2.1-1 is installed
       become: true
       ansible.builtin.shell:

infra/ansible/playbooks/elixir.yaml

@@ -0,0 +1,54 @@
+groups:
+
+- name: AllInstances
+  rules:
+  - alert: InstanceDown
+    # Condition for alerting
+    expr: up == 0
+    for: 1m
+    # Annotation - additional informational labels to store more information
+    annotations:
+      title: 'Instance {{ $labels.job }} down'
+      description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute'
+    # Labels - additional labels to be attached to the alert
+    labels:
+      severity: 'critical'
+      text: 'Instance {{ $labels.job }} down'
+
+  - alert: TaskDifferenceAggregatorBatcher
+    # Condition for alerting
+    expr: floor(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m])) - on() floor(increase(sent_batches{job="aligned-batcher"}[15m])) > 1
+    for: 30s
+    # Annotation - additional informational labels to store more information
+    annotations:
+      title: 'Tasks not being received by the aggregator'
+      description: 'The difference between aggregator recevied tasks and batcher sent batches is greater than 1 for more than 30 seconds'
+    # Labels - additional labels to be attached to the alert
+    labels:
+      severity: 'critical'
+      text: 'Tasks not being received by the aggregator'
+
+  - alert: TaskDifferenceAggregator
+    # Condition for alerting
+    expr: floor(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m])) - floor(increase(aligned_aggregated_responses{job="aligned-aggregator"}[15m])) > 1
+    for: 30s
+    # Annotation - additional informational labels to store more information
+    annotations:
+      title: 'Tasks not being verified'
+      description: 'The difference between aggregator received tasks and verified tasks is greater than 1 for more than 30 seconds'
+    # Labels - additional labels to be attached to the alert
+    labels:
+      severity: 'critical'
+      text: 'Tasks not being verified'
+
+  - alert: UserErrorRate
+    # Condition for alerting
+    expr: rate(user_errors[5m]) > 2
+    for: 1m
+    # Annotation - additional informational labels to store more information
+    annotations:
+      title: 'High error rate {{ $labels.error_type }}'
+      description: 'User error rate is greater than 2 for more than 1 minute'
+    labels:
+      severity: 'critical'
+      text: 'High error rate {{ $labels.error_type }}'

infra/ansible/playbooks/files/prometheus/rules.yml

@@ -36,8 +36,8 @@
     file:
       path: /etc/grafana/
       state: directory
-      owner: root
-      group: root
+      owner: grafana
+      group: grafana
       mode: '0755'
     vars:
       ansible_ssh_user: "{{ admin_user }}"
@@ -47,27 +47,33 @@
     template:
       src: grafana/grafana.ini.j2
       dest: /etc/grafana/grafana.ini
-      owner: root
-      group: root
+      owner: grafana
+      group: grafana
       mode: '0755'
     vars:
       ansible_ssh_user: "{{ admin_user }}"
-      grafana_http_port: "{{ lookup('ini', 'grafana_http_port', file=ini_file) }}"
+      grafana_domain: "{{ lookup('ini', 'grafana_domain', file=ini_file) }}"
+      grafana_oath_client_id: "{{ lookup('ini', 'grafana_oath_client_id', file=ini_file) }}"
+      grafana_oath_client_secret: "{{ lookup('ini', 'grafana_oath_client_secret', file=ini_file) }}"
+      grafana_oath_auth_url: "{{ lookup('ini', 'grafana_oath_auth_url', file=ini_file) }}"
+      grafana_oath_token_url: "{{ lookup('ini', 'grafana_oath_token_url', file=ini_file) }}"
+      grafana_oath_api_url: "{{ lookup('ini', 'grafana_oath_api_url', file=ini_file) }}"
 
   - name: Clone Aligned repository
     git:
       repo: https://github.com/yetanotherco/aligned_layer.git
       dest: /home/{{ ansible_user }}/repos/telemetry/aligned_layer
-      version: v0.10.2
+      #version: v0.12.0
+      version: staging
       recursive: false
 
   - name: Ensure /etc/grafana/provisioning directory exists
     become: true
     file:
       path: /etc/grafana/provisioning/
       state: directory
-      owner: root
-      group: root
+      owner: grafana
+      group: grafana
       mode: '0755'
     vars:
       ansible_ssh_user: "{{ admin_user }}"
@@ -77,8 +83,8 @@
     copy:
       src: /home/{{ ansible_user }}/repos/telemetry/aligned_layer/grafana/provisioning/
       dest: /etc/grafana/provisioning/
-      owner: root
-      group: root
+      owner: grafana
+      group: grafana
       mode: '0755'
       remote_src: yes
     vars:
@@ -89,11 +95,43 @@
     template:
       src: grafana/datasource.yaml.j2
       dest: "/etc/grafana/provisioning/datasources/datasource.yaml"
+      owner: grafana
+      group: grafana
       mode: '0755'
     vars:
       ansible_ssh_user: "{{ admin_user }}"
       grafana_prometheus_datasource: "{{ lookup('ini', 'grafana_prometheus_datasource', file=ini_file) }}"
 
+  - name: Change admin password for grafana
+    shell: 
+      cmd: sudo grafana-cli admin reset-admin-password {{ lookup('ini', 'grafana_admin_password', file=ini_file) }}
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
+
+  - name: Ensure /etc/grafana/ directory is owned by user grafana
+    become: true
+    file:
+      path: /etc/grafana/
+      recurse: true
+      state: directory
+      owner: grafana
+      group: grafana
+      mode: '0755'
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
+
+  - name: Ensure /var/lib/grafana/ directory is owned by user grafana
+    become: true
+    file:
+      path: /var/lib/grafana/
+      recurse: true
+      state: directory
+      owner: grafana
+      group: grafana
+      mode: '0755'
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
+
   - name: Restart Grafana
     become: true
     service:

infra/ansible/playbooks/grafana.yaml

@@ -3,8 +3,14 @@ caddy_metrics_url=
 caddy_telemetry_url=
 caddy_jaeger_url=
 
+grafana_admin_password=
 grafana_prometheus_datasource=
-grafana_http_port=
+grafana_domain=
+grafana_oath_client_id=
+grafana_oath_client_secret=
+grafana_oath_auth_url=
+grafana_oath_token_url=
+grafana_oath_api_url=
 
 prometheus_aggregator_ip=
 prometheus_operator_ip=
@@ -15,10 +21,12 @@ postgresql_telemetry_db_name=
 postgresql_telemetry_user=
 postgresql_telemetry_pass=
 
-cassandra_telemetry_user=admidn
+cassandra_telemetry_user=
 cassandra_telemetry_pass=
 
 telemetry_aligned_rpc=
 telemetry_api_phx_host=
 telemetry_api_elixir_hostname=
 telemetry_api_secret_key_base=
+
+pagerduty_routing_key=

infra/ansible/playbooks/ini/config-telemetry.ini.example

@@ -24,7 +24,7 @@
     become: true
     template:
       src: open_telemetry/otel-collector.yaml
-      dest: /etc/otel-collector.yaml
+      dest: /etc/otelcol/config.yaml
     vars:
       ansible_ssh_user: "{{ admin_user }}"
 

infra/ansible/playbooks/open_telemetry.yaml

@@ -111,6 +111,17 @@
       prometheus_batcher_ip: "{{ lookup('ini', 'prometheus_batcher_ip', file='ini/config-telemetry.ini') }}"
       prometheus_tracker_ip: "{{ lookup('ini', 'prometheus_tracker_ip', file='ini/config-telemetry.ini') }}"
 
+  - name: Add prometheus rules file
+    become: true
+    copy:
+      src: prometheus/rules.yml
+      dest: /etc/prometheus/rules.yml
+      owner: prometheus
+      group: prometheus
+      mode: '0755'
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
+
   - name: Create Prometheus systemd service
     become: true
     template:

infra/ansible/playbooks/prometheus.yaml

@@ -16,6 +16,13 @@
     prometheus_version: "2.53.2"
     ini_file: ini/config-telemetry.ini
 
+- name: Run Alert Manager playbook
+  ansible.builtin.import_playbook: alert_manager.yaml
+  vars:
+    host: telemetry
+    alert_manager_version: 0.27.0
+    ini_file: ini/config-telemetry.ini
+
 - name: Run Grafana playbook
   ansible.builtin.import_playbook: grafana.yaml
   vars:
@@ -53,10 +60,23 @@
   vars:
     host: telemetry
 
+- name: Run Go playbook
+  ansible.builtin.import_playbook: go.yaml
+  vars:
+    host: telemetry
+
 - name: Setup Telemetry
   hosts: telemetry
 
   tasks:
+  - name: Make sure /etc/default/tailscaled exists
+    become: true
+    file:
+      path: /etc/default/tailscaled
+      state: touch
+    vars:
+      ansible_ssh_user: "{{ admin_user }}"
+
   - name: Add permit to tailscale for caddy
     become: true
     lineinfile:
@@ -73,9 +93,17 @@
     ansible.builtin.git:
       repo: https://github.com/yetanotherco/aligned_layer.git
       dest: /home/{{ ansible_user }}/repos/telemetry/aligned_layer
-      version: v0.10.2
+      #version: v0.12.0
+      version: staging
       recursive: false
 
+  - name: Run telemetry_compile_bls_verifier target
+    make:
+      target: telemetry_compile_bls_verifier
+      chdir: /home/{{ ansible_user }}/repos/telemetry/aligned_layer
+    environment:
+      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+
   - name: Add environment file for Telemetry API
     template:
       src: telemetry_api/telemetry_env.j2
@@ -102,14 +130,19 @@
 
   - name: Build release for Telemetry API
     shell:
-      cmd: |
-        source .env && mix release
+      cmd: source .env && mix release
       chdir: /home/{{ ansible_user }}/repos/telemetry/aligned_layer/telemetry_api
       executable: /bin/bash
       creates: /home/{{ ansible_user }}/repos/telemetry/aligned_layer/telemetry_api/_build/prod/rel/telemetry_api/bin/
     environment:
       MIX_ENV: prod
 
+  - name: Run migrations for Telemetry API
+    shell:
+      cmd: source .env && _build/prod/rel/telemetry_api/bin/migrate
+      chdir: /home/{{ ansible_user }}/repos/telemetry/aligned_layer/telemetry_api
+      executable: /bin/bash
+
   - name: Ensure ~/.config/systemd/user/ directory exists
     file:
       path: /home/{{ ansible_user }}/.config/systemd/user/