Telemetry ansible (#1428)

Co-authored-by: JuArce <[email protected]>

Author: samoht9277

Makefile

@@ -1268,6 +1268,15 @@ ansible_operator_deploy: ## Deploy the Operator. Parameters: INVENTORY
 		-e "ecdsa_keystore_path=$(ECDSA_KEYSTORE)" \
 		-e "bls_keystore_path=$(BLS_KEYSTORE)"
 
+ansible_telemetry_create_env:
+	@cp -n infra/ansible/playbooks/ini/config-telemetry.ini.example infra/ansible/playbooks/ini/config-telemetry.ini
+	@echo "Config files for Telemetry created in infra/ansible/playbooks/ini"
+	@echo "Please complete the values and run make ansible_telemetry_deploy"
+
+ansible_telemetry_deploy:
+	@ansible-playbook infra/ansible/playbooks/telemetry.yaml \
+		-i $(INVENTORY)
+
 __ETHEREUM_PACKAGE__:  ## ____
 
 ethereum_package_start: ## Starts the ethereum_package environment

infra/ansible/README.md

@@ -137,7 +137,6 @@ This will create the following file in `infra/ansible/playbooks/ini`:
 ```shell
 make ansible_aggregator_create_env
 ```
-
 - `config-aggregator.ini`
 
 The `config-aggregator.ini` contains the variables to run the Operator in Aligned:
@@ -166,6 +165,49 @@ make ansible_aggregator_deploy INVENTORY=</path/to/inventory> ECDSA_KEYSTORE=<pa
 > [!Note]
 > ECDSA_KEYSTORE and BLS_KEYSTORE are the paths of the keystores in your machine.
 
+## Telemetry
+To deploy the Telemetry Services you need to set some variables and then run the Telemetry playbook
+
+This will create the following file in `infra/ansible/playbooks/ini`:
+
+```shell
+make ansible_telemetry_create_env
+```
+
+- `config-telemetry.ini`
+
+The `config-telemetry.ini` contains the variables to run the Telemetry in Aligned:
+
+| Variable                      | Description                                       | Testnet                                       | Stage                                              | Mainnet
+|-------------------------------|---------------------------------------------------|-----------------------------------------------|----------------------------------------------------|------------------------------------|
+| grafana_admin_password        | Password for Grafana's admin user                 | <your_grafana_admin_password>                 | <your_grafana_admin_password>                      | <your_grafana_admin_password>
+| grafana_prometheus_datasource | Prometheus location for Grafana to pull data from | localhost                                     | localhost                                          | localhost
+| grafana_domain                | Public URL for Grafana Metrics                    | `holesky.metrics.alignedlayer.com`            | `stage1.metrics.alignedlayer.com`                  | `mainnet.metrics.alignedlayer.com`
+| grafana_oath_client_id        | Generic OAuth Client ID                           | <your_oauth_client_id>                        | <your_oauth_client_id>                             | <your_oauth_client_id>
+| grafana_oath_client_secret    | Generic OAuth Client Secret                       | <your_oauth_client_secret>                    | <your_oauth_client_secret>                         | <your_oauth_client_secret>
+| grafana_oath_auth_url         | Generic OAuth Auth URL                            | <your_oauth_auth_url>                         | <your_oauth_auth_url>                              | <your_oauth_auth_url>        
+| grafana_oath_token_url        | Generic OAuth Token URL                           | <your_oauth_token_url>                        | <your_oauth_token_url>                             | <your_oauth_token_url>
+| grafana_oath_api_url          | Generic OAuth API URL (User Info)                 | <your_oauth_api_url>                          | <your_oauth_api_url>                               | <your_oauth_api_url>
+| prometheus_aggregator_ip      | Aggregator IP+PORT location for prometheus        | aligned-holesky-aggregator:9091               | aligned-holesky-stage-1-aggregator:9091            | aligned-mainnet-aggregator:9091
+| prometheus_operator_ip        | Operators IP+PORT location for prometheus         | aligned-holesky-operator-1:9092               | aligned-holesky-stage-1-operator-1:9092            | aligned-mainnet-operator-1:9092
+| prometheus_batcher_ip         | Batcher IP+PORT location for prometheus           | aligned-holesky-batcher:9093                  | aligned-holesky-stage-1-batcher-1:9093             | aligned-mainnet-batcher:9093
+| prometheus_tracker_ip         | Tracker IP+PORT location for prometheus           | aligned-holesky-tracker:4001                  | aligned-holesky-stage-1-tracker:4001               | aligned-mainnet-telemetry:4001
+| postgresql_telemetry_db_name  | Postgres Database Name                            | <your_postgres_db_name>                       | <your_postgres_db_name>                            | <your_postgres_db_name>
+| postgresql_telemetry_user     | Postgres Username                                 | <your_postgres_db_username>                   | <your_postgres_db_username>                        | <your_postgres_db_username>
+| postgresql_telemetry_pass     | Postgres Username's Password                      | <your_postgres_db_password>                   | <your_postgres_db_password>                        | <your_postgres_db_password>
+| cassandra_telemetry_user      | Cassandra Username                                | <your_cassandra_db_name>                      | <your_cassandra_db_name>                           | <your_cassandra_db_name>
+| cassandra_telemetry_pass      | Cassandra Username's Password                     | <your_cassandra_db_password>                  | <your_cassandra_db_password>                       | <your_cassandra_db_password>
+| telemetry_aligned_rpc         | HTTP ETH RPC URL                                  | `https://aligned-rpc-node-2.tail665ae.ts.net` | `https://aligned-rpc-node-2.tail665ae.ts.net`      | `https://aligned-mainnet-rpc-1.tail665ae.ts.net`
+| telemetry_api_phx_host        | Telemetry API Elixir's Phoenix host for           | `holesky.telemetry.alignedlayer.com`          | `stage1.telemetry.alignedlayer.com`                | `mainnet.metrics.alignedlayer.com`
+| telemetry_api_elixir_hostname | Telemetry API Server's Hostname                   | `aligned-holesky-metrics`                     | `aligned-holesky-stage-1-telemetry`                | `aligned-mainnet-telemetry`
+| telemetry_api_secret_key_base | Telemetry API Secret Key Base                     | <your_secret_key_base>                        | <your_secret_key_base>                             | <your_secret_key_base>
+| pagerduty_routing_key         | Pagerduty Routing Key                             | <your_pagerduty_routing_key>                  | <your_pagerduty_routing_key>                       | <your_pagerduty_routing_key>
+
+Deploy the Telemetry Services:
+
+```shell
+make ansible_telemetry_deploy INVENTORY=</path/to/inventory> 
+
 # How to Create Keystores
 
 ## Create ECDSA Keystore
@@ -266,3 +308,4 @@ Public Key: E([...,...])
 
 Refer to this link for more details about keystore creation https://docs.eigenlayer.xyz/eigenlayer/operator-guides/operator-installation#create-keys
 
+

infra/ansible/aggregator.yaml

@@ -0,0 +1,60 @@
+- name: Install Alert Manager
+  hosts: "{{ host }}"
+
+  tasks:
+    - name: Install Alert Manager
+      get_url:
+        url: https://github.com/prometheus/alertmanager/releases/download/v{{ alert_manager_version }}/alertmanager-{{ alert_manager_version }}.linux-amd64.tar.gz
+        dest: /tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz
+        mode: '0644'
+
+    - name: Install Alert Manager package
+      become: true
+      unarchive:
+        src: "/tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz"
+        dest: "/usr/local/bin/"
+        remote_src: yes
+        extra_opts:
+        - --strip-components=1
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Clean up Alert Manager tarball package
+      file:
+        path: /tmp/alert_manager-{{ alert_manager_version }}-linux-amd64.tar.gz
+        state: absent
+
+    - name: Make sure /etc/alertmanager directory exists
+      become: true
+      file:
+        path: /etc/alertmanager
+        state: directory
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Create Alert Manager config file
+      become: true
+      template:
+        src: alert_manager/alertmanager.yml.j2
+        dest: /etc/alertmanager/alertmanager.yml
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+        pagerduty_routing_key: "{{ lookup('ini', 'pagerduty_routing_key', file=ini_file) }}"
+
+    - name: Create Alert Manager systemd service
+      become: true
+      template:
+        src: services/alert_manager.service.j2
+        dest: /etc/systemd/system/alert_manager.service
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Start Alert Manager
+      become: true
+      systemd_service:
+        name: alert_manager
+        state: started
+        daemon_reload: true
+        enabled: true
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"

infra/ansible/playbooks/alert_manager.yaml

@@ -0,0 +1,73 @@
+- name: Cassandra Setup
+  hosts: "{{ host }}"
+  become: true
+  vars:
+    ansible_ssh_user: "{{ admin_user }}"
+
+  tasks:
+  - name: Check if cassandra is installed
+    stat:
+      path: /opt/cassandra/bin/cassandra
+    register: cassandra_exists
+
+  - name: Install java
+    apt:
+      pkg: openjdk-17-jre
+
+  - name: Create cassandra group
+    group:
+      name: cassandra
+      system: yes
+
+  - name: Create cassandra user
+    user:
+      name: cassandra
+      group: cassandra
+      shell: /sbin/nologin
+      system: yes
+
+  - name: Download cassandra
+    when: not cassandra_exists.stat.exists
+    get_url:
+      url: "https://dlcdn.apache.org/cassandra/{{ cassandra_version }}/apache-cassandra-{{ cassandra_version }}-bin.tar.gz "
+      dest: "/tmp/cassandra-{{ cassandra_version }}.tar.gz"
+      mode: '0644'
+
+  - name: Ensure /opt/cassandra directory exists
+    file:
+      path: /opt/cassandra
+      owner: "cassandra"
+      group: "cassandra"
+      state: directory
+
+  - name: Extract cassandra
+    when: not cassandra_exists.stat.exists
+    unarchive:
+      src: "/tmp/cassandra-{{ cassandra_version }}.tar.gz"
+      dest: /opt/cassandra/
+      remote_src: yes
+      owner: "cassandra"
+      group: "cassandra"
+      extra_opts:
+        - --strip-components=1
+
+  - name: Add /opt/cassandra/bin to PATH
+    lineinfile:
+      path: /home/{{ ansible_user }}/.bashrc
+      line: PATH=/opt/cassandra/bin:$PATH
+      state: present
+
+  - name: Create Cassandra systemd service
+    template:
+      src: services/cassandra.service.j2
+      dest: /etc/systemd/system/cassandra.service
+    vars:
+      cassandra_telemetry_user: "{{ lookup('ini', 'cassandra_telemetry_user', file=ini_file) }}"
+      cassandra_telemetry_pass: "{{ lookup('ini', 'cassandra_telemetry_pass', file=ini_file) }}"
+
+  - name: Start Cassandra
+    systemd_service:
+      name: cassandra
+      state: started
+      daemon_reload: true
+      enabled: true