diff --git a/COLLECTOR_VERSION b/COLLECTOR_VERSION index ba44a52..def571d 100644 --- a/COLLECTOR_VERSION +++ b/COLLECTOR_VERSION @@ -1 +1 @@ -v0.9.4 \ No newline at end of file +v0.9.5 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b006756..58335b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,4 +97,4 @@ concatenate all commit messages right before you hit the "Confirm squash and mer button. Maintainers must make sure to edit this concatenated message to make it right before merging. In some cases, if the commit messages are lacking the easiest approach to have at least something useful is copy/pasting the PR description into the commit message box -before merging (but see the above paragraph about writing good commit messages in the first place). +before merging (but see the above paragraph about writing good commit messages in the first place). \ No newline at end of file diff --git a/README.md b/README.md index ee24ae1..4f43771 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ configuration, troubleshooting (REST changes) info, etc. > See the [F5 Application Study Tool Labs](https://clouddocs.f5.com/training/community/ast/html/) for an educational guided lab experience. -The Application Study Tool is intended to provide enhanced insights into (classic) BIG-IP products, leveraging best in class +The F5 Application Study Tool is intended to provide enhanced insights into (classic) BIG-IP products, leveraging best in class open source telemetry tools. The full installation includes: * Custom Instance of OpenTelemetry Collector with enhanced BIG-IP data receivers (data fetched via iControlRest) [Full List of Metrics Collected](pages/components/otel_collector/receiver_metrics.md). @@ -173,6 +173,8 @@ Create a file called .env.device-secrets, and add your BIP passwords like so: BIGIP_PASSWORD_1=foo-bar123! BIGIP_PASSWORD_2=bar-foo123! ``` +> **Note:** Ensure that the permissions on the (_.env.device-secrets_) file are restricted to allow read access only to the user running the Docker containers. +> This ensures that credential information remains protected from unauthorized access. The variable name (the part on the left of the equal sign) must match the configured value for the devices that use this password in config/ast_defaults.yaml or device specific @@ -348,7 +350,7 @@ cp .env-example .env ``` ### Run Application Study Tool -Once the above configurations have been made, the tool can be started with: +After the above configurations have been made, start the tool with: ```shell # `docker compose up -d` to start in background mode @@ -357,7 +359,7 @@ docker compose up #### View The Dashboards The default Grafana user/pass is `admin/admin`, and can be accessed at -`http://:3000`. +`http://:3000`. If HTTPS is configured, use `https://:3001`. ## Updating AST Versions @@ -371,7 +373,7 @@ special instructions / breaking changes. git stash git fetch --tags git pull origin main -git checkout tags/RELEASE_VERSION #(e.g. tags/v0.9.4) +git checkout tags/RELEASE_VERSION #(e.g. tags/v0.9.5) git stash pop # # @@ -379,7 +381,6 @@ docker compose down # `docker compose up -d` to start in background mode docker compose up ``` - ## Support For support, please open a GitHub issue. Note, the code in this repository is community supported and is not supported by F5 Networks. For a complete list of supported projects please reference [SUPPORT.md](SUPPORT.md). @@ -394,7 +395,7 @@ Please refer to the [F5 DevCentral Community Code of Conduct](code_of_conduct.md ## Copyright -Copyright 2014-2024 F5 Networks Inc. +Copyright 2014-2025 F5 Networks Inc. ### F5 Networks Contributor License Agreement @@ -406,4 +407,4 @@ Otherwise by submitting a CLA you represent that you are legally entitled to gra If your employer has rights to intellectual property that you create, such as your contributions, you represent that you have received permission to make contributions on behalf of that employer, that your employer has waived such rights for your contributions, or that your employer has executed a separate CLA with F5. If you are signing on behalf of a company, you represent that you are legally entitled to grant the license recited therein. -You represent further that each employee of the entity that submits contributions is authorized to submit such contributions on behalf of the entity pursuant to the CLA. +You represent further that each employee of the entity that submits contributions is authorized to submit such contributions on behalf of the entity pursuant to the CLA. \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 7001466..53b2bb4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,7 +6,8 @@ volumes: services: prometheus: - image: prom/prometheus:v2.54.1 + # prom/prometheus:v2.53.5 + image: prom/prometheus@sha256:7a34573f0b9c952286b33d537f233cd5b708e12263733aa646e50c33f598f16c container_name: prometheus restart: unless-stopped stop_grace_period: 5m @@ -21,13 +22,14 @@ services: - '--web.enable-lifecycle' - '--enable-feature=otlp-write-receiver' - '--storage.tsdb.retention.time=1y' - ports: - - 9090:9090 + expose: + - 9090 networks: - 7lc_network otel-collector: - image: ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.9.4 + # ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.9.5 + image: ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.9.5 restart: unless-stopped volumes: - ./services/otel_collector:/etc/otel-collector-config @@ -40,7 +42,8 @@ services: - 7lc_network grafana: - image: grafana/grafana:11.2.0 + # grafana/grafana:v11.6.3 + image: grafana/grafana@sha256:6128afd8174f01e39a78341cb457588f723bbb9c3b25c4d43c4b775881767069 container_name: grafana restart: unless-stopped ports: diff --git a/https_setup.md b/https_setup.md new file mode 100644 index 0000000..96a0bbf --- /dev/null +++ b/https_setup.md @@ -0,0 +1,46 @@ +### Enabling HTTPS for Grafana + +This section outlines the steps required to enable HTTPS for Grafana when deployed using Docker Compose. + +#### 1. Generate SSL Certificate and Key + +To enable HTTPS, you need a certificate `(cert.pem)` and a private key `(key.pem)`. For local development, you can generate self-signed certificates using OpenSSL with the following commands: + +```sh +mkdir -p ./services/grafana/ssl +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout ./services/grafana/ssl/key.pem -out ./services/grafana/ssl/cert.pem \ + -subj "/CN=localhost" +``` + +> **Note:** In production environments, always use certificates from a trusted Certificate Authority (CA). +> It is recommended to rotate these certificates regularly before they expire to minimize the risk of security breaches. + +> **Reference:** For more detailed guidance on configuring HTTPS, refer to the [official Grafana documentation](https://grafana.com/docs/grafana/latest/setup-grafana/set-up-https/). + +#### 2. Modify the Docker Compose Configuration + +Update your `docker-compose.yaml` file with the necessary configurations to enable HTTPS for Grafana. Below is an example snippet for the Grafana service: + +```yaml +grafana: + image: grafana/grafana:11.6.3 + container_name: grafana + restart: unless-stopped + ports: + - 3000:3000 + - 3001:3001 # HTTPS port + volumes: + - grafana:/var/lib/grafana + - ./services/grafana/provisioning/:/etc/grafana/provisioning + - ./services/grafana/ssl/cert.pem:/etc/grafana/cert.pem:ro + - ./services/grafana/ssl/key.pem:/etc/grafana/key.pem:ro + env_file: ".env" + environment: + - GF_SERVER_PROTOCOL=https + - GF_SERVER_CERT_FILE=/etc/grafana/cert.pem + - GF_SERVER_CERT_KEY=/etc/grafana/key.pem + - GF_SERVER_HTTP_PORT=3001 +``` + +By following these steps, you will successfully enable HTTPS for your Grafana deployment. Ensure you test your configuration in both development and production environments to verify functionality and security compliance. \ No newline at end of file diff --git a/pages/Gemfile b/pages/Gemfile index a736a29..9cfd417 100644 --- a/pages/Gemfile +++ b/pages/Gemfile @@ -32,8 +32,12 @@ gem "wdm", "~> 0.1", :platforms => [:mingw, :x64_mingw, :mswin] # do not have a Java counterpart. gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] -gem 'nokogiri' -gem 'rack', '~> 2.2.4' +gem 'nokogiri', '>= 1.18.8' +gem 'rack', '~> 2.2.14' gem 'rspec' +gem 'google-protobuf', '>= 4.28.2' +gem 'webrick', '>= 1.8.2' +gem 'rexml', '>= 3.3.9' + gem "html-proofer", "~> 5.0", :group => :development \ No newline at end of file diff --git a/pages/Gemfile.lock b/pages/Gemfile.lock index 9dcee98..61696a6 100644 --- a/pages/Gemfile.lock +++ b/pages/Gemfile.lock @@ -30,10 +30,10 @@ GEM fiber-storage fiber-storage (1.0.0) forwardable-extended (2.6.0) - google-protobuf (4.28.1-arm64-darwin) + google-protobuf (4.31.1-arm64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.28.1-x86_64-linux) + google-protobuf (4.31.1-x86_64-linux-gnu) bigdecimal rake (>= 13) hashery (2.1.2) @@ -89,9 +89,9 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) - nokogiri (1.16.7-arm64-darwin) + nokogiri (1.18.8-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.18.8-x86_64-linux-gnu) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -103,13 +103,13 @@ GEM ttfunk public_suffix (6.0.1) racc (1.8.1) - rack (2.2.10) + rack (2.2.17) rainbow (3.1.1) rake (13.2.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - rexml (3.3.7) + rexml (3.4.1) rouge (4.3.0) rspec (3.13.0) rspec-core (~> 3.13.0) @@ -137,7 +137,7 @@ GEM typhoeus (1.4.1) ethon (>= 0.9.0) unicode-display_width (2.6.0) - webrick (1.8.1) + webrick (1.9.1) yell (2.2.2) zeitwerk (2.7.1) @@ -146,16 +146,19 @@ PLATFORMS x86_64-linux-gnu DEPENDENCIES + google-protobuf (>= 4.28.2) html-proofer (~> 5.0) http_parser.rb (~> 0.6.0) jekyll (~> 4.3.4) just-the-docs (= 0.10.0) - nokogiri - rack (~> 2.2.4) + nokogiri (>= 1.18.8) + rack (~> 2.2.14) + rexml (>= 3.3.9) rspec tzinfo (>= 1, < 3) tzinfo-data wdm (~> 0.1) + webrick (>= 1.8.2) BUNDLED WITH 2.5.9 diff --git a/pages/troubleshooting/troubleshooting.md b/pages/troubleshooting/troubleshooting.md index 07748cb..a9c4aba 100644 --- a/pages/troubleshooting/troubleshooting.md +++ b/pages/troubleshooting/troubleshooting.md @@ -65,9 +65,9 @@ This output shows everything looks good: ```shell $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -cb4cf8867390 grafana/grafana:11.2.0 "/run.sh" About a minute ago Up 49 seconds 0.0.0.0:3000->3000/tcp grafana -bb8891f2cd47 prom/prometheus:v2.54.1 "/bin/prometheus --c…" About a minute ago Up 49 seconds 0.0.0.0:9090->9090/tcp prometheus -df2739cd67cb ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.6.0 "/otelcol-custom --c…" About a minute ago Up 49 seconds 4317/tcp, 55679-55680/tcp application-study-tool-otel-collector-1 +cb4cf8867390 grafana/grafana:11.6.3 "/run.sh" About a minute ago Up 49 seconds 0.0.0.0:3000->3000/tcp grafana +bb8891f2cd47 prom/prometheus:v2.53.5 "/bin/prometheus --c…" About a minute ago Up 49 seconds 0.0.0.0:9090->9090/tcp prometheus +df2739cd67cb ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.9.5 "/otelcol-custom --c…" About a minute ago Up 49 seconds 4317/tcp, 55679-55680/tcp application-study-tool-otel-collector-1 ``` This output shows a problem (Restarting container) for the `application-study-tool-otel-collector-1` @@ -75,9 +75,9 @@ container (the otel collector): ```shell $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -fdbde8a3ee16 ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.6.0 "/otelcol-custom --c…" 14 seconds ago Restarting (1) 5 seconds ago application-study-tool-otel-collector-1 -b7ef41accd46 grafana/grafana:11.2.0 "/run.sh" 14 seconds ago Up 13 seconds 0.0.0.0:3000->3000/tcp grafana -8edff3e8666e prom/prometheus:v2.54.1 "/bin/prometheus --c…" 14 seconds ago Up 13 seconds 0.0.0.0:9090->9090/tcp prometheus +fdbde8a3ee16 ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.9.5 "/otelcol-custom --c…" 14 seconds ago Restarting (1) 5 seconds ago application-study-tool-otel-collector-1 +b7ef41accd46 grafana/grafana:11.6.3 "/run.sh" 14 seconds ago Up 13 seconds 0.0.0.0:3000->3000/tcp grafana +8edff3e8666e prom/prometheus:v2.53.5 "/bin/prometheus --c…" 14 seconds ago Up 13 seconds 0.0.0.0:9090->9090/tcp prometheus ``` ### View Docker Container Logs diff --git a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json index 74b8e1a..b90d97a 100644 --- a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json +++ b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json @@ -1178,7 +1178,7 @@ "exemplar": false, "expr": "f5_system_memory_used_bytes{job=\"$device_name\", state=\"used\"} / ignoring(state) f5_system_memory_total_bytes{job=\"$device_name\"}", "instant": true, - "legendFormat": "__auto", + "legendFormat": "{{instance}}", "range": false, "refId": "A" } diff --git a/services/grafana/provisioning/dashboards/otel-collector/collector-health.json b/services/grafana/provisioning/dashboards/otel-collector/collector-health.json index 472dca3..f48e5b6 100644 --- a/services/grafana/provisioning/dashboards/otel-collector/collector-health.json +++ b/services/grafana/provisioning/dashboards/otel-collector/collector-health.json @@ -223,7 +223,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_accepted_metric_points${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "expr": "sum(${metric:value}(otelcol_receiver_accepted_metric_points_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -238,7 +238,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_refused_metric_points${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "expr": "sum(${metric:value}(otelcol_receiver_refused_metric_points_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -369,7 +369,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_sent_metric_points${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "expr": "sum(${metric:value}(otelcol_exporter_sent_metric_points_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -400,7 +400,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_send_failed_metric_points${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_metric_points_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -814,7 +814,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_batch_size_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_sum{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "instant": false, @@ -830,7 +830,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send_total{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "instant": false, @@ -1387,7 +1387,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "max(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "expr": "max(otelcol_process_memory_rss_bytes{job=\"$job\"}) by (job $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -1403,7 +1403,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "avg(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "expr": "avg(otelcol_process_memory_rss_bytes{job=\"$job\"}) by (job $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -1418,7 +1418,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "min(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "expr": "min(otelcol_process_memory_rss_bytes{job=\"$job\"}) by (job $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -2016,7 +2016,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "max(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "expr": "max(rate(otelcol_process_cpu${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -2032,7 +2032,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "avg(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "expr": "avg(rate(otelcol_process_cpu${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -2048,7 +2048,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "min(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "expr": "min(rate(otelcol_process_cpu${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -2158,7 +2158,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "count(count(otelcol_process_cpu_seconds${suffix}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id))", + "expr": "count(count(otelcol_process_cpu${suffix}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id))", "format": "time_series", "hide": false, "interval": "$minstep", @@ -3743,7 +3743,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(otelcol_process_memory_rss,job)", + "definition": "label_values(otelcol_process_memory_rss_bytes,job)", "hide": 0, "includeAll": false, "label": "Job", @@ -3752,7 +3752,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(otelcol_process_memory_rss,job)", + "query": "label_values(otelcol_process_memory_rss_bytes,job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1,