diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9a90286 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +--- +name: CI + +on: + push: + branches: [ "main" , "develop" ] + tags: + - 'v*' + pull_request: + branches: [ "main" ] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + lint: + name: Lint Go Code + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + cache: true + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + args: --timeout=5m + + build-binary: + name: Build Binary for Linux AMD64 + runs-on: ubuntu-latest + needs: lint + if: startsWith(github.ref, 'refs/tags/v') + + permissions: + contents: write + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify tag is on main or develop branch + run: | + TAG_NAME="${GITHUB_REF#refs/tags/}" + if ! git branch -r --contains "$TAG_NAME" | grep -qE "origin/(main|develop)"; then + echo "Error: Tag $TAG_NAME is not on main or develop branch" + exit 1 + fi + BRANCH=$(git branch -r --contains "$TAG_NAME" | grep -E "origin/(main|develop)" | head -1 | sed 's|origin/||' | tr -d ' ') + echo "Tag $TAG_NAME verified on $BRANCH branch" + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + cache: true + + - name: Get version from tag + id: version + run: | + VERSION="${GITHUB_REF#refs/tags/}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "Tag version: ${VERSION}" + + - name: Build binary + run: | + GOOS=linux GOARCH=amd64 go build \ + -ldflags="-s -w -X 'apatit/internal/version.Version=${{ steps.version.outputs.version }}'" \ + -o apatit-linux-amd64 \ + ./cmd/apatit + + - name: Create GitHub Release and Upload Binary + uses: softprops/action-gh-release@v1 + with: + files: | + apatit-linux-amd64 + locations.json + generate_release_notes: true + draft: false + prerelease: ${{ contains(github.ref, '-') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-docker-image: + name: Build and Push Docker Image + runs-on: ubuntu-latest + needs: [lint, build-binary] + if: startsWith(github.ref, 'refs/tags/v') + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify tag is on main or develop branch + run: | + TAG_NAME="${GITHUB_REF#refs/tags/}" + if ! git branch -r --contains "$TAG_NAME" | grep -qE "origin/(main|develop)"; then + echo "Error: Tag $TAG_NAME is not on main or develop branch" + exit 1 + fi + BRANCH=$(git branch -r --contains "$TAG_NAME" | grep -E "origin/(main|develop)" | head -1 | sed 's|origin/||' | tr -d ' ') + echo "Tag $TAG_NAME verified on $BRANCH branch" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..706fd07 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3143d79 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [v1.0.0] - 2025-12-03 + +### Added +- Prometheus metrics exporter for Ping-Admin monitoring service +- HTTP server with `/metrics` endpoint for Prometheus scraping +- JSON stats API endpoints (`/stats?type=task` and `/stats?type=all`) +- Support for multiple task monitoring with concurrent processing +- Automatic metrics collection with configurable refresh intervals +- Location name translation via `locations.json` file +- Automatic cleanup of stale metrics when monitoring points are removed +- Comprehensive Prometheus metrics: + - Exporter metrics (service info, refresh intervals, loops, errors) + - Monitoring point metrics (status, connection time, DNS lookup, server processing, total duration, speed, timestamps, staleness) +- Configuration via command-line flags and environment variables +- Docker image support with multi-stage build +- Graceful shutdown handling with signal support +- Request retry mechanism with configurable retry count +- Randomized request delays to prevent API throttling +- Rate limiting with configurable maximum requests per second (default: 2 requests/second) +- Staleness detection and reporting for monitoring points +- Support for English MP name translation +- Docker Compose configuration for easy deployment +- CI/CD workflow with linting and building diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b12e111 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:1.24-alpine AS build +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +ARG VERSION="v1.0.0" +RUN go build -v -ldflags="-s -w -X 'apatit/internal/version.Version=${VERSION}'" -o /app/apatit ./cmd/apatit + +FROM alpine:3.21 +ARG VERSION="latest" + +RUN apk add --no-cache ca-certificates + +LABEL org.opencontainers.image.title="APATIT (Advanced Ping-Admin Tasks Indicators Transducer)" +LABEL org.opencontainers.image.description="Transducer for Tasks Indicators from https://ping-admin.com/" +LABEL org.opencontainers.image.source="https://github.com/ostrovok-tech/apatit" +LABEL org.opencontainers.image.version="${VERSION}" + +COPY --from=build /app/apatit /usr/local/bin/apatit +COPY locations.json /app/locations.json +WORKDIR /app + +RUN addgroup -S appgroup && adduser -S appuser -G appgroup + +USER appuser +EXPOSE 8080 +ENV LISTEN_ADDRESS=:8080 + +ENTRYPOINT [ "apatit" ] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f3ccb3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Ostrovok! Tech + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 8636c67..330f67e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,192 @@ -# ping-admin-exporter +
+# APATIT +## Advanced Ping-Admin Task Indicators Transducer + +
+ +
+ + +
+ +--- + +
+ +## About + +**APATIT** is a set of exporters for the Website and Server Monitoring Service [Ping-Admin.com](https://ping-admin.com/?lang=en). **APATIT** collects, processes and publishes the tasks monitoring metrics and statistics. + +## Features + +- 🔄 **Automatic Metrics Collection**: Periodically fetches metrics from Ping-Admin API for multiple tasks +- 📊 **Prometheus Integration**: Exposes metrics in standard Prometheus format at `/metrics` +- 📈 **JSON Stats API**: Provides additional JSON endpoints for task statistics +- 🌍 **Location Translation**: Supports translation of location names via `locations.json` +- 🚀 **Concurrent Processing**: Efficiently processes multiple tasks in parallel +- 🔁 **Automatic Cleanup**: Removes stale metrics when monitoring points are no longer available +- 🐳 **Docker Support**: Ready-to-use Docker image + +## Installation + +### Using Docker + +```bash +docker run --rm -d \ + --name apatit \ + -p 8080:8080 \ + -e API_KEY=your-api-key \ + -e TASK_IDS=1,2,3 \ + ghcr.io/ostrovok-tech/apatit:latest +``` + +### From Source + +1. Clone the repository: +```bash +git clone https://github.com/ostrovok-tech/apatit.git +cd apatit +``` + +2. Build the binary: +```bash +go build -o apatit ./cmd/apatit +``` + +3. Run the exporter: +```bash +./apatit --api-key=your-api-key --task-ids=1,2,3 +``` + +## Configuration + +The exporter can be configured via command-line flags or environment variables. + +### Required Parameters + +| Flag | Environment Variable | Description | Default | +|------|---------------------|-------------|---------| +| `--api-key` | `API_KEY` | Ping-Admin API key | *required* | +| `--task-ids` | `TASK_IDS` | Comma-separated list of task IDs | *required* | + +### Optional Parameters + +| Flag | Environment Variable | Description | Default | +|------|---------------------|-------------|---------| +| `--listen-address` | `LISTEN_ADDRESS` | HTTP server listen address | `:8080` | +| `--log-level` | `LOG_LEVEL` | Log level (debug, info, warn, error) | `info` | +| `--locations-file` | `LOCATIONS_FILE` | Path to locations.json file | `locations.json` | +| `--eng-mp-names` | `ENG_MP_NAMES` | Translate MP names to English | `true` | +| `--refresh-interval` | `REFRESH_INTERVAL` | Metrics refresh interval | `3m` | +| `--api-update-delay` | `API_UPDATE_DELAY` | Ping-Admin API data update delay | `4m` | +| `--api-data-time-step` | `API_DATA_TIME_STEP` | Time between API data points | `3m` | +| `--max-allowed-staleness-steps` | `MAX_ALLOWED_STALENESS_STEPS` | Max staleness steps before marking MP as unavailable | `3` | +| `--max-requests-per-second` | `MAX_REQUESTS_PER_SECOND` | Maximum number of API requests allowed per second | `2` | +| `--request-delay` | `REQUEST_DELAY` | Minimum delay before API request (randomized) | `3s` | +| `--request-retries` | `REQUEST_RETRIES` | Maximum number of retries for API requests | `3` | + +### Example Configuration + +```bash +./apatit \ + --api-key=your-api-key \ + --task-ids=1,2,3 \ + --listen-address=:9090 \ + --refresh-interval=5m \ + --log-level=debug +``` + +Or using environment variables: + +```bash +export API_KEY=your-api-key +export TASK_IDS=1,2,3 +export REFRESH_INTERVAL=5m +export LOG_LEVEL=debug +./apatit +``` + +## Usage + +### HTTP Endpoints + +- **`/`** - Home page with links to metrics and stats +- **`/metrics`** - Prometheus metrics endpoint +- **`/stats?type=task`** - JSON endpoint for task statistics +- **`/stats?type=all`** - JSON endpoint for all tasks information + +### Prometheus Configuration + +Add the following to your `prometheus.yml`: + +```yaml +scrape_configs: + - job_name: 'apatit' + static_configs: + - targets: ['localhost:8080'] +``` + +## Metrics + +The exporter exposes the following Prometheus metrics: + +### Service Metrics + +- `apatit_service_info` - Information about the APATIT service (version, name, owner) + +### Exporter Metrics + +- `apatit_exporter_refresh_interval_seconds` - Configured refresh interval +- `apatit_exporter_max_allowed_staleness_steps` - Configured staleness threshold +- `apatit_exporter_refresh_duration_seconds{task_id, task_name}` - Duration of last refresh cycle +- `apatit_exporter_loops_total{exporter_type}` - Total number of refresh loops for each exporter type +- `apatit_exporter_errors_total{error_module, error_type, task_id, task_name}` - Total number of errors + +### Monitoring Point Metrics + +All MP metrics include labels: `task_id`, `task_name`, `mp_id`, `mp_name`, `mp_ip`, `mp_gps` + +- `apatit_mp_status` - Status of monitoring point (1 = up, 0 = down/stale) +- `apatit_mp_data_status` - Status of the data for the monitoring point (1 = has data, 0 = no data) +- `apatit_mp_connect_seconds` - Connection establishment time +- `apatit_mp_dns_lookup_seconds` - DNS lookup time +- `apatit_mp_server_processing_seconds` - Server processing time +- `apatit_mp_total_duration_seconds` - Total request duration +- `apatit_mp_speed_bytes_per_second` - Download speed +- `apatit_mp_last_success_timestamp_seconds` - Timestamp of last successful data point +- `apatit_mp_last_success_delta_seconds` - Time since last successful data point +- `apatit_mp_data_staleness_steps` - Number of missed API data steps (0 = fresh) + +## Project Structure + +``` +apatit/ +├── cmd/ +│ └── apatit/ +│ └── main.go # Application entry point +├── internal/ +│ ├── cache/ # Cache implementation +│ ├── client/ # Ping-Admin API client +│ ├── config/ # Configuration management +│ ├── exporter/ # Metrics and stats exporters logic +│ ├── log/ # Logging setup +│ ├── scheduler/ # Metrics and stats schedulers +│ ├── server/ # HTTP server +│ ├── translator/ # Location name translation +│ ├── utils/ # Utility functions +│ └── version/ # Version information +├── deploy/ +│ └── docker-compose.yaml # Docker Compose configuration +├── Dockerfile # Container image definition +├── locations.json # Location translation mappings +└── go.mod # Go module definition +``` + +## Support + +For issues and feature requests, please use the [GitHub Issues](https://github.com/ostrovok-tech/apatit/issues) page. + +
diff --git a/branding/banner.png b/branding/banner.png new file mode 100644 index 0000000..4b60829 Binary files /dev/null and b/branding/banner.png differ diff --git a/branding/logo.png b/branding/logo.png new file mode 100644 index 0000000..43f31c2 Binary files /dev/null and b/branding/logo.png differ diff --git a/cmd/apatit/main.go b/cmd/apatit/main.go new file mode 100644 index 0000000..0b5ddfd --- /dev/null +++ b/cmd/apatit/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "fmt" + "os/signal" + "syscall" + "time" + + "github.com/sirupsen/logrus" + + "apatit/internal/client" + "apatit/internal/config" + "apatit/internal/exporter" + "apatit/internal/log" + "apatit/internal/scheduler" + "apatit/internal/server" + "apatit/internal/translator" +) + +// createExporters creates and returns a list of exporters for the specified tasks. +func createExporters(apiClient *client.Client, cfg *config.Config) ([]*exporter.Exporter, error) { + exportersLog := logrus.WithField("component", "initializer") + exportersLog.Infof("Creating exporters for %d tasks...", len(cfg.TaskIDs)) + + exporters := make([]*exporter.Exporter, 0, len(cfg.TaskIDs)) + + // get account tasks + tasks, err := apiClient.GetAllTasks() + if err != nil { + return nil, fmt.Errorf("failed to get tasks metadata: %w", err) + } + + // get all available monitoring points + mps, err := apiClient.GetMPs() + if err != nil { + return nil, fmt.Errorf("failed to get MPs metadata: %w", err) + } + + // create exporter for each task + for _, taskID := range cfg.TaskIDs { + + expConfig := &exporter.Config{ + TaskID: taskID, + EngMPNames: cfg.EngMPNames, + ApiUpdateDelay: cfg.ApiUpdateDelay, + ApiDataTimeStep: cfg.ApiDataTimeStep, + } + + exp, err := exporter.New(expConfig, apiClient, tasks, mps) + if err != nil { + exportersLog.Errorf("Unable to create exporter for TaskID %d: %v", taskID, err) + continue + } + exporters = append(exporters, exp) + } + + if len(exporters) == 0 { + return nil, fmt.Errorf("no exporters were created, check task IDs and API key") + } + + exportersLog.Infof("Successfully created %d exporters.", len(exporters)) + return exporters, nil +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + cfg, err := config.New() + if err != nil { + logrus.Fatalf("Failed to load configuration: %v", err) + } + + app, err := newApp(cfg) + if err != nil { + logrus.Fatalf("Application initialization failed: %v", err) + } + + if err := app.Run(ctx); err != nil { + logrus.Fatalf("Application terminated with error: %v", err) + } +} + +type application struct { + cfg *config.Config + exporters []*exporter.Exporter + stop chan struct{} +} + +func newApp(cfg *config.Config) (*application, error) { + // Set logger for components + log.Init(cfg.LogLevel) + + // Set translator + if err := translator.Init(cfg.LocationsFilePath); err != nil { + logrus.Warnf("Failed to initialize translator, location names will not be translated: %v", err) + } + + // Create API client + apiClient := client.New(cfg.APIKey, nil, cfg.RequestDelay, cfg.RequestRetries, cfg.MaxRequestsPerSecond) + + // Register metrics, set ServiceInfo metric + exporter.RegisterMetrics() + exporter.AServiceInfo.Set(1) + + // Create Exporters for each task + exporters, err := createExporters(apiClient, cfg) + if err != nil { + return nil, fmt.Errorf("failed to create exporters: %w", err) + } + + return &application{ + cfg: cfg, + exporters: exporters, + stop: make(chan struct{}), + }, nil +} + +func (a *application) Run(ctx context.Context) error { + // Run HTTP server + go server.StartServer(a.cfg.ListenAddress) + + // Task Statistic Loop + go scheduler.RunStatsScheduler(a.exporters, a.cfg, a.stop) + + // Exporter Metrics Loop + go scheduler.RunMetricsScheduler(a.exporters, a.cfg, a.stop) + + logrus.Info("Exporters are running. Press Ctrl+C to exit.") + + // Stop goroutines once context is canceled + <-ctx.Done() + + logrus.Info("Shutdown signal received. Stopping schedulers...") + close(a.stop) + + time.Sleep(2 * time.Second) + + logrus.Info("Shutdown complete. Bye!") + return nil +} diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..35aabeb --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,11 @@ +### required +API_KEY=your_secret_api_key_here +TASK_IDS=1,2,3.. + +### optional +# ENG_MP_NAMES=true +# REFRESH_INTERVAL=3m +# REQUEST_PAUSE=2s +# LISTEN_ADDRESS=:8080 +# LOCATIONS_FILE=location.json +# LOG_LEVEL=info \ No newline at end of file diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml new file mode 100644 index 0000000..d24f025 --- /dev/null +++ b/deploy/docker-compose.yaml @@ -0,0 +1,22 @@ +version: '3.8' + +services: + apatit: + restart: unless-stopped + build: + context: .. + dockerfile: Dockerfile + args: + VERSION: "local-build" + + image: apatit:local + + ports: + - "8080:8080" + + env_file: + - .env + +networks: + default: + name: apatit-net \ No newline at end of file diff --git a/deploy/grafana/monitoring-dashboard.json b/deploy/grafana/monitoring-dashboard.json new file mode 100644 index 0000000..14fedf7 --- /dev/null +++ b/deploy/grafana/monitoring-dashboard.json @@ -0,0 +1,2811 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ping-Admin.com Tasks Monitoring Dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 2513, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [], + "title": "All Monitoring Tasks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of Monitoring Points with actual information.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count by (task_name) (apatit_mp_status{task_name=\"$task_name\"} == 1)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Available Locations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of problem tasks for each location. The point will be colored according to number problem resources.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": 0 + }, + { + "color": "light-orange", + "value": 3 + }, + { + "color": "semi-dark-orange", + "value": 5 + }, + { + "color": "#E24D42", + "value": 7 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 22, + "x": 2, + "y": 1 + }, + "id": 15, + "options": { + "basemap": { + "config": { + "server": "streets", + "showLabels": true, + "theme": "auto" + }, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": false, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": false + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "Number of Tasks", + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 10, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "mode": "auto" + }, + "name": "Problem Tasks", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lastOnly": false, + "lat": 35, + "layer": "Layer 1", + "lon": 5, + "shared": false, + "zoom": 2.5 + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name, mp_gps, mp_ip, task_name) (\n apatit_mp_data_staleness_steps{} * $refresh_interval_seconds / 60\n ) >= $max_actual_minutes", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Actual Problems GEOMap", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "regexp", + "keepTime": false, + "regExp": "/(?[0-9.\\-]+),\\s*(?[0-9.\\-]+)/", + "replace": false, + "source": "mp_gps" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "gps": true, + "mp_gps": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Last succes update", + "ip": "IP", + "latitude": "Latitude", + "longitude": "Longtitude", + "mp_ip": "IP", + "mp_name": "Monitoring Point", + "task_name": "Task Name", + "tm_name": "Monitoring Point" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "IP": { + "aggregations": [], + "operation": "groupby" + }, + "Last succes update": { + "aggregations": [ + "min", + "max" + ], + "operation": "aggregate" + }, + "Latitude": { + "aggregations": [], + "operation": "groupby" + }, + "Longtitude": { + "aggregations": [], + "operation": "groupby" + }, + "Monitoring Point": { + "aggregations": [ + "count" + ], + "operation": "groupby" + }, + "Task Name": { + "aggregations": [ + "count", + "allValues" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [] + }, + "latitude": { + "aggregations": [], + "operation": "groupby" + }, + "longitude": { + "aggregations": [], + "operation": "groupby" + }, + "tm_name": { + "aggregations": [] + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "IP": 0, + "Last succes update (max)": 7, + "Last succes update (median)": 8, + "Last succes update (min)": 6, + "Latitude": 2, + "Longtitude": 3, + "Monitoring Point": 1, + "Task Name (allValues)": 5, + "Task Name (count)": 4 + }, + "renameByName": { + "IP": "", + "Last succes update (max)": "Latest status update was (minutes ago)", + "Last succes update (mean)": "Last Success Update (MEAN)", + "Last succes update (median)": "Median", + "Last succes update (min)": "Most recent status update was (minutes ago)", + "Last succes update (sum)": "Last Success Update (SUM)", + "Task Name (allValues)": "Tasks", + "Task Name (count)": "Number of Tasks" + } + } + } + ], + "transparent": true, + "type": "geomap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Temporarily unavailable Monitoring Points according to Ping-Admin API.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 5 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (task_name) (apatit_mp_status{task_name=\"$task_name\"} == 0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Unavailable Locations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Monitoring Points with no data according to Ping-Admin API.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 9 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(count by (task_name) (apatit_mp_data_status{task_name=\"$task_name\"} == 0))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "No Data Locations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Intervals when location of any task was unavailable for more than $max_actual_minutes minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 50, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "1": { + "index": 0, + "text": "⁤" + } + }, + "type": "value" + } + ], + "noValue": "No problems", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "eq", + "reducer": "lastNotNull", + "value": 1 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 21, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "perPage": 20, + "rowHeight": 0.75, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(\n (sum by (mp_name, task_name) (\n apatit_mp_data_staleness_steps{}\n ) >= 3)\n /\n (sum by (mp_name, task_name) (\n apatit_mp_data_staleness_steps{}\n ) >= 3)\n)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{mp_name}} - {{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Problem Locations Timeline", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Intervals when task had more than 1 failed location (more than $max_actual_minutes minutes of absent data for location).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 50, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "spanNulls": false + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "No problems", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": 0 + }, + { + "color": "semi-dark-green", + "value": 2 + }, + { + "color": "semi-dark-yellow", + "value": 3 + }, + { + "color": "semi-dark-orange", + "value": 5 + }, + { + "color": "semi-dark-red", + "value": 10 + }, + { + "color": "dark-red", + "value": 20 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 22, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "perPage": 20, + "rowHeight": 0.75, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(\n (count by (task_name) (sum by (mp_name, task_name) (apatit_mp_data_staleness_steps{}) >= 3)) > 1\n)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Problem Tasks Timeline", + "type": "state-timeline" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 14, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 6 + }, + { + "color": "red", + "value": 9 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 9, + "options": { + "basemap": { + "config": { + "server": "streets", + "showLabels": true, + "theme": "auto" + }, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": false, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": false + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "Value", + "fixed": "dark-green" + }, + "opacity": 0.3, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 12, + "max": 100, + "min": 1 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "text": { + "fixed": "", + "mode": "field" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "mode": "auto" + }, + "name": "Last Success Status Update (minutes ago)", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lastOnly": false, + "lat": 50, + "layer": "Layer 1", + "lon": 30, + "shared": true, + "zoom": 3 + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name, mp_gps, mp_ip) (\n apatit_mp_data_staleness_steps{\n task_name=\"$task_name\", \n mp_name=~\"$location_name\"\n }\n ) * $refresh_interval_seconds / 60", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "regexp", + "keepTime": false, + "regExp": "/(?[0-9.\\-]+),\\s*(?[0-9.\\-]+)/", + "replace": false, + "source": "mp_gps" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "gps": true, + "mp_gps": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Last Succes Status Update (minutes ago)", + "ip": "IP", + "latitude": "Latitude", + "longitude": "Longtitude", + "mp_gps": "GPS", + "mp_ip": "IP", + "mp_name": "Monitoring Point", + "tm_name": "Monitoring Point" + } + } + } + ], + "type": "geomap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 11, + "options": { + "basemap": { + "config": { + "server": "streets", + "showLabels": true, + "theme": "auto" + }, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": false, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": false + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "Value", + "fixed": "dark-green" + }, + "opacity": 0.3, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 12, + "max": 100, + "min": 1 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "text": { + "fixed": "", + "mode": "field" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "mode": "auto" + }, + "name": "Request Time (seconds)", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lastOnly": false, + "lat": 50, + "layer": "Layer 1", + "lon": 30, + "shared": true, + "zoom": 3 + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name, mp_gps, mp_ip) (\n apatit_mp_total_duration_seconds{\n task_name=\"$task_name\",\n mp_name=~\"$location_name\"\n }\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "regexp", + "keepTime": false, + "regExp": "/(?[0-9.\\-]+),\\s*(?[0-9.\\-]+)/", + "replace": false, + "source": "mp_gps" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "gps": true, + "mp_gps": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Request Time (s)", + "ip": "IP", + "latitude": "Latitude", + "longitude": "Longtitude", + "mp_ip": "IP", + "mp_name": "Monitoring Point", + "tm_name": "Monitoring Point" + } + } + } + ], + "type": "geomap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The delay of last success response according to Ping Admin API update", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 437 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (\n apatit_mp_data_staleness_steps{\n task_name=\"$task_name\",\n mp_name=~\"$location_name\"\n }\n) * $refresh_interval_seconds / 60\n", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last Success Status Update (Minutes Ago)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The total time spent to receive a response from the site (and for tasks with the \"Ping\" type, the response time of the verified address is indicated here).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 437 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (\n apatit_mp_total_duration_seconds{\n task_name=\"$task_name\",\n mp_name=~\"$location_name\"\n }\n)", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Time (Seconds)", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + } + ], + "title": "Task GEOMaps", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 13, + "panels": [], + "title": "Task Monitoring", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The delay of last success response according to Ping Admin API update.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (\n apatit_mp_data_staleness_steps{\n task_name=\"$task_name\",\n mp_name=~\"$location_name\"\n }\n) * $refresh_interval_seconds / 60", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last Success Status Update (Minutes Ago)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The total time spent to receive a response from the site (and for tasks with the \"Ping\" type, the response time of the verified address is indicated here).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (\n apatit_mp_total_duration_seconds{\n task_name=\"$task_name\",\n mp_name=~\"$location_name\"\n }\n)", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Time | Total", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The time spent determining the IP address of the site.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (apatit_mp_dns_lookup_seconds{task_name=\"$task_name\", mp_name=~\"$location_name\"})", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Time | DNS", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The time spent connecting to the site.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (apatit_mp_connect_seconds{task_name=\"$task_name\", mp_name=~\"$location_name\"})", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Time | Connect", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The time spent waiting for the server response (between the connection and the start of data transfer).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (apatit_mp_server_processing_seconds{task_name=\"$task_name\", mp_name=~\"$location_name\"})", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Time | Server", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Monitoring Points with no data according to Ping-Admin API.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-blue", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 30, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "perPage": 20, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name) (apatit_mp_data_status{task_name=\"$task_name\"} == 0)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mp_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Lack of Data", + "type": "state-timeline" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Temporarily unavailable Monitoring Points according to Ping-Admin API.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-blue", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 192 + }, + "id": 26, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "perPage": 20, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name) (apatit_mp_status{} == 0)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mp_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Exporter | Unavailable Monitoring Points", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Available Monitoring Points.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 199 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count by (task_name) (apatit_mp_data_staleness_steps{task_name=\"$task_name\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Exporter | Number of Available Monitoring Points", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Exporter request time to Ping-Admin API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 207 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Min", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (task_name) (apatit_exporter_refresh_duration_seconds)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Exporter | API Request Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Will be zero if service is unavailable according to API data or it was responded more than 24h ago.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 214 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mp_name) (apatit_mp_status{mp_name=~\"$location_name\", task_name=\"$task_name\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Task | Monitoring Point Availability", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Request Speed according to Ping-Admin API response.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 221 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (mp_name) (apatit_mp_speed_bytes_per_second{task_name=\"$task_name\", mp_name=~\"$location_name\"})", + "instant": false, + "legendFormat": "{{tm_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Task | Request Speed", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Австралия, Сидней": "", + "Болгария, София": "", + "Вьетнам, Ханой": "", + "Канада, Монреаль": "", + "Кипр, Лимассол": "", + "Литва, Вильнюс": "", + "Россия, Москва, восток 1": "Russia, Moscow - East 1", + "Тайвань, Тайбэй": "" + } + } + } + ], + "type": "timeseries" + } + ], + "title": "Miscellaneous", + "type": "row" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { + "text": "VictoriaMetrics", + "value": "vbYsCp3nz" + }, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allowCustomValue": false, + "current": { + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(apatit_exporter_refresh_duration_seconds,task_name)", + "label": "Task", + "name": "task_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(apatit_exporter_refresh_duration_seconds,task_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "^${country_name},.*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(apatit_mp_total_duration_seconds{task_name=\"$task_name\"},mp_name)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(apatit_mp_total_duration_seconds{task_name=\"$task_name\"},mp_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(apatit_mp_connect_seconds{task_name=\"$task_name\"},mp_name)", + "includeAll": true, + "label": "Country", + "multi": true, + "name": "country_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(apatit_mp_connect_seconds{task_name=\"$task_name\"},mp_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/^([^,]+)/", + "sort": 1, + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "180", + "value": "180" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "query_result(apatit_exporter_refresh_interval_seconds)", + "description": "Refresh interval for Ping-Admin Exporter (configured as env variable for application)", + "hide": 2, + "label": "Refresh Interval", + "name": "refresh_interval_seconds", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(apatit_exporter_refresh_interval_seconds)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/(?(?<=\\s)\\d+(?=\\s\\d+$))/g", + "type": "query" + }, + { + "current": { + "text": "9", + "value": "9" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "query_result(apatit_exporter_refresh_interval_seconds/60*apatit_exporter_max_allowed_staleness_steps)", + "description": "Predefined actual data minutes.\n\nIt calculates from ENV variables:\n1) configured exporter refresh interval (in seconds);\n2) number of staleness steps (max allowed steps for actual data).\n\nFor example, if exporter runs every 180 seconds and allowable delay is 3 exporter runs (steps) then variable value is 9 minutes.", + "hide": 2, + "name": "max_actual_minutes", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(apatit_exporter_refresh_interval_seconds/60*apatit_exporter_max_allowed_staleness_steps)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/(?(?<=\\s)\\d+(?=\\s\\d+$))/g", + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Ping-Admin | World Wide Sites Monitoring", + "uid": "de4pre2zlmwowa", + "version": 4 +} \ No newline at end of file diff --git a/deploy/grafana/statistics-dashboard.json b/deploy/grafana/statistics-dashboard.json new file mode 100644 index 0000000..e34774f --- /dev/null +++ b/deploy/grafana/statistics-dashboard.json @@ -0,0 +1,554 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2466, + "links": [], + "panels": [ + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 293 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 69 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "STATUS" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "BLOCKING STATUS" + }, + "properties": [ + { + "id": "custom.width", + "value": 167 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VIRUS STATUS" + }, + "properties": [ + { + "id": "custom.width", + "value": 206 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "NAME" + }, + "properties": [ + { + "id": "custom.width", + "value": 289 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*STATUS/" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "applyToRow": false, + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "PROBLEM" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 1081 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "BLACKLIST STATUS" + }, + "properties": [ + { + "id": "custom.width", + "value": 209 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "ID" + } + ] + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "columns": [], + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "${datasource}" + }, + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET", + "params": [ + { + "key": "type", + "value": "all" + } + ] + } + } + ], + "title": "All Tasks Status", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "EnabledStatus": true, + "Timestamp": true, + "last_data": true, + "log_data": true, + "log_status": false, + "period": true, + "period_error": true, + "rk": true, + "rk_ip": true, + "rk_log_data": true, + "rrd": true, + "rss": true, + "sb": true, + "sb_log_data": true, + "status": true, + "tasks_ims_icq_list": true, + "tasks_ims_jabber_list": true, + "tasks_ims_skype_list": true, + "tasks_ims_telegram_list": true, + "tip": true, + "uptime_nw": true, + "uptime_w": true, + "uveddva": true + }, + "includeByName": {}, + "indexByName": { + "BlackListStatus": 3, + "EnabledStatus": 6, + "ID": 0, + "ServiceName": 1, + "TaskStatus": 2, + "URL": 5, + "VirusStatus": 4 + }, + "renameByName": { + "BlackListStatus": "BLACKLIST STATUS", + "EnabledStatus": "Enabled", + "ID": "", + "ServiceName": "NAME", + "TaskStatus": "STATUS", + "URL": "URL", + "VirusStatus": "VIRUS STATUS", + "log_status": "STATUS", + "name": "URL", + "nazv": "NAME", + "rk_log_status": "BLACKLIST STATUS", + "sb_log_status": "VIRUS STATUS", + "status": "", + "tid": "ID" + } + } + } + ], + "transparent": true, + "type": "table" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": true + }, + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Description" + }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "STATUS" + }, + "properties": [ + { + "id": "custom.width", + "value": 138 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "semi-dark-red", + "index": 1, + "text": "ERROR" + }, + "1": { + "color": "semi-dark-green", + "index": 0, + "text": "RESTORED" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TIMESTAMP (UTC)" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "LOCATION" + }, + "properties": [ + { + "id": "custom.width", + "value": 265 + } + ] + } + ] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 1, + "options": { + "cellHeight": "lg", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "columns": [], + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "${datasource}" + }, + "filterExpression": "", + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$[TaskName='${task_name}'].TaskLogs", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET", + "params": [ + { + "key": "type", + "value": "task" + } + ] + } + } + ], + "title": "Specified Task Events", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "MPID": true + }, + "includeByName": {}, + "indexByName": { + "Data": 0, + "Description": 3, + "MPID": 5, + "MPName": 2, + "Status": 1, + "Traceroute": 4 + }, + "renameByName": { + "Data": "TIMESTAMP (UTC)", + "Description": "DESCRIPTION", + "MPID": "", + "MPName": "LOCATION", + "Status": "STATUS", + "Traceroute": "TRACEROUTE" + } + } + } + ], + "transparent": true, + "type": "table" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ping-admin-exporter", + "value": "ff56j2xsn6pz4e" + }, + "name": "datasource", + "options": [], + "query": "yesoreyeram-infinity-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(apatit_exporter_refresh_duration_seconds,task_name)", + "label": "Task Name", + "name": "task_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(apatit_exporter_refresh_duration_seconds,task_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Ping-Admin | Tasks Status", + "uid": "90df5e88-2493-4b47-a1b0-6289d3fb3d1f", + "version": 1 + } \ No newline at end of file diff --git a/deploy/helm/.helmignore b/deploy/helm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deploy/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml new file mode 100644 index 0000000..9914d85 --- /dev/null +++ b/deploy/helm/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: apatit +description: Set of exporters for https://ping-admin.com/ +keywords: + - exporter + - prometheus + - ping-admin +sources: + - https://github.com/ostrovok-tech/apatit + - https://ping-admin.com/ +maintainers: + - name: Ostrovok! Tech + url: https://github.com/ostrovok-tech +type: application +version: 1.0.0 +appVersion: "1.0.0" diff --git a/deploy/helm/README.md b/deploy/helm/README.md new file mode 100644 index 0000000..d48f0d0 --- /dev/null +++ b/deploy/helm/README.md @@ -0,0 +1,336 @@ +# APATIT Helm Chart + +A Helm chart for deploying APATIT, a set of Prometheus exporters for [ping-admin.com](https://ping-admin.com/). + +## Description + +APATIT is a Prometheus exporter that collects metrics from ping-admin.com monitoring service. This Helm chart provides a complete Kubernetes deployment configuration with security best practices, including non-root execution, read-only filesystem, and minimal capabilities. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.0+ +- Access to the container image: `ghcr.io/ostrovok-tech/apatit` + +## Installation + +### Add the chart repository (if applicable) + +```bash +helm repo add +helm repo update +``` + +### Install the chart + +```bash +helm install my-apatit ./ +``` + +Or with a custom values file: + +```bash +helm install my-apatit ./ -f my-values.yaml +``` + +## Configuration + +The following table lists the configurable parameters and their default values. + +### General Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `replicaCount` | Number of replicas | `1` | +| `image.repository` | Container image repository | `ghcr.io/ostrovok-tech/apatit` | +| `image.tag` | Container image tag (defaults to chart appVersion) | `""` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `imagePullSecrets` | Secrets for pulling images from private registries | `[]` | +| `nameOverride` | Override the chart name | `""` | +| `fullnameOverride` | Override the full name | `""` | + +### Security Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `securityContext.runAsUser` | User ID to run the container | `65534` | +| `securityContext.runAsNonRoot` | Run as non-root user | `true` | +| `securityContext.readOnlyRootFilesystem` | Mount root filesystem as read-only | `true` | +| `securityContext.allowPrivilegeEscalation` | Allow privilege escalation | `false` | +| `securityContext.privileged` | Run in privileged mode | `false` | +| `securityContext.capabilities.drop` | Capabilities to drop | `["ALL"]` | + +### Service Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Service port | `8080` | + +### Ingress Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `ingress.enabled` | Enable ingress | `false` | +| `ingress.className` | Ingress class name | `""` | +| `ingress.annotations` | Ingress annotations | `{}` | +| `ingress.hosts` | Ingress hosts configuration | See values.yaml | +| `ingress.tls` | TLS configuration | `[]` | + +### Resource Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `resources.requests.cpu` | CPU request | `15m` | +| `resources.requests.memory` | Memory request | `64Mi` | +| `resources.limits.cpu` | CPU limit | `100m` | +| `resources.limits.memory` | Memory limit | `128Mi` | + +### Health Check Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `livenessProbe` | Liveness probe configuration | HTTP GET on `/metrics` | +| `readinessProbe` | Readiness probe configuration | HTTP GET on `/metrics` | + +### Application Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `env` | Environment variables | `{}` | +| `envSecrets` | Environment variables from secrets | `[]` | +| `args` | Command-line arguments | `[]` | + +### Additional Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | See values.yaml | +| `volumes` | Additional volumes | `[]` | +| `volumeMounts` | Additional volume mounts | `[]` | +| `nodeSelector` | Node selector | `{}` | +| `tolerations` | Pod tolerations | `[]` | +| `affinity` | Pod affinity | `{}` | + +## Application Configuration + +APATIT requires configuration to connect to ping-admin.com API. You can configure it using environment variables or command-line arguments. + +### Required Configuration + +- **API_KEY**: API key from [ping-admin.com/users/edit/](https://ping-admin.com/users/edit/) +- **TASK_IDS**: Task IDs from the ID column in [ping-admin.com/tasks/](https://ping-admin.com/tasks/) + +### Configuration Methods + +#### Method 1: Environment Variables + +```yaml +env: + API_KEY: "" + TASK_IDS: "" + LISTEN_ADDRESS: ":8080" + LOG_LEVEL: "info" +``` + +#### Method 2: Environment Variables from Secrets (Recommended) + +```yaml +envSecrets: + - name: API_KEY + secretName: ping-admin-secret + secretKey: readonly-api-key + - name: TASK_IDS + secretName: ping-admin-secret + secretKey: task-ids +``` + +#### Method 3: Command-line Arguments + +```yaml +args: + - "--api-key=" + - "--task-ids=" + - "--listen-address=:8080" + - "--log-level=info" +``` + +### Available Configuration Options + +| Environment Variable | Command-line Argument | Description | +|---------------------|----------------------|-------------| +| `API_KEY` | `--api-key` | API key from ping-admin.com | +| `TASK_IDS` | `--task-ids` | Task IDs to monitor | +| `LISTEN_ADDRESS` | `--listen-address` | Address to listen on | `:8080` | +| `LOG_LEVEL` | `--log-level` | Logging level | `info` | +| `LOCATIONS_FILE` | `--locations-file` | Locations file path | `locations.json` | +| `ENG_MP_NAMES` | `--eng-mp-names` | Use English MP names | `true` | +| `REFRESH_INTERVAL` | `--refresh-interval` | Refresh interval | `3m` | +| `API_UPDATE_DELAY` | `--api-update-delay` | API update delay | `4m` | +| `API_DATA_TIME_STEP` | `--api-data-time-step` | API data time step | `3m` | +| `MAX_ALLOWED_STALENESS_STEPS` | `--max-allowed-staleness-steps` | Max allowed staleness steps | `3` | +| `MAX_REQUESTS_PER_SECOND` | `--max-requests-per-second` | Max requests per second | `2` | +| `REQUEST_DELAY` | `--request-delay` | Request delay | `2s` | +| `REQUEST_RETRIES` | `--request-retries` | Request retries | `3` | + +## Examples + +### Basic Installation + +```bash +helm install apatit ./ +``` + +### Installation with Custom Values + +Create a `custom-values.yaml`: + +```yaml +replicaCount: 2 + +env: + API_KEY: "your-api-key-here" + TASK_IDS: "1,2,3" + LOG_LEVEL: "debug" + +service: + type: ClusterIP + port: 8080 + +ingress: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + hosts: + - host: apatit.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: apatit-tls + hosts: + - apatit.example.com + +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi +``` + +Install with custom values: + +```bash +helm install apatit ./ -f custom-values.yaml +``` + +### Using Secrets for Sensitive Data + +First, create a Kubernetes secret: + +```bash +kubectl create secret generic ping-admin-secret \ + --from-literal=readonly-api-key='your-api-key' \ + --from-literal=task-ids='1,2,3' +``` + +Then configure the chart to use the secret: + +```yaml +envSecrets: + - name: API_KEY + secretName: ping-admin-secret + secretKey: readonly-api-key + - name: TASK_IDS + secretName: ping-admin-secret + secretKey: task-ids +``` + +### Prometheus Scraping Configuration + +To enable Prometheus scraping, add annotations: + +```yaml +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" +``` + +## Metrics + +The exporter exposes Prometheus metrics at the `/metrics` endpoint. The default port is `8080`. + +## Upgrading + +```bash +helm upgrade apatit ./ +``` + +Or with custom values: + +```bash +helm upgrade apatit ./ -f custom-values.yaml +``` + +## Uninstalling + +```bash +helm uninstall apatit +``` + +## Troubleshooting + +### Check Pod Status + +```bash +kubectl get pods -l app.kubernetes.io/name=apatit +``` + +### View Pod Logs + +```bash +kubectl logs -l app.kubernetes.io/name=apatit +``` + +### Check Service + +```bash +kubectl get svc -l app.kubernetes.io/name=apatit +``` + +### Test Metrics Endpoint + +```bash +kubectl port-forward svc/apatit 8080:8080 +curl http://localhost:8080/metrics +``` + +## Security Considerations + +This chart is configured with security best practices: + +- Runs as non-root user (UID 65534) +- Read-only root filesystem +- No privilege escalation +- All capabilities dropped +- Minimal resource requests/limits + +## Links + +- [Source Code](https://github.com/ostrovok-tech/apatit) +- [ping-admin.com](https://ping-admin.com/) +- [Helm Documentation](https://helm.sh/docs/) + +## Maintainers + +- **Ostrovok! Tech** - [GitHub](https://github.com/ostrovok-tech) + +## License + +Please refer to the source repository for license information. + diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tpl new file mode 100644 index 0000000..e42a051 --- /dev/null +++ b/deploy/helm/templates/_helpers.tpl @@ -0,0 +1,65 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "apatit.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "apatit.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "apatit.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "apatit.labels" -}} +helm.sh/chart: {{ include "apatit.chart" . }} +{{ include "apatit.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "apatit.selectorLabels" -}} +app.kubernetes.io/name: {{ include "apatit.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "apatit.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "apatit.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml new file mode 100644 index 0000000..ae249cc --- /dev/null +++ b/deploy/helm/templates/deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "apatit.fullname" . }} + labels: + {{- include "apatit.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "apatit.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "apatit.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- if .Values.env }} + {{- range $key, $value := .Values.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.envSecrets }} + {{- range $item := .Values.envSecrets }} + - name: {{ $item.name }} + valueFrom: + secretKeyRef: + name: {{ $item.secretName }} + key: {{ $item.secretKey }} + {{- end }} + {{- end }} + {{- with .Values.args }} + args: + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/templates/ingress.yaml b/deploy/helm/templates/ingress.yaml new file mode 100644 index 0000000..de81b64 --- /dev/null +++ b/deploy/helm/templates/ingress.yaml @@ -0,0 +1,43 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "apatit.fullname" . }} + labels: + {{- include "apatit.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- with .pathType }} + pathType: {{ . }} + {{- end }} + backend: + service: + name: {{ include "apatit.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/templates/service.yaml b/deploy/helm/templates/service.yaml new file mode 100644 index 0000000..68be064 --- /dev/null +++ b/deploy/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "apatit.fullname" . }} + labels: + {{- include "apatit.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "apatit.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml new file mode 100644 index 0000000..76a62c2 --- /dev/null +++ b/deploy/helm/values.yaml @@ -0,0 +1,148 @@ +# Default values for APATIT. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: ghcr.io/ostrovok-tech/apatit + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +# This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: + application: apatit + part-of: observability + +podSecurityContext: + +commonLabels: {} + +securityContext: + runAsUser: 65534 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + privileged: false + capabilities: + drop: + - ALL + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 8080 + +# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: apatit.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + requests: + cpu: 15m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: /metrics + port: http +readinessProbe: + httpGet: + path: /metrics + port: http + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Set variables as secret +envSecrets: + - name: # API_KEY + secretName: # ping-admin-secret + secretKey: # readonly-api-key + +# App configuration +env: {} + # API_KEY: "" # API KEY from here https://ping-admin.com/users/edit/ + # TASK_IDS: "" # Task IDs from table ID Column https://ping-admin.com/tasks/ + # LISTEN_ADDRESS: ":8080" + # LOG_LEVEL: "info" + # LOCATIONS_FILE: "locations.json" + # ENG_MP_NAMES: "true" + # REFRESH_INTERVAL: 3m + # API_UPDATE_DELAY: 4m + # API_DATA_TIME_STEP: 3m + # MAX_ALLOWED_STALENESS_STEPS: 3 + # MAX_REQUESTS_PER_SECOND: 2 + # REQUEST_DELAY: 2s + # REQUEST_RETRIES: 3 + +args: [] + # - "--api-key=" # API KEY from here https://ping-admin.com/users/edit/ + # - "--task-ids=" # Task IDs from table ID Column https://ping-admin.com/tasks/" + # - "--listen-address=:8080" + # - "--log-level=info" + # - "--locations-file=locations.json" + # - "--eng-mp-names=true" + # - "--refresh-interval=3m" + # - "--api-update-delay=4m" + # - "--api-data-time-step=3m" + # - "--max-allowed-staleness-steps=3" + # - "--max-requests-per-second=2" + # - "--request-delay=2s" + # - "--request-retries=3" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..50e0225 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module apatit + +go 1.24.0 + +require ( + github.com/prometheus/client_golang v1.23.2 + github.com/sirupsen/logrus v1.9.3 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/sys v0.39.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..11067b7 --- /dev/null +++ b/go.sum @@ -0,0 +1,53 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..0d8a307 --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,30 @@ +package cache + +import "sync" + +var TaskDataCache = &TaskCache{} +var AllTasksInfoCache []byte + +// TaskCache +// is a cache of TaskStat in JSON +type TaskCache struct { + mu sync.RWMutex // RWMutex allows readings + data []byte +} + +// UpdateCache safely updates data in cache +func (c *TaskCache) UpdateCache(data []byte) { + c.mu.Lock() + c.data = data + c.mu.Unlock() +} + +// GetFromCache safely reads data from cache +func (c *TaskCache) GetFromCache() []byte { + c.mu.RLock() + // Returns copy here to avoid data race + dataCopy := make([]byte, len(c.data)) + copy(dataCopy, c.data) + c.mu.RUnlock() + return dataCopy +} diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..ddacc5e --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,247 @@ +package client + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "apatit/internal/utils" + "apatit/internal/version" +) + +const ( + defaultEndpoint = "https://ping-admin.com" +) + +var apiKeyMasker = regexp.MustCompile(`(api_key=)(\w+)`) + +// Client is a client to connect with Ping-Admin API. +type Client struct { + httpClient *http.Client + apiKey string + endpoint string + requestDelay time.Duration + requestRetries int + + // Rate limiting: configurable max requests per second + maxRequestsPerSecond int + rateLimitMu sync.Mutex + lastRequestTimes []time.Time // Track recent request times +} + +// New creates a new API client entity. +func New(apiKey string, httpClient *http.Client, requestDelay time.Duration, requestRetries int, maxRequestsPerSecond int) *Client { + if httpClient == nil { + httpClient = http.DefaultClient + } + if maxRequestsPerSecond <= 0 { + maxRequestsPerSecond = 2 + } + return &Client{ + httpClient: httpClient, + apiKey: apiKey, + endpoint: defaultEndpoint, + requestDelay: requestDelay, + requestRetries: requestRetries, + maxRequestsPerSecond: maxRequestsPerSecond, + } +} + +// waitForRateLimit ensures we don't exceed maxRequestsPerSecond requests per second. +// It waits if necessary before allowing the next request. +// This method is thread-safe and can be called from multiple goroutines. +func (c *Client) waitForRateLimit() { + c.rateLimitMu.Lock() + defer c.rateLimitMu.Unlock() + + now := time.Now() + oneSecondAgo := now.Add(-time.Second) + + // Remove request times older than 1 second + validRequests := make([]time.Time, 0, c.maxRequestsPerSecond) + for _, t := range c.lastRequestTimes { + if t.After(oneSecondAgo) { + validRequests = append(validRequests, t) + } + } + c.lastRequestTimes = validRequests + + // If we have maxRequestsPerSecond requests in the last second, wait until the oldest one expires + if len(c.lastRequestTimes) >= c.maxRequestsPerSecond { + oldestRequest := c.lastRequestTimes[0] + // Wait 1.1 seconds instead of 1 second to add a small buffer and avoid exact 1 second API restriction + waitDuration := 1100*time.Millisecond - now.Sub(oldestRequest) + if waitDuration > 0 { + logrus.WithField("wait_seconds", waitDuration.Seconds()). + Debug("Rate limit: waiting before next API request") + // Sleep while holding the lock - this ensures other goroutines wait + // and see the updated state after we're done + time.Sleep(waitDuration) + now = time.Now() // Update now after waiting + + // Recalculate valid requests after sleep (time has passed) + oneSecondAgo = now.Add(-time.Second) + validRequests = make([]time.Time, 0, 2) + for _, t := range c.lastRequestTimes { + if t.After(oneSecondAgo) { + validRequests = append(validRequests, t) + } + } + c.lastRequestTimes = validRequests + } + } + c.lastRequestTimes = append(c.lastRequestTimes, now) +} + +// getAPI make a request to Ping-Admin API. +// Request could be delayed to avoid "Server Unavailable" error. +func (c *Client) getAPI(path string, result interface{}, delayed bool) error { + log := logrus.WithField("component", "api_client") + + // Enforce rate limit: max 2 requests per second + c.waitForRateLimit() + + req, err := http.NewRequest(http.MethodGet, path, nil) + if err != nil { + return fmt.Errorf("failed to create request: %s", maskAPIKey(err.Error())) + } + req.Header.Set("User-Agent", fmt.Sprintf("%s/%s", version.Name, version.Version)) + + var resp *http.Response + for i := 1; i < c.requestRetries+1; i++ { + + if delayed { + utils.RandomizedPause(c.requestDelay) + } + resp, err = c.httpClient.Do(req) + + log.WithField("url", maskAPIKey(req.URL.String())).Debug("Sending API request") + + if err != nil { + log.WithFields(logrus.Fields{ + "url": maskAPIKey(req.URL.String()), + "error": maskAPIKey(err.Error()), + }).Warn("Failed to send API request") + + if i < c.requestRetries { + log.WithField("url", maskAPIKey(req.URL.String())).Info("Trying to send this request again..") + utils.RandomizedPause(c.requestDelay) + } else { + return fmt.Errorf("request failed: %s", maskAPIKey(err.Error())) + } + + } else { + break + } + } + + if resp == nil { + return fmt.Errorf("no response after \"%s\" request", maskAPIKey(req.URL.String())) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + if err := json.Unmarshal(body, result); err != nil { + return fmt.Errorf("failed to unmarshal JSON response: %w", err) + } + + err = resp.Body.Close() + if err != nil { + return fmt.Errorf("failed to close response body: %w", err) + } + + return nil +} + +// GetTaskGraphStat get task statistics using sa=task_graph_stat request. +func (c *Client) GetTaskGraphStat(taskID int) ([]*MonitoringPointEntry, error) { + u := fmt.Sprintf( + "%s/?a=api&sa=task_graph_stat&enc=utf8&api_key=%s&id=%d¬null=1&limit=1", + c.endpoint, c.apiKey, taskID, + ) + + var resultsRaw []*EntryRaw + if err := c.getAPI(u, &resultsRaw, false); err != nil { + return nil, err + } + + results := make([]*MonitoringPointEntry, len(resultsRaw)) + for i, r := range resultsRaw { + results[i] = r.ProcessMonitoringPointEntry() + } + + return results, nil +} + +// GetTaskStat get task status using sa=task_stat request. +func (c *Client) GetTaskStat(taskID int) (*TaskStatEntry, error) { + u := fmt.Sprintf( + "%s/?a=api&sa=task_stat&enc=utf8&api_key=%s&id=%d&limit=100", + c.endpoint, c.apiKey, taskID, + ) + + var resultsRaw []*TaskStatRaw + if err := c.getAPI(u, &resultsRaw, false); err != nil { + return nil, err + } + + if len(resultsRaw) == 0 { + return nil, fmt.Errorf("no task stat entries returned for task %d", taskID) + } + + processedResult := resultsRaw[0].ProcessTaskEntry() + + return processedResult, nil +} + +// GetMPs get monitoring points info by sa=tm request. +func (c *Client) GetMPs() ([]*MonitoringPointInfo, error) { + u := fmt.Sprintf("%s/?a=api&sa=tm&enc=utf8&api_key=%s", c.endpoint, c.apiKey) + + var mps []*MonitoringPointRaw + if err := c.getAPI(u, &mps, true); err != nil { + return nil, err + } + + processedMonitoringPointsInfo := make([]*MonitoringPointInfo, 0, len(mps)) + for _, mp := range mps { + processedMonitoringPointsInfo = append(processedMonitoringPointsInfo, mp.ProcessMonitoringPointInfo()) + } + + return processedMonitoringPointsInfo, nil +} + +// GetAllTasks get all tasks list. +func (c *Client) GetAllTasks() ([]*TaskInfo, error) { + u := fmt.Sprintf("%s/?a=api&sa=tasks&enc=utf8&api_key=%s", c.endpoint, c.apiKey) + + var tasks []*TaskRaw + if err := c.getAPI(u, &tasks, true); err != nil { + return nil, err + } + + processedTasks := make([]*TaskInfo, 0, len(tasks)) + for _, task := range tasks { + processedTasks = append(processedTasks, task.ProcessTaskInfo()) + } + + return processedTasks, nil +} + +// maskAPIKey change api_key in string on '***' for safe logging. +func maskAPIKey(str string) string { + return apiKeyMasker.ReplaceAllString(str, "${1}***") +} diff --git a/internal/client/models.go b/internal/client/models.go new file mode 100644 index 0000000..0c33d38 --- /dev/null +++ b/internal/client/models.go @@ -0,0 +1,361 @@ +package client + +import ( + "strconv" + "time" + + "github.com/sirupsen/logrus" +) + +// --- Raw API Structures (direct JSON parsing) --- + +// EntryRaw +// is a 'task_graph_stat' API response. +// It contains TM (tochka monitoringa) info with TmID; TmName and TmRes (results with time, speed). +// P.S. TM will be used as MP (Monitoring Point) after processing. +type EntryRaw struct { + TmID string `json:"tm_id"` + TmName string `json:"tm_name"` + TmRes []*TmResRaw `json:"tm_res"` +} + +// TmResRaw +// is a 'tm_res' JSON from 'task_graph_stat' API response. +// It contains TM (tochka monitoringa) info with speed and times for task. +// P.S. TM (tochka monitoringa) will be used as MP (Monitoring Point) after processing. +type TmResRaw struct { + Connect *string `json:"connect"` + DNS *string `json:"dns"` + Server *string `json:"server"` + TmStamp *string `json:"tmstamp"` + Speed *string `json:"speed"` + Total *string `json:"total"` +} + +// TaskRaw +// is a result of 'tasks' API request. +// It contains info about specified task. +type TaskRaw struct { + // Console Task Status (enabled/disabled) + Status int `json:"status"` + // TaskID + ID int `json:"tid"` + // Service Name (in task) + SName string `json:"nazv"` + // IP / DNS-name of service + Address string `json:"name"` + // Task Status (1 — works; 0 — doesn't work) + TaskStatus int `json:"log_status"` + // Blacklist status (if address was found in RKN, Spamhause, etc. blacklists) + BlackListStatus int `json:"rk_log_status"` + // Virus status + VirusStatus int `json:"sb_log_status"` + // This extra data will be obtained via API and may be used in the future. + // Datetime of last check + LastData string `json:"last_data"` + // Datetime of last service status change + LogData string `json:"log_data"` + // Checking period (default) + Period int `json:"period"` + // Checking period (during error status) + PeriodError int `json:"period_error"` + // Blacklist status check settings + Rk int `json:"rk"` + RkIp int `json:"rk_ip"` + RkLogData interface{} `json:"rk_log_data"` + Rrd int `json:"rrd"` + // Virus status check settings + Sb int `json:"sb"` + SbLogData interface{} `json:"sb_log_data"` + // Contacts + TasksImsIcqList []interface{} `json:"tasks_ims_icq_list"` + TasksImsJabberList []interface{} `json:"tasks_ims_jabber_list"` + TasksImsSkypeList []interface{} `json:"tasks_ims_skype_list"` + TasksImsTelegramList []interface{} `json:"tasks_ims_telegram_list"` + // Check type + Tip string `json:"tip"` + // Unavailability time + UptimeNw int `json:"uptime_nw"` + // Availability time + UptimeW int `json:"uptime_w"` + // ??? + Uveddva int `json:"uveddva"` +} + +// MonitoringPointRaw +// is a result of 'tm' API request. +// It contains information about specified Monitoring Point. +type MonitoringPointRaw struct { + // Monitoring Point ID + ID string `json:"id"` + // Monitoring Point Name + Name string `json:"name"` + // Monitoring Point IP + IP string `json:"ip"` + // Monitoring Point GPS + GPS string `json:"gps"` + // Monitoring point availability status + Status string `json:"status"` +} + +// TaskStatRaw +// is a result of 'task_stat' API request. +// Contains info about last task events. +type TaskStatRaw struct { + TasksLogs []*TasksLogsRaw `json:"tasks_logs"` + Uptime string `json:"uptime"` + UptimeNw int `json:"uptime_nw"` + UptimeW int `json:"uptime_w"` +} + +// TasksLogsRaw +// is a 'TasksLogs' array element in 'TaskStatRaw'. +type TasksLogsRaw struct { + Comment *any `json:"comment"` + Data *string `json:"data"` + Descr *string `json:"descr"` + Status *int `json:"status"` + Tm *string `json:"tm"` + TmID *string `json:"tm_id"` + Traceroute *string `json:"traceroute"` +} + +// --- Processed Data Structures --- + +// TaskInfo +// is a processed TaskRaw +type TaskInfo struct { + EnabledStatus int + ID int + ServiceName string + URL string + TaskStatus int + BlackListStatus int + VirusStatus int + Timestamp time.Time +} + +// MonitoringPointInfo +// is a processed MonitoringPointRaw. +type MonitoringPointInfo struct { + ID string + Name string + IP string + GPS string + Status int64 +} + +// MonitoringPointEntry +// is a processed EntryRaw. +type MonitoringPointEntry struct { + ID string + Name string + Status int + Result []*MonitoringPointConnectionResult +} + +// MonitoringPointConnectionResult +// is a processed TmResRaw. +type MonitoringPointConnectionResult struct { + Connect float64 + DNS float64 + Server float64 + Timestamp int64 + Speed int64 + Total float64 +} + +// TaskStatEntry +// is a processed TaskStatRaw. +type TaskStatEntry struct { + TaskID string + TaskName string + Timestamp time.Time + TaskLogs []*TaskLog +} + +// TaskLog +// is a processed TasksLogsRaw. +type TaskLog struct { + Data string + Description string + Status int64 + MPName string + MPID string + Traceroute string +} + +// ProcessMonitoringPointInfo +// converts TaskRaw to TaskInfo. +func (mp *MonitoringPointRaw) ProcessMonitoringPointInfo() *MonitoringPointInfo { + return &MonitoringPointInfo{ + ID: mp.ID, + Name: mp.Name, + IP: mp.IP, + GPS: mp.GPS, + Status: parseInt(&mp.Status, "status"), + } +} + +// ProcessTaskInfo +// converts TaskRaw to TaskInfo. +func (t *TaskRaw) ProcessTaskInfo() *TaskInfo { + return &TaskInfo{ + EnabledStatus: t.Status, + ID: t.ID, + ServiceName: t.SName, + URL: t.Address, + TaskStatus: t.TaskStatus, + BlackListStatus: t.BlackListStatus, + VirusStatus: t.VirusStatus, + Timestamp: time.Now(), + } +} + +// ProcessMonitoringPointEntry +// converts "raw" structure EntryRaw into Entry with correct types of data. +func (e *EntryRaw) ProcessMonitoringPointEntry() *MonitoringPointEntry { + entry := &MonitoringPointEntry{ + ID: e.TmID, + Name: e.TmName, + Result: make([]*MonitoringPointConnectionResult, 0, len(e.TmRes)), + } + + for _, resRaw := range e.TmRes { + MPRes := &MonitoringPointConnectionResult{ + Connect: parseFloat(resRaw.Connect, "connect"), + DNS: parseFloat(resRaw.DNS, "dns"), + Server: parseFloat(resRaw.Server, "server"), + Total: parseFloat(resRaw.Total, "total"), + Speed: parseInt(resRaw.Speed, "speed"), + Timestamp: parseInt(resRaw.TmStamp, "timestamp"), + } + entry.Result = append(entry.Result, MPRes) + } + return entry +} + +// ProcessTaskEntry +// converts "raw" structure TaskStatRaw to TaskStatEntry +func (t *TaskStatRaw) ProcessTaskEntry() *TaskStatEntry { + entry := &TaskStatEntry{ + TaskLogs: make([]*TaskLog, 0, len(t.TasksLogs)), + } + + for _, resRaw := range t.TasksLogs { + TaskStatRes := &TaskLog{ + Data: *resRaw.Data, + Description: *resRaw.Descr, + Status: int64(*resRaw.Status), + MPName: *resRaw.Tm, + MPID: *resRaw.TmID, + Traceroute: *resRaw.Traceroute, + } + entry.TaskLogs = append(entry.TaskLogs, TaskStatRes) + } + return entry +} + +// parseFloat safely parse string to float64. +func parseFloat(s *string, fieldName string) float64 { + if s == nil || *s == "" { + return 0 + } + f, err := strconv.ParseFloat(*s, 64) + if err != nil { + logrus.WithFields(logrus.Fields{"field": fieldName, "value": *s}). + Warn("Failed to parse float value") + return 0 + } + return f +} + +// parseInt safely parse string to int64. +func parseInt(s *string, fieldName string) int64 { + if s == nil || *s == "" { + return 0 + } + i, err := strconv.ParseInt(*s, 10, 64) + if err != nil { + logrus.WithFields(logrus.Fields{"field": fieldName, "value": *s}). + Warn("Failed to parse int value") + return 0 + } + return i +} + +//// Transpose +//// TransposedTaskLogs +//// is just a transposed TaskLog structure. +//type TransposedTaskLogs struct { +// Data []string `json:"Data"` +// Description []string `json:"Description"` +// Status []int64 `json:"Status"` +// MPName []string `json:"MPName"` +// MPID []string `json:"MPID"` +// Traceroute []string `json:"Traceroute"` +//} +// +//// TransposedTaskStatEntry +//// is a transposed 'TaskStatEntry'. +//type TransposedTaskStatEntry struct { +// TaskID string `json:"TaskID"` +// TaskName string `json:"TaskName"` +// TaskLogs TransposedTaskLogs `json:"TaskLogs"` +//} +// +//// transposes TaskLogs. +//func (entry *TaskStatEntry) Transpose() *TransposedTaskStatEntry { +// logCount := len(entry.TaskLogs) +// +// data := make([]string, 0, logCount) +// description := make([]string, 0, logCount) +// status := make([]int64, 0, logCount) +// mpName := make([]string, 0, logCount) +// mpID := make([]string, 0, logCount) +// traceroute := make([]string, 0, logCount) +// +// for _, log := range entry.TaskLogs { +// data = append(data, log.Data) +// description = append(description, log.Description) +// status = append(status, log.Status) +// mpName = append(mpName, log.MPName) +// mpID = append(mpID, log.MPID) +// traceroute = append(traceroute, log.Traceroute) +// } +// +// transposedEntry := &TransposedTaskStatEntry{ +// TaskID: entry.TaskID, +// TaskName: entry.TaskName, +// TaskLogs: TransposedTaskLogs{ +// Data: data, +// Description: description, +// Status: status, +// MPName: mpName, +// MPID: mpID, +// Traceroute: traceroute, +// }, +// } +// +// return transposedEntry +//} +// +//// --- Formatting Helpers --- +// +//// FormatMonitoringPointSliceToMap gets MonitoringPoint slice and returns map[ID]MonitoringPoint +//func FormatMonitoringPointSliceToMap(items []*MonitoringPointRaw) map[string]*MonitoringPointRaw { +// result := make(map[string]*MonitoringPointRaw, len(items)) +// for _, value := range items { +// result[value.ID] = value +// } +// return result +//} +// +//// FormatTaskSliceToMap gets TaskRaw slice and returns map[TaskID]Task +//func FormatTaskSliceToMap(items []*TaskRaw) map[string]*TaskRaw { +// result := make(map[string]*TaskRaw, len(items)) +// for _, value := range items { +// result[strconv.Itoa(value.ID)] = value +// } +// return result +//} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..ee42af3 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,125 @@ +package config + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + "time" +) + +// Config is exporter's configuration parameters defined by ENV or execution keys. +type Config struct { + APIKey string + TaskIDs []int + EngMPNames bool + ApiUpdateDelay time.Duration + ApiDataTimeStep time.Duration + RefreshInterval time.Duration + MaxAllowedStalenessSteps int + RequestDelay time.Duration + RequestRetries int + MaxRequestsPerSecond int + ListenAddress string + LocationsFilePath string + LogLevel string +} + +// New create exporter config. +func New() (*Config, error) { + cfg := &Config{} + + flag.StringVar(&cfg.APIKey, "api-key", envString("API_KEY", ""), "API key for Ping-Admin") + taskIDsStr := flag.String("task-ids", envString("TASK_IDS", ""), "Comma-separated list of task IDs") + flag.BoolVar(&cfg.EngMPNames, "eng-mp-names", envBool("ENG_MP_NAMES", true), "Translate monitoring points (MP) names to English") + flag.DurationVar(&cfg.ApiUpdateDelay, "api-update-delay", envDuration("API_UPDATE_DELAY", 4*time.Minute), "Fixed Ping-Admin API delay for new data update") + flag.DurationVar(&cfg.ApiDataTimeStep, "api-data-time-step", envDuration("API_DATA_TIME_STEP", 3*time.Minute), "Fixed Ping-Admin API time between data points") + flag.DurationVar(&cfg.RefreshInterval, "refresh-interval", envDuration("REFRESH_INTERVAL", 3*time.Minute), "Exporter's refresh interval") + flag.IntVar(&cfg.MaxAllowedStalenessSteps, "max-allowed-staleness-steps", envInt("MAX_ALLOWED_STALENESS_STEPS", 3), "Maximum allowed staleness steps") + flag.DurationVar(&cfg.RequestDelay, "request-delay", envDuration("REQUEST_DELAY", 3*time.Second), "Minimum delay before API request (will be set to random between this and doubled values)") + flag.IntVar(&cfg.RequestRetries, "request-retries", envInt("REQUEST_RETRIES", 3), "Maximum number of retries for API requests") + flag.IntVar(&cfg.MaxRequestsPerSecond, "max-requests-per-second", envInt("MAX_REQUESTS_PER_SECOND", 2), "Maximum number of API requests allowed per second") + flag.StringVar(&cfg.ListenAddress, "listen-address", envString("LISTEN_ADDRESS", ":8080"), "Address to listen on for HTTP requests") + flag.StringVar(&cfg.LocationsFilePath, "locations-file", envString("LOCATIONS_FILE", "locations.json"), "Path to the locations.json translation file") + flag.StringVar(&cfg.LogLevel, "log-level", envString("LOG_LEVEL", "info"), "Log level (e.g., debug, info, warn, error)") + + flag.Parse() + + if cfg.APIKey == "" { + return nil, fmt.Errorf("API key is required, please set --api-key or API_KEY environment variable") + } + + if *taskIDsStr == "" { + return nil, fmt.Errorf("task IDs are required, please set --task-ids or TASK_IDS environment variable") + } + + var err error + cfg.TaskIDs, err = parseTaskIDs(*taskIDsStr) + if err != nil { + return nil, fmt.Errorf("invalid task IDs format: %w", err) + } + + return cfg, nil +} + +// parseTaskIDs get ID for each task from TaskIDs. +func parseTaskIDs(taskIDsStr string) ([]int, error) { + if taskIDsStr == "" { + return nil, nil + } + parts := strings.Split(taskIDsStr, ",") + ids := make([]int, 0, len(parts)) + for _, part := range parts { + trimmedPart := strings.TrimSpace(part) + if trimmedPart == "" { + continue + } + id, err := strconv.Atoi(trimmedPart) + if err != nil { + return nil, fmt.Errorf("'%s' is not a valid integer", trimmedPart) + } + ids = append(ids, id) + } + return ids, nil +} + +// envString string env variables helper. +func envString(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +// envDuration duration env variables helper. +func envDuration(key string, def time.Duration) time.Duration { + if v := os.Getenv(key); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + } + return def +} + +// envBool bool env variables helper. +func envBool(key string, def bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return def +} + +// envInt int env variables helper. +func envInt(env string, def int) int { + if v, ok := os.LookupEnv(env); ok { + i, err := strconv.Atoi(v) + if err != nil { + return def + } + return i + } + return def +} diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go new file mode 100644 index 0000000..a6ca2b9 --- /dev/null +++ b/internal/exporter/exporter.go @@ -0,0 +1,301 @@ +package exporter + +import ( + "fmt" + "math" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + + "apatit/internal/client" + "apatit/internal/translator" +) + +// Exporter collects metrics for a single task. +type Exporter struct { + Config *Config + apiClient *client.Client + log *logrus.Entry + + taskInfo *client.TaskInfo + monitoringPoints []*client.MonitoringPointInfo +} + +// Config contains the configuration for a specific Exporter instance. +type Config struct { + TaskID int + EngMPNames bool + ApiUpdateDelay time.Duration + ApiDataTimeStep time.Duration +} + +// New creates a new Exporter instance. +// Metadata (tasks, monitoring_points) is passed to avoid repeated API requests. +// allTasks map[string]*client.TaskRaw +func New(conf *Config, apiClient *client.Client, allTasks []*client.TaskInfo, mps []*client.MonitoringPointInfo) (*Exporter, error) { + + var taskInfo *client.TaskInfo + for _, task := range allTasks { + if task.ID == conf.TaskID { + taskInfo = task + break + } + } + if taskInfo == nil { + return nil, fmt.Errorf("task with ID %d not found in provided metadata", conf.TaskID) + } + + log := logrus.WithFields(logrus.Fields{ + "component": "exporter", + "task_id": conf.TaskID, + "task_name": taskInfo.ServiceName, + }) + + log.Debug("Exporter instance created") + + return &Exporter{ + Config: conf, + apiClient: apiClient, + log: log, + taskInfo: taskInfo, + monitoringPoints: mps, + }, nil +} + +func (e *Exporter) UpdateAllTasksInfo() ([]*client.TaskInfo, error) { + e.log.Info("Updating all tasks info...") + + allTasks, err := e.apiClient.GetAllTasks() + if err != nil { + EErrorsTotal.WithLabelValues( + "api_client", "get_all_tasks", + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName).Inc() + return nil, fmt.Errorf("error getting all tasks info: %w", err) + } + + return allTasks, nil +} + +// UpdateTaskStats get task_stat data from the API and converts it to JSON. +func (e *Exporter) UpdateTaskStats() (*client.TaskStatEntry, error) { + + e.log.Info("Updating task stats...") + + // Get and process task stats + taskStatResults, err := e.apiClient.GetTaskStat(e.Config.TaskID) + if err != nil { + EErrorsTotal.WithLabelValues( + "api_client", "get_task_stat", + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName).Inc() + return nil, fmt.Errorf("failed to get task stat: %w", err) + } + + e.processTaskStatResults(taskStatResults) + ELoopsTotal.WithLabelValues("stats").Inc() + + return taskStatResults, nil +} + +// RefreshMetrics requests new data from the API, updates Prometheus metrics +// and returns a list of labels for the processed series. +func (e *Exporter) RefreshMetrics() ([]prometheus.Labels, error) { + startTime := time.Now() + e.log.Info("Refreshing metrics...") + + // Updating the exporter's metrics + defer func() { + duration := time.Since(startTime).Seconds() + ELoopsTotal.WithLabelValues("metrics").Inc() + ERefreshDurationSeconds.WithLabelValues(strconv.Itoa(e.taskInfo.ID), e.taskInfo.ServiceName).Set(duration) + e.log.WithField("duration_s", duration).Info("Refresh finished") + }() + + // Get and process task graph stats (metrics) + taskStatGraphResults, err := e.apiClient.GetTaskGraphStat(e.Config.TaskID) + if err != nil { + EErrorsTotal.WithLabelValues( + "api_client", + "get_task_graph_stat", + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName).Inc() + return nil, fmt.Errorf("failed to get task graph stat: %w", err) + } + + if len(taskStatGraphResults) == 0 { + EErrorsTotal.WithLabelValues( + "api_client", + "get_task_graph_stat", + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName).Inc() + e.log.Error("No MP data from API.") + } else { + e.log.Debugf("Received %d data items from API", len(taskStatGraphResults)) + } + + mpsInfo, err := e.apiClient.GetMPs() + if err != nil { + EErrorsTotal.WithLabelValues( + "api_client", "get_mps", + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName).Inc() + return nil, fmt.Errorf("failed to get monitoring points info: %w", err) + } + + processedLabels := make([]prometheus.Labels, 0) + + for _, item := range taskStatGraphResults { + for _, mp := range mpsInfo { + if mp.ID == item.ID { + item.Status = int(mp.Status) + break + } + } + // check monitoring point status and set it as ZERO if it was incorrect + if item.Status > 1 || item.Status < 0 { + e.log.WithFields( + logrus.Fields{ + "mp_id": item.ID, + "mp_name": item.Name, + "status": item.Status, + }).Errorf("incorrect monitoring points status: %d", item.Status) + item.Status = 0 + } + + labels := e.processTaskStatGraphResultItem(item, startTime) + if labels != nil { + processedLabels = append(processedLabels, labels...) + } + } + + return processedLabels, nil +} + +func (e *Exporter) processTaskStatResults(taskStatResults *client.TaskStatEntry) { + + taskStatResults.TaskID = strconv.Itoa(e.taskInfo.ID) + taskStatResults.TaskName = e.taskInfo.ServiceName + taskStatResults.Timestamp = time.Now() + + for _, entry := range taskStatResults.TaskLogs { + entry.Traceroute = strings.ReplaceAll(entry.Traceroute, "\\n", "\n") + entry.MPName = translator.GetEngLocation(entry.MPName) + } +} + +// processTaskStatGraphResultItem processes one record (monitoring point) and updates metrics. +func (e *Exporter) processTaskStatGraphResultItem(item *client.MonitoringPointEntry, refreshStartTime time.Time) []prometheus.Labels { + if len(item.Result) == 0 { + locationName := item.Name + if e.Config.EngMPNames { + locationName = translator.GetEngLocation(item.Name) + } + MPDataStatus.WithLabelValues( + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName, + item.ID, + locationName, + ).Set(0) + + e.log.WithFields( + logrus.Fields{ + "mp_id": item.ID, + "mp_name": item.Name}).Warn("No results found for MP") + return nil + } + + // Usually there is only one element in the MPResult in the response, but just in case we go through them all. + processedLabels := make([]prometheus.Labels, 0, len(item.Result)) + for _, res := range item.Result { + labels := e.buildLabels(item) + e.updateMetrics(res, labels, item.Status, refreshStartTime) + processedLabels = append(processedLabels, labels) + } + + MPDataStatus.WithLabelValues( + strconv.Itoa(e.taskInfo.ID), + e.taskInfo.ServiceName, + item.ID, + processedLabels[0][LabelMPName], + ).Set(1) + + return processedLabels +} + +// buildLabels creates a set of Prometheus labels for a monitoring point. +func (e *Exporter) buildLabels(item *client.MonitoringPointEntry) prometheus.Labels { + locationName := item.Name + if e.Config.EngMPNames { + locationName = translator.GetEngLocation(item.Name) + } + + ipAddress := "unknown" + gpsCoordinates := "unknown" + for _, mp := range e.monitoringPoints { + if mp.ID == item.ID { + ipAddress = mp.IP + gpsCoordinates = mp.GPS + break + } + } + + return prometheus.Labels{ + LabelTaskID: strconv.Itoa(e.taskInfo.ID), + LabelTaskName: e.taskInfo.ServiceName, + LabelMPID: item.ID, + LabelMPName: locationName, + LabelMPIP: ipAddress, + LabelMPGPS: gpsCoordinates, + } +} + +// updateMetrics sets values for all metrics based on data. +func (e *Exporter) updateMetrics(res *client.MonitoringPointConnectionResult, labels prometheus.Labels, mpStatus int, refreshStartTime time.Time) { + ts := time.Unix(res.Timestamp, 0) + lastCheckDelta := refreshStartTime.Sub(ts) + + // remove time related metrics and set MPStatus as ZERO if monitoring point was unavailable according to 'mp' API + if mpStatus == 0 { + e.log.WithFields(logrus.Fields{"mp_id": labels["mp_id"], "mp_name": labels["mp_name"]}). + Warn("Monitoring point is unavailable") + DeleteSeries(labels) + MPStatus.With(labels).Set(0) + return + } + + // remove time related metrics and set MPStatus as ZERO if MP data is older than 24 hours + if lastCheckDelta >= 24*time.Hour { + e.log.WithFields(logrus.Fields{"mp_id": labels["mp_id"], "mp_name": labels["mp_name"]}). + Warn("Data for MP is older than 24 hours") + DeleteSeries(labels) + MPStatus.With(labels).Set(0) + return + } + + // Calculate the latency in "steps" (how many API intervals have passed since the data was received) + // This helps us understand how "old" the data is. 0 is the most recent. + + delayInSteps := math.Floor(math.Abs(lastCheckDelta.Seconds()-e.Config.ApiUpdateDelay.Seconds()) / e.Config.ApiDataTimeStep.Seconds()) + + MPConnectSeconds.With(labels).Set(res.Connect) + MPDNSLookupSeconds.With(labels).Set(res.DNS) + MPServerProcessingSeconds.With(labels).Set(res.Server) + MPTotalDurationSeconds.With(labels).Set(res.Total) + MPSpeedBytesPerSecond.With(labels).Set(float64(res.Speed)) + MPLastSuccessTimestampSeconds.With(labels).Set(float64(res.Timestamp)) + MPLastSuccessDeltaSeconds.With(labels).Set(lastCheckDelta.Seconds()) + MPDataStalenessSteps.With(labels).Set(delayInSteps) + MPStatus.With(labels).Set(1) + + e.log.WithFields(logrus.Fields{ + "mp_id": labels["mp_id"], + "mp_name": labels["mp_name"], + "delta": lastCheckDelta, + "steps": delayInSteps, + }).Debug("Metrics updated for MP") + +} diff --git a/internal/exporter/labels.go b/internal/exporter/labels.go new file mode 100644 index 0000000..3b07928 --- /dev/null +++ b/internal/exporter/labels.go @@ -0,0 +1,16 @@ +package exporter + +// Prometheus Labels +// Task is a monitoring task created in Ping-Admin.com +// MP (Monitoring Point) is a monitoring point for each task (tm in API response) +const ( + LabelErrorModule = "error_module" + LabelErrorType = "error_type" + LabelExporterType = "exporter_type" + LabelTaskID = "task_id" + LabelTaskName = "task_name" + LabelMPID = "mp_id" + LabelMPName = "mp_name" + LabelMPIP = "mp_ip" + LabelMPGPS = "mp_gps" +) diff --git a/internal/exporter/metrics.go b/internal/exporter/metrics.go new file mode 100644 index 0000000..796bfd5 --- /dev/null +++ b/internal/exporter/metrics.go @@ -0,0 +1,229 @@ +package exporter + +import ( + "github.com/prometheus/client_golang/prometheus" + + "apatit/internal/version" +) + +const ( + namespace = "apatit" + subsystemExporter = "exporter" + subsystemMP = "mp" +) + +// Monitoring Point metrics labels +var ( + mpLabels = []string{ + LabelTaskID, + LabelTaskName, + LabelMPID, + LabelMPName, + LabelMPIP, + LabelMPGPS, + } +) + +// Metrics starts with "A" are related to "APATIT" itself +// Metrics starts with "E" are related to "Exporter" +// Metrics starts with "MP" are related to "Monitoring Point" +var ( + AServiceInfo = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_info", + Help: "Information about the APATIT service.", + ConstLabels: prometheus.Labels{ + "language": version.Language, + "name": version.Name, + "owner": version.Owner, + "version": version.Version, + }, + }, + ) + + ERefreshIntervalSeconds = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemExporter, + Name: "refresh_interval_seconds", + Help: "The configured interval for refreshing metrics.", + }, + ) + + EMaxAllowedStalenessSteps = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemExporter, + Name: "max_allowed_staleness_steps", + Help: "Configured staleness threshold in steps. " + + "If `apatit_mp_data_staleness_steps` exceeds this value the MP " + + "is considered potentially unavailable.", + }, + ) + + ERefreshDurationSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemExporter, + Name: "refresh_duration_seconds", + Help: "The duration of the last metrics refresh cycle for a specific task.", + }, + []string{LabelTaskID, LabelTaskName}, + ) + + ELoopsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystemExporter, + Name: "loops_total", + Help: "Total number of refresh loops started for a specific task.", + }, + []string{LabelExporterType}, + ) + + EErrorsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystemExporter, + Name: "errors_total", + Help: "Total number of errors during metrics refresh for a specific task.", + }, + []string{LabelErrorModule, LabelErrorType, LabelTaskID, LabelTaskName}, + ) + + MPStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "status", + Help: "Status of the monitoring point (1 = up/processed, 0 = stale/down).", + }, + mpLabels, + ) + + MPDataStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "data_status", + Help: "Status of the data for the monitoring point (1 = has data, 0 = no data).", + }, + []string{LabelTaskID, LabelTaskName, LabelMPID, LabelMPName}, + ) + + MPConnectSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "connect_seconds", + Help: "Time spent establishing a connection.", + }, + mpLabels, + ) + + MPDNSLookupSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "dns_lookup_seconds", + Help: "Time spent on DNS lookup.", + }, + mpLabels, + ) + + MPServerProcessingSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "server_processing_seconds", + Help: "Time the server spent processing the request.", + }, + mpLabels, + ) + + MPTotalDurationSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "total_duration_seconds", + Help: "Total request time.", + }, + mpLabels, + ) + + MPSpeedBytesPerSecond = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "speed_bytes_per_second", + Help: "Download speed in bytes per second.", + }, + mpLabels, + ) + + MPLastSuccessTimestampSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "last_success_timestamp_seconds", + Help: "Timestamp of the last successful data point from the API.", + }, + mpLabels, + ) + + MPLastSuccessDeltaSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "last_success_delta_seconds", + Help: "Time since the last successful data point was received.", + }, + mpLabels, + ) + + MPDataStalenessSteps = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemMP, + Name: "data_staleness_steps", + Help: "How many API data steps have been missed for this MP. 0 means the data is fresh.", + }, + mpLabels, + ) +) + +func RegisterMetrics() { + prometheus.MustRegister( + AServiceInfo, + ERefreshIntervalSeconds, + EMaxAllowedStalenessSteps, + ERefreshDurationSeconds, + ELoopsTotal, + EErrorsTotal, + MPStatus, + MPDataStatus, + MPConnectSeconds, + MPDNSLookupSeconds, + MPServerProcessingSeconds, + MPTotalDurationSeconds, + MPSpeedBytesPerSecond, + MPLastSuccessTimestampSeconds, + MPLastSuccessDeltaSeconds, + MPDataStalenessSteps, + ) +} + +// DeleteSeries deletes "Monitoring Point" related metrics +func DeleteSeries(labels prometheus.Labels) { + MPStatus.Delete(labels) + MPDataStatus.Delete(labels) + MPConnectSeconds.Delete(labels) + MPDNSLookupSeconds.Delete(labels) + MPServerProcessingSeconds.Delete(labels) + MPTotalDurationSeconds.Delete(labels) + MPSpeedBytesPerSecond.Delete(labels) + MPLastSuccessTimestampSeconds.Delete(labels) + MPLastSuccessDeltaSeconds.Delete(labels) + MPDataStalenessSteps.Delete(labels) +} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..91b6d99 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,22 @@ +package log + +import ( + "github.com/sirupsen/logrus" +) + +// Init initialize logger +func Init(level string) { + logrus.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02T15:04:05.000Z07:00", + }) + + logLevel, err := logrus.ParseLevel(level) + if err != nil { + logrus.WithError(err).Warnf("Invalid log level '%s', defaulting to 'info'", level) + logrus.SetLevel(logrus.InfoLevel) + } else { + logrus.SetLevel(logLevel) + } + + logrus.Info("Logger initialized") +} diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go new file mode 100644 index 0000000..63eff25 --- /dev/null +++ b/internal/scheduler/metrics.go @@ -0,0 +1,84 @@ +package scheduler + +import ( + "fmt" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "apatit/internal/config" + "apatit/internal/exporter" + "apatit/internal/utils" +) + +// runMetricsScheduler starts a loop that periodically updates metrics and clears old ones. +func RunMetricsScheduler(exporters []*exporter.Exporter, cfg *config.Config, stop <-chan struct{}) { + var lastRunMPSeries = make(map[string]prometheus.Labels) + + runCycle := func() { + currentRunMPSeries := make(map[string]prometheus.Labels) + var mu sync.Mutex + var wg sync.WaitGroup + + metricsLog := logrus.WithField("component", "scheduler") + cycleStartTime := time.Now() + metricsLog.Info("Starting new metrics refresh cycle...") + exporter.ERefreshIntervalSeconds.Set(cfg.RefreshInterval.Seconds()) + exporter.EMaxAllowedStalenessSteps.Set(float64(cfg.MaxAllowedStalenessSteps)) + + wg.Add(len(exporters)) + for _, exp := range exporters { + + utils.RandomizedPause(cfg.RequestDelay) + + go func(e *exporter.Exporter) { + defer wg.Done() + + processedLabels, err := e.RefreshMetrics() + if err != nil { + metricsLog.WithFields(logrus.Fields{ + "task_id": e.Config.TaskID, + "error": err, + }).Error("Exporter refresh failed") + return + } + + mu.Lock() + for _, labels := range processedLabels { + seriesKey := fmt.Sprintf("%s:%s", labels["task_id"], labels["mp_id"]) + currentRunMPSeries[seriesKey] = labels + } + mu.Unlock() + }(exp) + } + + wg.Wait() + metricsLog.Infof("All exporters finished refresh cycle in %s.", time.Since(cycleStartTime)) + + // cleaning up absent metrics: comparing series from the previous run with the current ones + for seriesKey, labels := range lastRunMPSeries { + if _, exists := currentRunMPSeries[seriesKey]; !exists { + metricsLog.WithField("series", seriesKey).Info("Deleting stale series") + exporter.DeleteSeries(labels) + } + } + lastRunMPSeries = currentRunMPSeries + metricsLog.Info("Metrics cleanup finished. Waiting for the next cycle.") + } + + ticker := time.NewTicker(cfg.RefreshInterval) + defer ticker.Stop() + + runCycle() // first run starts without ticker + + for { + select { + case <-ticker.C: + runCycle() + case <-stop: + logrus.Infof("Stopping metrics scheduler...") + return + } + } +} diff --git a/internal/scheduler/stats.go b/internal/scheduler/stats.go new file mode 100644 index 0000000..2c2c6ac --- /dev/null +++ b/internal/scheduler/stats.go @@ -0,0 +1,108 @@ +package scheduler + +import ( + "encoding/json" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "apatit/internal/cache" + "apatit/internal/client" + "apatit/internal/config" + "apatit/internal/exporter" + "apatit/internal/utils" +) + +// runStatsScheduler starts a loop that periodically updates task stats and publish them +func RunStatsScheduler(exporters []*exporter.Exporter, cfg *config.Config, stop <-chan struct{}) { + statsLog := logrus.WithField("component", "stats_scheduler") + + runCycle := func() { + cycleStartTime := time.Now() + statsLog.Info("Starting new stats refresh cycle...") + + var wg sync.WaitGroup + var mu sync.Mutex + + // All Task Stats will be here + allStats := make([]*client.TaskStatEntry, 0, len(exporters)) + + wg.Add(len(exporters)) + for _, exp := range exporters { + + utils.RandomizedPause(cfg.RequestDelay) + + go func(e *exporter.Exporter) { + defer wg.Done() + + // perform request about all tasks only once + if e == exporters[0] { + allTasksInfo, err := e.UpdateAllTasksInfo() + if err != nil { + statsLog.WithFields(logrus.Fields{ + "task_id": e.Config.TaskID, + "error": err, + }).Error("All Tasks info refresh failed") + return + } + + cache.AllTasksInfoCache, err = json.Marshal(allTasksInfo) + if err != nil { + statsLog.Errorf("Failed to marshal tasks info to JSON: %v", err) + return + } + + } + + stats, err := e.UpdateTaskStats() + if err != nil { + statsLog.WithFields(logrus.Fields{ + "task_id": e.Config.TaskID, + "error": err, + }).Error("Stats refresh failed") + return + } + + mu.Lock() + allStats = append(allStats, stats) + mu.Unlock() + + }(exp) + } + + wg.Wait() + statsLog.Infof("All exporters finished stats refresh cycle in %s.", time.Since(cycleStartTime)) + + //// transpose stats + //transposedStats := make([]*client.TransposedTaskStatEntry, 0, len(allStats)) + //for _, originalStat := range allStats { + // transposedStats = append(transposedStats, originalStat.Transpose()) + //} + + finalJSON, err := json.Marshal(allStats) + if err != nil { + statsLog.Errorf("Failed to marshal aggregated transposed stats to JSON: %v", err) + return + } + + // safely update cache + cache.TaskDataCache.UpdateCache(finalJSON) + statsLog.Info("Successfully updated tasks JSON cache.") + } + + ticker := time.NewTicker(cfg.RefreshInterval) + defer ticker.Stop() + + runCycle() // first run starts without ticker + + for { + select { + case <-ticker.C: + runCycle() + case <-stop: + logrus.Infof("Stopping stats scheduler...") + return + } + } +} diff --git a/internal/server/http.go b/internal/server/http.go new file mode 100644 index 0000000..c2ef80b --- /dev/null +++ b/internal/server/http.go @@ -0,0 +1,68 @@ +package server + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + + "apatit/internal/cache" +) + +// startServer runs HTTP-server. +func StartServer(listenAddress string) { + // JSON stats endpoint + http.HandleFunc("/stats", statsHandler) + + // Metrics endpoint + http.Handle("/metrics", promhttp.Handler()) + + // Root endpoint + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + + _, _ = w.Write([]byte(` +APATIT +

APATIT

+

Advanced Ping-Admin Task Indicators Transducer

+

Metrics

+

Tasks JSON

+

All Tasks Info JSON

+`)) + }) + + logrus.WithField("address", listenAddress).Info("Starting HTTP server") + if err := http.ListenAndServe(listenAddress, nil); err != nil { + logrus.Fatalf("Failed to start HTTP server: %v", err) + } +} + +// statsHandler handle /stats request with 'type' parameter. +func statsHandler(w http.ResponseWriter, r *http.Request) { + queryParams := r.URL.Query() + dataType := queryParams.Get("type") + + var jsonData []byte + + switch dataType { + case "task": + jsonData = cache.TaskDataCache.GetFromCache() + case "all": + jsonData = cache.AllTasksInfoCache + default: + jsonData = []byte(`{"error":"Invalid or missing 'type' parameter. Use 'type=task' or 'type=all'."}`) + } + + if len(jsonData) == 0 { + jsonData = []byte("[]") + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _, err := w.Write(jsonData) + if err != nil { + logrus.Errorf("Failed to write response: %v", err) + } +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go new file mode 100644 index 0000000..9f43440 --- /dev/null +++ b/internal/translator/translator.go @@ -0,0 +1,59 @@ +// Package translator needs to translate monitoring points (aka 'tochka monitoringa' from RUS to ENG). +// It uses the predefined 'locations.json' file. +package translator + +import ( + "encoding/json" + "fmt" + "os" + "sync" + + "github.com/sirupsen/logrus" +) + +var ( + locationRusToEng map[string]string + once sync.Once + initErr error +) + +// Init loads the locations file once. +func Init(filePath string) error { + once.Do(func() { + log := logrus.WithFields(logrus.Fields{ + "component": "translator", + "path": filePath, + }) + log.Info("Loading translations...") + + file, err := os.ReadFile(filePath) + if err != nil { + initErr = fmt.Errorf("failed to read translations file: %w", err) + log.Error(initErr) + return + } + + if err = json.Unmarshal(file, &locationRusToEng); err != nil { + initErr = fmt.Errorf("failed to parse translations file: %w", err) + log.Error(initErr) + return + } + log.Info("Translations loaded successfully.") + }) + + return initErr +} + +// GetEngLocation returns Monitoring Point name in English +func GetEngLocation(rus string) string { + if initErr != nil { + return rus + } + + if val, ok := locationRusToEng[rus]; ok { + return val + } + + logrus.WithField("location", rus).Warn("Translation not found for location") + return rus +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..e133f62 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,19 @@ +package utils + +import ( + "math/rand" + "time" + + "github.com/sirupsen/logrus" +) + +// RandomizedPause pauses for a random amount of time in the range [min, 2*min] +func RandomizedPause(minDuration time.Duration) { + if minDuration == time.Duration(0) { + return + } + pauseRange := minDuration.Milliseconds() + timeToSleep := time.Duration(pauseRange+rand.Int63n(pauseRange)) * time.Millisecond + logrus.WithField("duration", timeToSleep.String()).Debug("Pausing before next request") + time.Sleep(timeToSleep) +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..b52bfb3 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,14 @@ +package version + +// These constants define application metadata. +// They can be overridden at build time using ldflags. +// Example: go build -ldflags "-X 'apatit/internal/version.Version=1.1.0'" +var ( + Name = "apatit" + Version = "v1.0.0" + Owner = "ostrovok.tech" +) + +const ( + Language = "go" +) diff --git a/locations.json b/locations.json new file mode 100644 index 0000000..9d9d2bf --- /dev/null +++ b/locations.json @@ -0,0 +1,192 @@ +{ + "Россия, Москва, восток 1": "Russia, Moscow, East 1", + "Россия, Москва, восток 2": "Russia, Moscow, East 2", + "Россия, Москва, восток 3": "Russia, Moscow, East 3", + "Россия, Москва, восток 4": "Russia, Moscow, East 4", + "Россия, Москва, запад 1": "Russia, Moscow, West 1", + "Россия, Москва, запад 2": "Russia, Moscow, West 2", + "Россия, Москва, запад 3": "Russia, Moscow, West 3", + "Россия, Москва, запад 4": "Russia, Moscow, West 4", + "Россия, Москва, запад 5": "Russia, Moscow, West 5", + "Россия, Москва, северо-восток 1": "Russia, Moscow, North-East 1", + "Россия, Москва, северо-восток 2": "Russia, Moscow, North-East 2", + "Россия, Москва, северо-запад": "Russia, Moscow, North-West", + "Россия, Москва, север 1": "Russia, Moscow, North 1", + "Россия, Москва, север 2": "Russia, Moscow, North 2", + "Россия, Москва, север 3": "Russia, Moscow, North 3", + "Россия, Москва, центр 1": "Russia, Moscow, Center 1", + "Россия, Москва, центр 3": "Russia, Moscow, Center 3", + "Россия, Москва, центр 4": "Russia, Moscow, Center 4", + "Россия, Москва, юго-восток 1": "Russia, Moscow, South-East 1", + "Россия, Москва, юго-восток 2": "Russia, Moscow, South-East 2", + "Россия, Москва, юго-восток 3": "Russia, Moscow, South-East 3", + "Россия, Москва, юго-запад 1": "Russia, Moscow, South-West 1", + "Россия, Москва, юго-запад 2": "Russia, Moscow, South-West 2", + "Россия, Москва, юго-запад 3": "Russia, Moscow, South-West 3", + "Россия, Москва, юг 1": "Russia, Moscow, South 1", + "Россия, Москва, юг 2": "Russia, Moscow, South 2", + "Россия, Москва, юг 3": "Russia, Moscow, South 3", + "Россия, Владивосток": "Russia, Vladivostok", + "Россия, Владимир": "Russia, Vladimir", + "Россия, Вольно-Надеждинское": "Russia, Volno-Nadezhdinskoye", + "Россия, Воронеж, восток": "Russia, Voronezh, East", + "Россия, Воронеж, запад": "Russia, Voronezh, West", + "Россия, Дубровка": "Russia, Dubrovka", + "Россия, Евпатория": "Russia, Yevpatoria", + "Россия, Екатеринбург, восток": "Russia, Yekaterinburg, East", + "Россия, Екатеринбург, север": "Russia, Yekaterinburg, North", + "Россия, Екатеринбург, центр": "Russia, Yekaterinburg, Center", + "Россия, Иркутск": "Russia, Irkutsk", + "Россия, Казань": "Russia, Kazan", + "Россия, Калининград 1": "Russia, Kaliningrad 1", + "Россия, Калининград 2": "Russia, Kaliningrad 2", + "Россия, Кемерово": "Russia, Kemerovo", + "Россия, Королёв": "Russia, Korolyov", + "Россия, Краснодар, север": "Russia, Krasnodar, North", + "Россия, Краснодар, юг": "Russia, Krasnodar, South", + "Россия, Красноярск": "Russia, Krasnoyarsk", + "Россия, Нижний Новгород": "Russia, Nizhny Novgorod", + "Россия, Новокузнецк": "Russia, Novokuznetsk", + "Россия, Новосибирск, север": "Russia, Novosibirsk, North", + "Россия, Новосибирск, юг": "Russia, Novosibirsk, South", + "Россия, Омск": "Russia, Omsk", + "Россия, Пермь": "Russia, Perm", + "Россия, Петрозаводск": "Russia, Petrozavodsk", + "Россия, Ростов-на-Дону": "Russia, Rostov-on-Don", + "Россия, Санкт-Петербург, восток": "Russia, Saint Petersburg, East", + "Россия, Санкт-Петербург, север": "Russia, Saint Petersburg, North", + "Россия, Санкт-Петербург, центр 1": "Russia, Saint Petersburg, Center 1", + "Россия, Санкт-Петербург, центр 2": "Russia, Saint Petersburg, Center 2", + "Россия, Санкт-Петербург, центр 3": "Russia, Saint Petersburg, Center 3", + "Россия, Санкт-Петербург, юг": "Russia, Saint Petersburg, South", + "Россия, Саратов": "Russia, Saratov", + "Россия, Северск": "Russia, Seversk", + "Россия, Симферополь": "Russia, Simferopol", + "Россия, Тамбов": "Russia, Tambov", + "Россия, Томск, восток": "Russia, Tomsk, East", + "Россия, Томск, центр": "Russia, Tomsk, Center", + "Россия, Уфа": "Russia, Ufa", + "Россия, Хабаровск": "Russia, Khabarovsk", + "Россия, Химки": "Russia, Khimki", + "Россия, Челябинск": "Russia, Chelyabinsk", + "Россия, Челябинск 1": "Russia, Chelyabinsk 1", + "Россия, Челябинск 2": "Russia, Chelyabinsk 2", + "Россия, Южно-Сахалинск": "Russia, Yuzhno-Sakhalinsk", + "Россия, Ярославль": "Russia, Yaroslavl", + "Австралия, Сидней": "Australia, Sydney", + "Австрия, Вена": "Austria, Vienna", + "Азербайджан, Баку": "Azerbaijan, Baku", + "Армения, Абовян": "Armenia, Abovyan", + "Белоруссия, Гомель": "Belarus, Gomel", + "Белоруссия, Минск": "Belarus, Minsk", + "Болгария, София": "Bulgaria, Sofia", + "Бразилия, Сан-Паулу": "Brazil, Sao Paulo", + "Великобритания, Лондон": "United Kingdom, London", + "Великобритания, Хэмпшир": "United Kingdom, Hampshire", + "Вьетнам, Ханой": "Vietnam, Hanoi", + "Германия, Дюссельдорф": "Germany, Dusseldorf", + "Германия, Мюнхен": "Germany, Munich", + "Германия, Нюрнберг": "Germany, Nuremberg", + "Германия, Фалькенштайн": "Germany, Falkenstein", + "Германия, Франкфурт-на-Майне": "Germany, Frankfurt am Main", + "Германия, Эрфурт": "Germany, Erfurt", + "Гонконг": "Hong Kong", + "Греция, Салоники": "Greece, Thessaloniki", + "Грузия, Тбилиси": "Georgia, Tbilisi", + "Дания, Копенгаген": "Denmark, Copenhagen", + "Египет, Каир": "Egypt, Cairo", + "Израиль, Тель-Авив": "Israel, Tel Aviv", + "Индия, Бангалор": "India, Bangalore", + "Иран, Тегеран": "Iran, Tehran", + "Ирландия, Дублин": "Ireland, Dublin", + "Испания, Мадрид": "Spain, Madrid", + "Италия, Ареццо": "Italy, Arezzo", + "Италия, Милан": "Italy, Milan", + "Казахстан, Актау": "Kazakhstan, Aktau", + "Казахстан, Алатау": "Kazakhstan, Alatau", + "Казахстан, Алматы, восток 1": "Kazakhstan, Almaty, East 1", + "Казахстан, Алматы, восток 2": "Kazakhstan, Almaty, East 2", + "Казахстан, Астана": "Kazakhstan, Astana", + "Казахстан, Караганда": "Kazakhstan, Karaganda", + "Казахстан, Павлодар": "Kazakhstan, Pavlodar", + "Канада, Боарнуа": "Canada, Beauharnois", + "Канада, Ванкувер": "Canada, Vancouver", + "Канада, Монреаль": "Canada, Montreal", + "Канада, Торонто": "Canada, Toronto", + "Кипр, Лимассол": "Cyprus, Limassol", + "Киргизия, Бишкек 1": "Kyrgyzstan, Bishkek 1", + "Киргизия, Бишкек 2": "Kyrgyzstan, Bishkek 2", + "Китай, Нанкин": "China, Nanjing", + "Колумбия, Богота": "Colombia, Bogota", + "Латвия, Рига": "Latvia, Riga", + "Литва, Вильнюс": "Lithuania, Vilnius", + "Люксембург, Штейнсель": "Luxembourg, Steinsel", + "Малайзия, Куала-Лумпур": "Malaysia, Kuala Lumpur", + "Мексика, Пуэбла": "Mexico, Puebla", + "Молдавия, Кишинёв": "Moldova, Chisinau", + "Нигерия, Лагос": "Nigeria, Lagos", + "Нидерланды, Mеппел": "Netherlands, Meppel", + "Нидерланды, Амстердам, юго-запад": "Netherlands, Amsterdam, Southwest", + "Нидерланды, Амстердам, юг 1": "Netherlands, Amsterdam, South 1", + "Нидерланды, Дутинхем": "Netherlands, Doetinchem", + "Нидерланды, Налдвейк": "Netherlands, Naaldwijk", + "Новая Зеландия, Окленд": "New Zealand, Auckland", + "Норвегия, Сандефьорд": "Norway, Sandefjord", + "ОАЭ, Фуджайра": "UAE, Fujairah", + "Польша, Варшава": "Poland, Warsaw", + "Польша, Гданьск": "Poland, Gdansk", + "Португалия, Порту": "Portugal, Porto", + "Румыния, Бухарест": "Romania, Bucharest", + "Сербия, Белград": "Serbia, Belgrade", + "Сингапур": "Singapore", + "Словакия, Братислава": "Slovakia, Bratislava", + "США, Аризона, Финикс": "USA, Arizona, Phoenix", + "США, Вашингтон, Сиэтл": "USA, Washington, Seattle", + "США, Виргиния, Ашберн": "USA, Virginia, Ashburn", + "США, Джорджия, Атланта, север": "USA, Georgia, Atlanta, North", + "США, Джорджия, Атланта, юг": "USA, Georgia, Atlanta, South", + "США, Иллинойс, Чикаго": "USA, Illinois, Chicago", + "США, Калифония, Санта Клара": "USA, California, Santa Clara", + "США, Калифорния, Лос-Анджелес 1": "USA, California, Los Angeles 1", + "США, Калифорния, Лос-Анджелес 2": "USA, California, Los Angeles 2", + "США, Миссури, Канзас-Сити": "USA, Missouri, Kansas City", + "США, Невада, Лас-Вегас": "USA, Nevada, Las Vegas", + "США, Нью-Джерси, Клифтон": "USA, New Jersey, Clifton", + "США, Нью-Йорк, Гарден Сити": "USA, New York, Garden City", + "США, Нью-Йорк, Статен-Айленд": "USA, New York, Staten Island", + "США, Орегон, Бенд": "USA, Oregon, Bend", + "США, Техас, Даллас": "USA, Texas, Dallas", + "США, Флорида, Майами": "USA, Florida, Miami", + "США, Флорида, Тампа": "USA, Florida, Tampa", + "Тайвань, Тайбэй": "Taiwan, Taipei", + "Турция, Измир": "Turkey, Izmir", + "Турция, Стамбул": "Turkey, Istanbul", + "Узбекистан, Ташкент": "Uzbekistan, Tashkent", + "Украина, Винница, запад": "Ukraine, Vinnytsia, West", + "Украина, Винница, центр": "Ukraine, Vinnytsia, Center", + "Украина, Днепр": "Ukraine, Dnipro", + "Украина, Киев, запад": "Ukraine, Kyiv, West", + "Украина, Киев, центр": "Ukraine, Kyiv, Center", + "Украина, Киев, центр 2": "Ukraine, Kyiv, Center 2", + "Украина, Киев, юг": "Ukraine, Kyiv, South", + "Украина, Киев, юго-восток": "Ukraine, Kyiv, Southeast", + "Украина, Николаев": "Ukraine, Mykolaiv", + "Украина, Одесса, восток": "Ukraine, Odesa, East", + "Украина, Харьков, север": "Ukraine, Kharkiv, North", + "Украина, Харьков, юг": "Ukraine, Kharkiv, South", + "Украина, Хмельницкий": "Ukraine, Khmelnytskyi", + "Финляндия, Хельсинки": "Finland, Helsinki", + "Франция, Гравлин": "France, Gravelines", + "Франция, Париж": "France, Paris", + "Франция, Рубе": "France, Roubaix", + "Франция, Страсбург, север": "France, Strasbourg, North", + "Франция, Страсбург, юг": "France, Strasbourg, South", + "Чехия, Прага": "Czech Republic, Prague", + "Чили, Курико": "Chile, Curico", + "Швейцария, Хюненберг": "Switzerland, Hunenberg", + "Швеция, Стокгольм": "Sweden, Stockholm", + "Эстония, Нарва": "Estonia, Narva", + "ЮАР, Йоханнесбург": "South Africa, Johannesburg", + "Южная Корея, Сеул": "South Korea, Seoul", + "Япония, Токио": "Japan, Tokyo" +} \ No newline at end of file