Merge pull request #38 from tinybirdco/parameterize02

parameterize

Author: alrocar

tinybird/endpoints/ingestion.pipe

@@ -21,6 +21,8 @@ DESCRIPTION >
     for a tinybird organization, grouped by workspace_id, event_type and pipe name
     event_type can be 'append', 'append-hfi', 'append-kafka', 'append-dynamodb', 'replace', 'populateview', 'copy'
     Use from_datetime to get metrics since a date time, otherwise it returns last 5 minutes, except ingestion totals, which always sums all values
+    In the response each metric has a description of what it means in the help column
+    The format is compatible with Prometheus
 
 NODE ingestion_totals
 SQL >

tinybird/endpoints/jobs.pipe

@@ -1,8 +1,19 @@
 TOKEN "jobs_endpoint_read" READ
 
+DESCRIPTION >
+    Returns ingestion metrics:
+        jobs_total
+        jobs_duration_avg
+        jobs_duration_p99
+    for a tinybird organization, grouped by workspace_id, job_type and status
+    job_type can be copy, populate, import, gcs_sync, s3_sync, delete
+    Use from_datetime to get metrics since a date time, otherwise it returns last 48 hours
+    In the response each metric has a description of what it means in the help column
+    The format is compatible with Prometheus
+
 NODE jobs_stats
 SQL >
-
+    %
     SELECT
         workspace_id,
         job_type,
@@ -12,7 +23,11 @@ SQL >
         quantile(0.99)
         (date_diff('seconds', created_at, updated_at)) as p99_duration
     FROM organization.jobs_log as stats
-    where created_at > now() - interval 48 hours
+    {% if defined(from_datetime) %}
+        WHERE created_at >= toDateTime({{DateTime(from_datetime, description="timestamp >= from_datetime. Format YYYY-mm-DD HH:MM:ss")}})
+    {% else %}
+        where created_at > now() - interval 48 hours
+    {% end %}
     group by workspace_id, job_type, status
 
 

tinybird/endpoints/organization_metrics.pipe

@@ -1,5 +1,12 @@
 TOKEN "organization_metrics_endpoint_read" READ
 
+DESCRIPTION >
+    returns metrics from ingestion, storage, jobs and requests pipes
+    Use from_datetime to get metrics since a date time, except totals which always sums all values
+    In the response each metric has a description of what it means in the help column
+    The labels column, contains the columns by which the metrics are grouped by, including the workspace name
+    The format is compatible with Prometheus
+
 NODE organization_metrics_union
 SQL >
 

tinybird/endpoints/requests.pipe

@@ -1,5 +1,24 @@
 TOKEN "requests_endpoint_read" READ
 
+DESCRIPTION >
+    Returns requests to pipe endpoints metrics:
+        requests_views_total
+        requests_errors_total
+        requests_read_rows_total
+        requests_read_bytes_total
+        requests_latency_avg_5m
+        requests_latency_p90_5m
+        requests_latency_p95_5m
+        requests_latency_p99_5m
+        requests_cpu_time_avg_5m
+        requests_cpu_time_p90_5m
+        requests_cpu_time_p95_5m
+        requests_cpu_time_p99_5m
+    for a tinybird organization, grouped by workspace_id and pipe name
+    Use from_datetime to get metrics since a date time, otherwise it returns last 5 minutes, except requests totals, which always sums all values
+    In the response each metric has a description of what it means in the help column
+    The format is compatible with Prometheus
+
 NODE requests_totals
 SQL >
 
@@ -17,7 +36,7 @@ SQL >
 
 NODE requests_5m
 SQL >
-
+    %
     SELECT
         workspace_id,
         pipe_name,
@@ -30,7 +49,11 @@ SQL >
         quantile(0.95)(cpu_time) as cpu_time_p95,
         quantile(0.99)(cpu_time) as cpu_time_p99
     FROM organization.pipe_stats_rt
-    where start_datetime > now() - interval 5 minutes
+    {% if defined(from_datetime) %}
+        WHERE start_datetime >= toDateTime({{DateTime(from_datetime, description="timestamp >= from_datetime. Format YYYY-mm-DD HH:MM:ss")}})
+    {% else %}
+        WHERE start_datetime >= now() - INTERVAL 5 MINUTE
+    {% end %}
     group by workspace_id, pipe_name
 
 

tinybird/endpoints/storage.pipe

@@ -1,5 +1,15 @@
 TOKEN "storage_endpoint_read" READ
 
+DESCRIPTION > 
+    Returns storage metrics:
+        storage_rows
+        storage_bytes
+        storage_rows_quarantine
+        storage_bytes_quarantine
+    for a tinybird organization, grouped by workspace_id and datasource name
+    In the response each metric has a description of what it means in the help column
+    The format is compatible with Prometheus
+
 NODE storage_stats
 SQL >