add regex extractor (#38)

* add regex extractor

* Bump org.owasp.dependencycheck from 12.0.2 to 12.1.0 (#36)

Bumps org.owasp.dependencycheck from 12.0.2 to 12.1.0.

---
updated-dependencies:
- dependency-name: org.owasp.dependencycheck
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Bump org.cyclonedx.bom from 2.1.0 to 2.2.0 (#37)

Bumps org.cyclonedx.bom from 2.1.0 to 2.2.0.

---
updated-dependencies:
- dependency-name: org.cyclonedx.bom
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: EddeCCC <66794307+EddeCCC@users.noreply.github.com>

* update spring

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: EddeCCC

README.md

@@ -48,6 +48,9 @@ dwh:
   # secret to access the /dwh endpoint
   token: my_secret
 
+  # extract the database from the query via regex
+  derive-database-from-query: false
+
   # the metrics and related queries to use and export
   metrics:
     - name: my-metric|${service}|${http_path}
@@ -76,6 +79,13 @@ It is important that **exactly** the tags used for parameterization in `name` ar
 
 When the endpoint provided by the application is called, the exporter executes **all** configured queries and returns the corresponding result.
 
+#### Derive database from query
+
+If the property `dwh.derive-database-from-query` is set to true and no `database` was explicitly specified for a query,
+the service will try to extract the database via regex. By default, the property is set to false.
+If the `database` is specified, it will always be used for the particular query!
+The property should reduce the effort to specify the database for a long list of simple queries.
+
 ### Health Endpoint
 
 The application provides an endpoint under `/actuator/health that can be used to check if the application is running and the InfluxDB is available.

build.gradle

@@ -4,8 +4,8 @@ plugins {
     id 'org.springframework.boot' version "${springBootVersion}"
     id 'java'
     id 'idea'
-    id "org.cyclonedx.bom" version "2.1.0"
-    id "org.owasp.dependencycheck" version "12.0.2"
+    id "org.cyclonedx.bom" version "2.2.0"
+    id "org.owasp.dependencycheck" version "12.1.0"
     id "com.github.ben-manes.versions" version "0.52.0"
 }
 

gradle.properties

@@ -1,5 +1,5 @@
 # Spring Boot
-springBootVersion=3.4.2
+springBootVersion=3.4.3
 # Spring integration for InfluxDBv2
 influxdbSpringVersion=7.2.0
 

src/main/java/de/novatec/dwhexport/configuration/model/DwhExporterSettings.java

@@ -19,6 +19,12 @@ public class DwhExporterSettings {
      */
     private String token = "";
 
+    /**
+     * True, if the database for queries should be extracted via regex from the query body.
+     * False, if the database should be explicitly specified for every query.
+     */
+    private boolean deriveDatabaseFromQuery = false;
+
     @NotNull
     private List<MetricQuery> metrics;
 }

src/main/java/de/novatec/dwhexport/configuration/model/MetricQuery.java

@@ -1,5 +1,6 @@
 package de.novatec.dwhexport.configuration.model;
 
+import jakarta.validation.constraints.NotNull;
 import lombok.Data;
 
 import jakarta.validation.constraints.NotBlank;
@@ -12,8 +13,8 @@ public class MetricQuery {
     private String name;
 
     /** the database name - in InfluxDBv2 the bucket name */
-    @NotBlank
-    private String database;
+    @NotNull
+    private String database = "";
 
     /** the InfluxQL query to collect metrics */
     @NotBlank

src/main/java/de/novatec/dwhexport/service/MetricsService.java

@@ -6,9 +6,9 @@
 import de.novatec.dwhexport.data.Metric;
 import de.novatec.dwhexport.service.influx.MetricsMapper;
 import de.novatec.dwhexport.service.influx.MetricsQuerier;
+import de.novatec.dwhexport.service.influx.QueryExtractor;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.convert.DurationStyle;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
@@ -26,6 +26,8 @@ public class MetricsService {
 
     private MetricsMapper mapper;
 
+    private QueryExtractor extractor;
+
     private DwhExporterSettings metrics;
 
     public ResponseEntity<?> processRangeOffsetRequest(String range, String offset, Duration intervalDur) {
@@ -63,13 +65,28 @@ private Collection<Metric> queryMetric(MetricQuery metric, Duration interval, lo
         try {
             long intervalMillis = interval.toMillis();
             long extendedStartTime = startTime - intervalMillis;
+            String database = getDatabase(metric);
 
-            InfluxQLQueryResult data = querier.executeQuery(metric, extendedStartTime, endTime, interval);
+            InfluxQLQueryResult data = querier.executeQuery(metric, database, extendedStartTime, endTime, interval);
 
             return mapper.map(data, metric, startTime, endTime);
         } catch (Exception e) {
             log.error("Error fetching data for {}", metric.getName(), e);
         }
         return Collections.emptyList();
     }
+
+    /**
+     * Get the database for the {@link com.influxdb.client.domain.InfluxQLQuery}.
+     * If a database was specified explicitly, use the provided value.
+     * If no database was specified AND the option to derive it from the query is enabled, extract the database from
+     * the query body.
+     *
+     * @param metric the metric query
+     * @return the database to use for the query
+     */
+    private String getDatabase(MetricQuery metric) {
+        if(metric.getDatabase().isBlank() && metrics.isDeriveDatabaseFromQuery()) return extractor.extractDatabase(metric.getQuery());
+        else return metric.getDatabase();
+    }
 }

src/main/java/de/novatec/dwhexport/service/influx/MetricsQuerier.java

@@ -8,7 +8,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.text.StringSubstitutor;
 import org.apache.commons.text.lookup.StringLookup;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.convert.DurationStyle;
 import org.springframework.stereotype.Component;
 
@@ -24,7 +23,7 @@ public class MetricsQuerier {
 
     private InfluxDBClient influx;
 
-    public InfluxQLQueryResult executeQuery(MetricQuery metric, long startMillis, long endMillis, Duration interval) {
+    public InfluxQLQueryResult executeQuery(MetricQuery metric, String database, long startMillis, long endMillis, Duration interval) {
         StringLookup lookup = buildVariableLookUp(startMillis, endMillis, interval);
         StringSubstitutor subst = new StringSubstitutor(lookup);
 
@@ -33,7 +32,7 @@ public InfluxQLQueryResult executeQuery(MetricQuery metric, long startMillis, lo
         log.debug("Executing query: {}", queryString);
 
         // We still use InfluxQL instead of Flux
-        InfluxQLQuery influxQLQuery = new InfluxQLQuery(queryString, metric.getDatabase());
+        InfluxQLQuery influxQLQuery = new InfluxQLQuery(queryString, database);
         return influx.getInfluxQLQueryApi().query(influxQLQuery);
     }
 

src/main/java/de/novatec/dwhexport/service/influx/QueryExtractor.java

@@ -0,0 +1,38 @@
+package de.novatec.dwhexport.service.influx;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Extracts information from InfluxQL queries via regex patterns.
+ */
+@Slf4j
+@Component
+public class QueryExtractor {
+
+    /**
+     * Basically, extract the string after a FROM statement, which is enclosed by double quotes.
+     * For example: SELECT * FROM "inspectit"."raw"."measure" --> inspectit
+     */
+    private static final String DATABASE_PATTERN = "(?i)\\bFROM\\s+\"([^\"]+)";
+
+    /**
+     * @param query the InfluxQL query body
+     * @return the database
+     * @throws IllegalArgumentException if no database could be extracted
+     */
+    public String extractDatabase(String query) {
+        Pattern pattern = Pattern.compile(DATABASE_PATTERN);
+        Matcher matcher = pattern.matcher(query);
+
+        if (matcher.find()) {
+            String database = matcher.group(1);
+            log.debug("Extracted database from query: {}", database);
+            return database;
+        }
+        else throw new IllegalArgumentException("No match found for database. You can specify the database directly as well");
+    }
+}

src/main/resources/application.yml

@@ -34,6 +34,9 @@ dwh:
   # secret to access the /dwh endpoint
   token: my_secret
 
+  # extract the database from the query via regex
+  derive-database-from-query: false
+
   # the metrics and related queries to use and export
   metrics:
     - name: my-metric|${service}|${http_path}