feat: added detiler (#35)

* feat: added detiler

* refactor: detiler error won't cause tile failure

* feat: added detiler proceedOnFailure flag

* deps: updated detiler packages

* feat: added detiler set for process skip

* fix: added test interceptor

* fix: s3 provider endpoint evaluation

* fix: process skip will update state

* refactor: detiler determination is according to tile renderedAt

* chore(deps): updated detiler packages

* fix: changed wms base req service param

Author: melancholiai

config/custom-environment-variables.json

@@ -28,7 +28,14 @@
     "port": "SERVER_PORT"
   },
   "app": {
-    "projectName": "APP_PROJECT_NAME",
+    "project": {
+      "name": "APP_PROJECT_NAME",
+      "stateUrl": "APP_PROJECT_STATE_URL"
+    },
+    "forceProcess": {
+      "__name": "APP_FORCE_PROCESS",
+      "__format": "boolean"
+    },
     "queueName": "QUEUE_NAME",
     "parallelism": {
       "__name": "APP_JOB_PARALLELISM",
@@ -93,5 +100,46 @@
         }
       }
     }
+  },
+  "detiler": {
+    "enabled": {
+      "__name": "DETILER_ENABLED",
+      "__format": "boolean"
+    },
+    "proceedOnFailure": {
+      "enabled": {
+        "__name": "DETILER_PROCEED_ON_FAILURE",
+        "__format": "boolean"
+      }
+    },
+    "client": {
+      "url": "DETILER_CLIENT_URL",
+      "timeout": {
+        "__name": "DETILER_CLIENT_TIMEOUT",
+        "__format": "number"
+      },
+      "enableRetryStrategy": {
+        "__name": "DETILER_CLIENT_ENABLE_RETRY_STRATEGY",
+        "__format": "boolean"
+      },
+      "retryStrategy": {
+        "retries": {
+          "__name": "DETILER_CLIENT_RETRY_STRATEGY_RETRIES",
+          "__format": "number"
+        },
+        "shouldResetTimeout": {
+          "__name": "DETILER_CLIENT_RETRY_STRATEGY_SHOULD_RESET_TIMEOUT",
+          "__format": "boolean"
+        },
+        "isExponential": {
+          "__name": "DETILER_CLIENT_RETRY_STRATEGY_IS_EXPONENTIAL",
+          "__format": "boolean"
+        },
+        "delay": {
+          "__name": "DETILER_CLIENT_RETRY_STRATEGY_DELAY",
+          "__format": "number"
+        }
+      }
+    }
   }
 }

config/default.json

@@ -13,7 +13,11 @@
     "port": "8080"
   },
   "app": {
-    "projectName": "default",
+    "project": {
+      "name": "default",
+      "stateUrl": "defaultStateUrl"
+    },
+    "forceProcess": false,
     "queueName": "QUEUE_NAME",
     "parallelism": 5,
     "jobQueue": {
@@ -64,5 +68,8 @@
       "format": "prefix/{z}/{x}/{y}.png",
       "shouldFlipY": true
     }
+  },
+  "detiler": {
+    "enabled": false
   }
 }

config/test.json

@@ -2,6 +2,10 @@
   "app": {
     "parallelism": 2,
     "queueName": "TEST_QUEUE",
+    "project": {
+      "name": "test",
+      "stateUrl": "http://replication/state.txt"
+    },
     "jobQueue": {
       "waitTimeout": 50,
       "pgBoss": {
@@ -49,5 +53,12 @@
         "shouldFlipY": true
       }
     }
+  },
+  "detiler": {
+    "enabled": true,
+    "proceedOnFailure": true,
+    "client": {
+      "url": "http://detiler.com"
+    }
   }
 }

helm/templates/configmap.yaml

@@ -33,7 +33,26 @@ data:
   {{- end }}
   {{- with .Values.env }}
   NO_SUPERVISOR: {{ .jobQueue.pgBoss.noSupervisor | quote }}
-  APP_PROJECT_NAME: {{ .projectName }}
+  APP_PROJECT_NAME: {{ .project.name }}
+  APP_PROJECT_STATE_URL: {{ .project.stateUrl }}
+  APP_FORCE_PROCESS: {{ .forceProcess }}
+  DETILER_ENABLED: {{ .detiler.enabled | quote }}
+  {{- if .detiler.enabled }}
+  {{- if .detiler.proceedOnFailure }}
+  DETILER_PROCEED_ON_FAILURE: "true"
+  {{- else }}
+  DETILER_PROCEED_ON_FAILURE: "false"
+  {{- end }}
+  DETILER_CLIENT_URL: {{ .detiler.client.url | quote }}
+  DETILER_CLIENT_TIMEOUT: {{ .detiler.client.timeout | quote }}
+  DETILER_CLIENT_ENABLE_RETRY_STRATEGY: {{ .detiler.client.retryStrategy.enabled | quote }}
+  {{- if .detiler.client.retryStrategy.enabled }}
+  DETILER_CLIENT_RETRY_STRATEGY_RETRIES: {{ .detiler.client.retryStrategy.retries | quote }}
+  DETILER_CLIENT_RETRY_STRATEGY_SHOULD_RESET_TIMEOUT: {{ .detiler.client.retryStrategy.shouldResetTimeout | default false | quote }}
+  DETILER_CLIENT_RETRY_STRATEGY_IS_EXPONENTIAL: {{ .detiler.client.retryStrategy.isExponential | default false | quote }}
+  DETILER_CLIENT_RETRY_STRATEGY_DELAY: {{ .detiler.client.retryStrategy.delay | quote }}
+  {{- end }}
+  {{- end }}
   QUEUE_NAME: {{ .queueName }}
   APP_JOB_PARALLELISM: {{ .parallelism | quote }}
   QUEUE_WAIT_TIMEOUT: {{ .jobQueue.waitTimeout | quote}}

helm/values.yaml

@@ -39,7 +39,18 @@ dbConfig:
 
 env:
   logLevel: info
-  projectName: default
+  project:
+    name: default
+    stateUrl: stateUrl
+  detiler:
+    enabled: false
+    proceedOnFailure: true
+    client:
+      url: detilerUrl
+      timeout: 60000
+      retryStrategy:
+        enabled: fasle
+  forceProcess: false
   queueName: QUEUE_NAME
   parallelism: 5
   jobQueue: