feat: internal-gateway: add njs scripts

.github/workflows/lint-test.yaml

@@ -22,9 +22,6 @@ concurrency:
 permissions:
   contents: read
 
-env:
-  HELM_VERSION: 3.9.2 # Also update in release.yaml
-
 jobs:
   lint-charts:
     runs-on: ubuntu-latest
@@ -37,9 +34,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: ${{ env.HELM_VERSION }}
+        uses: azure/[email protected]
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -50,7 +45,7 @@ jobs:
         uses: dcarbone/[email protected]
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.1.0
+        uses: helm/chart-testing-action@v2.7.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -124,9 +119,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: ${{ env.HELM_VERSION }}
+        uses: azure/[email protected]
 
       - name: Run unit tests
         run: |
@@ -151,9 +144,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Helm
-      uses: azure/setup-helm@v3
-      with:
-        version: v3.8.2
+      uses: azure/[email protected]
 
     - name: Set up yq
       uses: chrisdickinson/setup-yq@latest

.github/workflows/release.yaml

@@ -10,9 +10,6 @@ on:
 concurrency:
   group: helm-release
 
-env:
-  HELM_VERSION: 3.9.2 # Also update in lint-test.yaml
-
 jobs:
   publish:
     permissions:
@@ -25,9 +22,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: ${{ env.HELM_VERSION }}
+        uses: azure/[email protected]
 
       - name: Configure Git
         run: |

charts/internal-gateway/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 appVersion: v0.0.0
 description: A Helm chart for Codefresh Internal Gateway
 name: internal-gateway
-version: 0.9.0
+version: 0.10.0
 home: https://github.com/codefresh-io/helm-charts
 keywords:
   - codefresh
@@ -13,4 +13,4 @@ maintainers:
 dependencies:
   - name: cf-common
     repository: oci://quay.io/codefresh/charts
-    version: "0.16.0"
+    version: "0.21.0"

charts/internal-gateway/README.md

@@ -1,6 +1,6 @@
 # internal-gateway
 
-![Version: 0.9.0](https://img.shields.io/badge/Version-0.9.0-informational?style=flat-square) ![AppVersion: v0.0.0](https://img.shields.io/badge/AppVersion-v0.0.0-informational?style=flat-square)
+![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![AppVersion: v0.0.0](https://img.shields.io/badge/AppVersion-v0.0.0-informational?style=flat-square)
 
 A Helm chart for Codefresh Internal Gateway
 
@@ -16,7 +16,7 @@ A Helm chart for Codefresh Internal Gateway
 
 | Repository | Name | Version |
 |------------|------|---------|
-| oci://quay.io/codefresh/charts | cf-common | 0.16.0 |
+| oci://quay.io/codefresh/charts | cf-common | 0.21.0 |
 
 ## Values
 
@@ -33,7 +33,8 @@ A Helm chart for Codefresh Internal Gateway
 | global.dnsService | string | `"kube-dns"` | configures DNS service name |
 | hpa | object | See below | HPA parameters |
 | ingress | object | See below | Ingress parameters |
-| libraryMode | bool | `true` |  |
+| keda.enabled | bool | `false` |  |
+| libraryMode | bool | `false` |  |
 | nginx.config.accessLogEnabled | bool | `true` | Enables NGINX access logs |
 | nginx.config.errorLogLevel | string | `"error"` | Sets the log level of the NGINX error log. One of `debug`, `info`, `notice`, `warn`, `error`, `crit`, `alert`, or `emerg` |
 | nginx.config.file | string | See below | Config file contents for Nginx. Passed through the `tpl` function to allow templating. !! Moved into separate template at `templates/nginx/configmap.yaml` |
@@ -44,12 +45,16 @@ A Helm chart for Codefresh Internal Gateway
 | nginx.config.locations | object | `{}` | Allow add custom locations |
 | nginx.config.logFormat | string | `"main escape=json '{ \"time\": \"$time_iso8601\", \"remote_addr\": \"$proxy_protocol_addr\", \"x-forward-for\": \"$proxy_add_x_forwarded_for\", \"remote_user\": \"$remote_user\", \"bytes_sent\": $bytes_sent, \"request_time\": $request_time, \"status\": $status, \"vhost\": \"$host\", \"request_proto\": \"$server_protocol\", \"path\": \"$uri\", \"request_query\": \"$args\", \"request_length\": $request_length, \"duration\": $request_time, \"method\": \"$request_method\", \"http_referrer\": \"$http_referer\", \"http_user_agent\": \"$http_user_agent\", \"http_x_github_delivery\": \"$http_x_github_delivery\", \"http_x_hook_uuid\": \"$http_x_hook_uuid\", \"metadata\": { \"correlationId\": \"$request_id\", \"service\": \"ingress\", \"time\": \"$time_iso8601\" } }';"` | NGINX log format |
 | nginx.config.resolver | string | `nil` | Allows to set a custom resolver |
+| nginx.config.rootDirectives | object | `{"load_module":"modules/ngx_http_js_module.so"}` | Allows appending custom directives to the root block (map) |
+| nginx.config.rootSnippet | string | `""` | Allows appending custom directives to the root block (string) |
 | nginx.config.serverDirectives | object | `{}` | Allows appending custom directives to the server block (map) |
 | nginx.config.serverSnippet | string | `""` | Allows appending custom configuration to the server block (string) |
 | nginx.config.verboseLogging | bool | `false` | Enable logging of 2xx and 3xx HTTP requests |
 | nginx.config.workerConnections | string | `"16384"` | Sets the maximum number of simultaneous connections that can be opened by a worker process. |
 | nginx.config.workerProcesses | string | `"8"` | Defines the number of worker processes. |
 | nginx.config.workerRlimitNofile | string | `"1047552"` | Changes the limit on the largest size of a core file (RLIMIT_CORE) for worker processes. Used to increase the limit without restarting the main process. |
+| nginx.extraConfigsPatterns[0] | string | `"files/conf.d/**"` |  |
+| nginx.scriptFilesPatterns | list | `["files/njs/**"]` | Path to NJS scripts |
 | pdb | object | See below | PDB parameters |
 | podAnnotations | object | See below | Pod annotations |
 | podSecurityContext | object | See below | Pod Security Context parameters |

charts/internal-gateway/files/conf.d/s3-gateway.conf

@@ -0,0 +1,41 @@
+{{- $vals := include "internal-gateway.default-values" . | fromYaml -}}
+{{- $mergedValues := mergeOverwrite $vals .Values -}}
+{{- $_ := set . "Values" $mergedValues -}}
+server {
+    listen 8080;
+    server_name {{ index $vals "codefresh" "serviceEndpoints" "workflow-logs-s3-proxy" "domain" }};
+
+    js_import scripts/auth.js;
+    location ~ /(.+) {
+        client_body_buffer_size 32k;
+        client_max_body_size 10M;
+        proxy_buffer_size 128k;
+        proxy_buffers 4 128k;
+        proxy_connect_timeout 5s;
+        proxy_read_timeout 60s;
+        proxy_send_timeout 60s;
+
+        auth_request /api/auth/authenticate;
+        auth_request_set $auth_entity $upstream_http_x_cf_auth_entity;
+
+        js_set $account_id auth.account_id;
+
+        proxy_pass http://{{ index $vals "codefresh" "serviceEndpoints" "workflow-logs-s3-proxy" "svc" }}:{{ index $vals "codefresh" "serviceEndpoints" "workflow-logs-s3-proxy" "port" }}/logs/$account_id/$1;
+    }
+
+    location = /api/auth/authenticate {
+        client_body_buffer_size 32k;
+        client_max_body_size 10M;
+        proxy_buffer_size 128k;
+        proxy_buffers 4 128k;
+        proxy_connect_timeout 5s;
+        proxy_read_timeout 60s;
+        proxy_send_timeout 60s;
+
+        js_set $auth_header auth.setAuthHeader;
+
+        proxy_set_header Authorization $auth_header;
+
+        proxy_pass http://{{ index $vals "codefresh" "serviceEndpoints" "cfapi-auth" "svc" }}:{{ index $vals "codefresh" "serviceEndpoints" "cfapi-auth" "port" }};
+    }
+}

charts/internal-gateway/files/njs/auth.js

@@ -0,0 +1,28 @@
+function account_id(r) {
+    try {
+        const auth_entity = r.variables["auth_entity"];
+        const b64decoded = Buffer.from(auth_entity, 'base64');
+        const json = JSON.parse(b64decoded);
+        const account_id = json.account.id;
+
+        return account_id;
+    } catch (e) {
+        r.error('Failed to extract account id', e);
+        return "";
+    }
+}
+
+
+function setAuthHeader(r) {
+    let auth = r.headersIn['authorization'];
+    if (auth) {
+        // Look for the pattern: Credential=<value>/...
+        let matches = auth.match(/Credential=([^\/]+)\//);
+        if (matches && matches.length > 1) {
+            return matches[1];
+        }
+    }
+    return "";
+}
+
+export default { account_id, setAuthHeader };

charts/internal-gateway/templates/_components/_configmap.tpl

@@ -19,6 +19,14 @@ data:
     pid        /tmp/nginx.pid;
     worker_rlimit_nofile {{ $nginxConfig.workerRlimitNofile }};
 
+    {{- with $nginxConfig.rootSnippet }}
+    {{ . | nindent 4 }}
+    {{- end }}
+
+    {{- range $key, $val := $nginxConfig.rootDirectives }}
+    {{ printf "%s %s;" $key $val }}
+    {{- end }}
+
     events {
       worker_connections  {{ $nginxConfig.workerConnections }};
     }
@@ -117,5 +125,6 @@ data:
           {{- end }}
         {{- end }}
       }
+      include /etc/nginx/conf.d/*.conf;
     }
 {{- end }}

charts/internal-gateway/templates/_default_values.tpl

@@ -58,4 +58,8 @@ codefresh:
     jira-addon:
       svc: '{{ .Release.Name }}-{{ index .Values.codefresh "jira-addon-svc" }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}'
       port: {{ index .Values.codefresh "jira-addon-port" }}
+    workflow-logs-s3-proxy:
+      domain: logs.sandbox-1.codefresh.io
+      svc: '{{ .Release.Name }}-{{ index .Values.codefresh "workflow-logs-s3-proxy-svc" }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}'
+      port: {{ index .Values.codefresh "workflow-logs-s3-proxy-port" }}
 {{- end }}

charts/internal-gateway/templates/_location_map.tpl

@@ -26,78 +26,8 @@ locationDirectives:
   proxy_read_timeout: "60s"
 {{- end }}
 
-{{- define "internal-gateway.platform-endpoints-defaults" }}
-serviceEndpoints:
-  cfapi-auth:
-    svc: cfapi-auth
-    port: 80
-  cfapi-endpoints:
-    svc: cfapi-endpoints
-    port: 80
-  cfapi-environments:
-    svc: cfapi-environments
-    port: 80
-  cfapi-downloadlogmanager:
-    svc: cfapi-downloadlogmanager
-    port: 80
-  cfapi-gitops-resource-receiver:
-    svc: cfapi-gitops-resource-receiver
-    port: 80
-  cfapi-test-reporting:
-    svc: cfapi-test-reporting
-    port: 80
-  cfapi-kubernetesresourcemonitor:
-    svc: cfapi-kubernetesresourcemonitor
-    port: 80
-  cfapi-kubernetes-endpoints:
-    svc: cfapi-kubernetes-endpoints
-    port: 80
-  cfapi-admin:
-    svc: cfapi-admin
-    port: 80
-  cfapi-teams:
-    svc: cfapi-teams
-    port: 80
-  cfapi-ws:
-    svc: cfapi-ws
-    port: 80
-  cfui:
-    svc: cfui
-    port: 80
-  argo-platform-api-graphql:
-    svc: argo-platform-api-graphql
-    port: 80
-  argo-platform-api-events:
-    svc: argo-platform-api-events
-    port: 80
-  argo-platform-broadcaster:
-    svc: argo-platform-broadcaster
-    port: 80
-  argo-platform-ui:
-    svc: argo-platform-ui
-    port: 4200
-  argo-hub:
-    svc: argo-hub-platform
-    port: 80
-  nomios:
-    svc: nomios
-    port: 80
-  jira-addon:
-    svc: cf-jira-addon
-    port: 9000
-{{- end }}
-
-{{- define "internal-gateway.platform-endpoints" }}
-{{- $endpointDefaults := include "internal-gateway.platform-endpoints-defaults" . | fromYaml}}
-{{- $mergedEndpoints := deepCopy $endpointDefaults }}
-  {{- if .Values.codefresh.serviceEndpoints }}
-    {{- $mergedEndpoints = mergeOverwrite $endpointDefaults .Values.codefresh }}
-  {{- end }}
-{{ $mergedEndpoints | toYaml }}
-{{- end }}
-
-{{- define "internal-gateway.nginx-config-defaults"}}
-  {{- $endpoints := include "internal-gateway.platform-endpoints" . | fromYaml }}
+{{- define "internal-gateway.nginx-config-defaults" }}
+  {{- $endpoints := .Values.codefresh }}
   {{- $presets := include "internal-gateway.location-presets" . | fromYaml }}
   {{- $_  := set $presets "locationDirectives" (mergeOverwrite $presets.locationDirectives .Values.nginx.config.locationDirectives) }}
 nginx:
@@ -353,7 +283,7 @@ nginx:
 {{- end }}
 
 {{- define "internal-gateway.nginx-config" }}
-{{- $configDefaults := include "internal-gateway.nginx-config-defaults" . | fromYaml}}
+{{- $configDefaults := include "internal-gateway.nginx-config-defaults" . | fromYaml }}
 {{- $mergedConfig := deepCopy $configDefaults }}
   {{- if .Values.nginx }}
     {{- $mergedConfig = mergeOverwrite $configDefaults .Values }}

charts/internal-gateway/templates/extra-configs.yaml

@@ -0,0 +1,13 @@
+{{- if not .Values.libraryMode }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "internal-gateway.fullname" . }}-extra-configs
+data:
+  {{- $rootContext := $ }}
+  {{- range $globPattern := $rootContext.Values.nginx.extraConfigsPatterns }}
+    {{- range $path, $_ := $rootContext.Files.Glob $globPattern }}
+  {{ base $path }}: {{ tpl ($rootContext.Files.Get $path) $rootContext | toYaml | nindent 4 }}
+    {{- end }}
+  {{- end }}
+{{- end }}

charts/internal-gateway/templates/njs-scripts.yaml

@@ -0,0 +1,13 @@
+{{- if not .Values.libraryMode }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "internal-gateway.fullname" . }}-njs-scripts
+data:
+  {{- $rootContext := . }}
+  {{- range $globPattern := $rootContext.Values.nginx.scriptFilesPatterns }}
+    {{- range $path, $_ := $rootContext.Files.Glob $globPattern }}
+  {{ base $path }}: {{ $rootContext.Files.Get $path | toYaml | nindent 4 }}
+    {{- end }}
+  {{- end }}
+{{- end }}