feat: functional render pipeline ytt and helm

Author: vancauwe

.github/workflows/build-images-manifests.yml

@@ -1,8 +1,8 @@
 name: Presidio Docker Build
 
 on:
-  push: 
-  # pull_request: # Future run on main pushes only. 
+  push:
+  # pull_request: # Future run on main pushes only.
   #   branches: [ main ]
   workflow_dispatch:
 
@@ -35,7 +35,7 @@ jobs:
         run: |
           tag=$(curl -s https://api.github.com/repos/microsoft/presidio/releases/latest | jq -r .tag_name)
           echo "tag=$tag" >> $GITHUB_OUTPUT
-          
+
       # SDSC ADD-ON
       - name: Checkout Presidio (latest release)
         uses: actions/checkout@v5
@@ -87,7 +87,7 @@ jobs:
       - name: Create all multi-platform manifests
         run: |
           IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor")
-          
+
           for image in "${IMAGES[@]}"; do
             echo "Creating manifest for $image"
             docker buildx imagetools create \

docs/presidio-poc.md

@@ -8,14 +8,17 @@
 - can be deployed as an API server using a compose stack
 
 ## API usage
+
 2-steps:
+
 - analyze: NER from raw text using models
 - anonymize: config (rule) based processing of pre-detected PII
 
 ### analyze
+
 - Minimal requirements: text + language. By default, all recognizers for that language are enabled.
   ```sh
-  $ curl http://localhost:5002/analyze -s --header "Content-Type: application/json" --request POST --data '{"text": "John Smith drivers license is AC432223","language": "en"}' | jq                                                             
+  $ curl http://localhost:5002/analyze -s --header "Content-Type: application/json" --request POST --data '{"text": "John Smith drivers license is AC432223","language": "en"}' | jq
   [
     {
       "analysis_explanation": null,
@@ -33,19 +36,22 @@
     }
   ]
   ```
-- analysis can be controlled by setting detection score, selecting entities, adding context words and  adding a correlation id(?)
+- analysis can be controlled by setting detection score, selecting entities, adding context words and adding a correlation id(?)
 - ad-hoc pattern (regex) recognizers can be provided as json objects
 - a correlation-id (hash) can be given to append to logs for easier grouping of analyses in logs / traces.
 
 ### anonymize
+
 - By default, the anonymization replaces all detected identifies by their type (e.g. <PERSON>) in the input text.
 - An anonymizer dictionary can be provided to associate specific anonymization procedure to specific entity types.
 - Two inputs must be given to the endpoint:
   - the raw text
   - the response from the analyze step (detected entities and their positions)
 
 ### artificial sample
+
 Input:
+
 ```
 Prof. Gérard Waeber, Chef de service
 Tél: +41 21 314 68 85 / Fax: +41 21 314 08 95
@@ -77,8 +83,10 @@ jfldéijf
 Dr Médecin 00 Formateur
 Chef de clinique
 ```
+
 - ## initial tests
-Works with example artifical lettre de sortie.
+  Works with example artifical lettre de sortie.
+
 ```python
 import json
 import requests
@@ -129,7 +137,9 @@ print(
 ## limitations
 
 ### potential improvements
+
 Model configuration
+
 ```yaml
 # config.yaml
 nlp_engine_name: spacy
@@ -157,30 +167,28 @@ ner_model_configuration:
 ```
 
 Recognizer configuration
+
 ```yaml
 # recognizers.yaml
 recognizers:
-  -
-    name: "Swiss Zip code Recognizer"
+  - name: "Swiss Zip code Recognizer"
     supported_languages:
       - language: fr
         context: [adresse, postal]
       - language: de
-        context: [ort,]
+        context: [ort]
       - language: it
         context: [...]
 
     patterns:
-      -
-         name: "zip code (weak)"
-         regex: "(\\b\\d{5}(?:\\-\\d{4})?\\b)"
-         score: 0.01
+      - name: "zip code (weak)"
+        regex: "(\\b\\d{5}(?:\\-\\d{4})?\\b)"
+        score: 0.01
     context:
-     - zip
-     - code
+      - zip
+      - code
     supported_entity: "ZIP"
-  -
-    name: "Titles recognizer"
+  - name: "Titles recognizer"
     supported_language: "en"
     supported_entity: "TITLE"
     deny_list:
@@ -190,5 +198,4 @@ recognizers:
       - Miss
       - Dr.
       - Prof.
-
 ```

external/vendir.yaml

@@ -8,3 +8,13 @@ directories:
           url: https://github.com/microsoft/presidio
           ref: refs/tags/2.2.360
         newRootPath: docs/samples/deployments/k8s/charts/presidio
+    # - path: helm/presidio
+    # contents:
+    #   - path: .
+    #     helmChart:
+    #       name: presidio
+    #       version: 2.2.360
+    #       git:
+    #         url: https://github.com/microsoft/presidio
+    #         ref: refs/tags/2.2.360
+    #         subPath: docs/samples/deployments/k8s/charts/presidio

justfile

@@ -33,11 +33,18 @@ render-ytt dir="src":
   fd '^ytt$' {{dir}} \
     -x sh -c 'ytt -f {}/values.yaml -f external/ytt/$(basename {//}) --output-files {}/out'
 
+# Render when the code was pulled in via ytt but is a helm template
+[private]
+render-ytt-extract-helm-template dir="src":
+  # render mixed ytt + helm templates with our values into src/<service>/mix/out
+  fd '^helm$' {{dir}} \
+    -x sh -c 'helm template $(basename {//}) external/ytt/$(basename {//}) -f {}/values.yaml --output-dir {}/out'
+
 # Render manifests
 render dir="src":
   just fetch && \
-    just render-helm {{dir}} && \
     just render-ytt {{dir}} && \
+    just render-ytt-extract-helm-template {{dir}} && \
     just format
 
 # Apply manifests in dir to the cluster.

src/presidio/helm/out/presidio/templates/analyzer-deployment.yaml

@@ -0,0 +1,37 @@
+---
+# Source: presidio/templates/analyzer-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: presidio-presidio-analyzer
+  labels:
+    app: presidio-presidio-analyzer
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: presidio-presidio-analyzer
+  template:
+    metadata:
+      labels:
+        app: presidio-presidio-analyzer
+    spec:
+      containers:
+      - name: presidio
+        image: "ghcr.io/presidio-analyzer:latest"
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        resources:
+          requests:
+            memory: 1500Mi
+            cpu: 1500m
+          limits:
+            memory: 3000Mi
+            cpu: 2000m
+        env:
+          - name: PORT
+            value: "8080"

src/presidio/helm/out/presidio/templates/analyzer-service.yaml

@@ -0,0 +1,21 @@
+---
+# Source: presidio/templates/analyzer-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: presidio-presidio-analyzer
+  labels:
+    app: presidio-presidio-analyzer
+    service: presidio-presidio-analyzer
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: presidio-presidio-analyzer

src/presidio/helm/out/presidio/templates/anonymizer-deployment.yaml

@@ -0,0 +1,37 @@
+---
+# Source: presidio/templates/anonymizer-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: presidio-presidio-anonymizer
+  labels:
+    app: presidio-presidio-anonymizer
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: presidio-presidio-anonymizer
+  template:
+    metadata:
+      labels:
+        app: presidio-presidio-anonymizer
+    spec:
+      containers:
+      - name: presidio
+        image: "ghcr.io/presidio-anonymizer:latest"
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        resources:
+          requests:
+            memory: 128Mi
+            cpu: 125m
+          limits:
+            memory: 512Mi
+            cpu: 500m
+        env:
+          - name: PORT
+            value: "8080"

src/presidio/helm/out/presidio/templates/anonymizer-image-deployment.yaml

@@ -0,0 +1,37 @@
+---
+# Source: presidio/templates/anonymizer-image-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: presidio-presidio-image-redactor
+  labels:
+    app: presidio-presidio-image-redactor
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: presidio-presidio-image-redactor
+  template:
+    metadata:
+      labels:
+        app: presidio-presidio-image-redactor
+    spec:
+      containers:
+      - name: presidio
+        image: "ghcr.io/presidio-image-redactor:latest"
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        resources:
+          requests:
+            memory: 1500Mi
+            cpu: 1500m
+          limits:
+            memory: 3000Mi
+            cpu: 2000m
+        env:
+          - name: PORT
+            value: "8080"

src/presidio/helm/out/presidio/templates/anonymizer-image-service.yaml

@@ -0,0 +1,21 @@
+---
+# Source: presidio/templates/anonymizer-image-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: presidio-presidio-image-redactor
+  labels:
+    app: presidio-presidio-image-redactor
+    service: presidio-presidio-image-redactor
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: presidio-presidio-image-redactor

src/presidio/helm/out/presidio/templates/anonymizer-service.yaml

@@ -0,0 +1,21 @@
+---
+# Source: presidio/templates/anonymizer-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: presidio-presidio-anonymizer
+  labels:
+    app: presidio-presidio-anonymizer
+    service: presidio-presidio-anonymizer
+    chart: "presidio-2"
+    release: "presidio"
+    heritage: "Helm"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: presidio-presidio-anonymizer