[internal] Adds: RPC-with-TLS, alert, validation (#49)

Signed-off-by: Pavel Karpov <[email protected]>
Signed-off-by: Aleksandr Stefurishin <[email protected]>
Signed-off-by: mortelumina98 <[email protected]>
Signed-off-by: v.oleynikov <[email protected]>
Signed-off-by: Aleksandr Zimin <[email protected]>
Co-authored-by: Aleksandr Stefurishin <[email protected]>
Co-authored-by: mortelumina98 <[email protected]>
Co-authored-by: v.oleynikov <[email protected]>
Co-authored-by: Aleksandr Zimin <[email protected]>

Author: 5 people

.github/workflows/build_dev.yml

@@ -12,35 +12,77 @@ env:
   SOURCE_REPO: ${{ secrets.SOURCE_REPO }}
 
 on:
-  pull_request:
+  #pull_request:
+  # call from trivy_image_check.yaml, which in turn call from pull_request
+  # https://stackoverflow.com/a/71489231
+  workflow_call:
   push:
     branches:
       - main
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
+  set_edition:
+    runs-on: [self-hosted, regular, selectel]
+    name: Set edition
+    outputs:
+      module_edition: ${{ steps.set-vars.outputs.MODULE_EDITION }}
+    steps:
+      - name: Get Pull Request Labels
+        id: get-labels
+        uses: actions/github-script@v7
+        with:
+          script: |
+            if (context.eventName === "pull_request" || context.eventName === "pull_request_target" ) {
+              const prNumber = context.payload.pull_request.number;
+              const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+              });
+              return labels.map(label => label.name);
+            } else {
+              return [];
+            }
+          result-encoding: string
+
+      - name: Set vars
+        id: set-vars
+        run: |
+          # Slect edition for build, default EE
+          if echo "${{ steps.get-labels.outputs.result }}" | grep -q "edition/ce"; then
+            echo "MODULE_EDITION=CE" >> "$GITHUB_OUTPUT"
+          else
+            echo "MODULE_EDITION=EE" >> "$GITHUB_OUTPUT"
+          fi
+
   dev_setup_build:
     runs-on: [self-hosted, regular, selectel]
     name: Build and Push images
+    needs: [set_edition]
+    env:
+      MODULE_EDITION: ${{needs.set_edition.outputs.module_edition}}
     steps:
       - name: Set vars for PR
         if: ${{ github.ref_name != 'main' }}
         run: |
           MODULES_MODULE_TAG="$(echo pr${{ github.ref_name }} | sed 's/\/.*//g')"
           echo "MODULES_MODULE_TAG=$MODULES_MODULE_TAG" >> "$GITHUB_ENV"
-        shell: bash
       - name: Set vars for main
         if: ${{ github.ref_name == 'main' }}
         run: |
           echo "MODULES_MODULE_TAG=${{ github.ref_name }}" >> "$GITHUB_ENV"
-        shell: bash
       - name: Print vars
         run: |
           echo MODULES_REGISTRY=$MODULES_REGISTRY
           echo CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME
           echo MODULES_MODULE_NAME=$MODULES_MODULE_NAME
           echo MODULES_MODULE_SOURCE=$MODULES_MODULE_SOURCE
           echo MODULES_MODULE_TAG=$MODULES_MODULE_TAG
-        shell: bash
+          echo MODULE_EDITION=$MODULE_EDITION
 
       - uses: actions/checkout@v4
       - uses: deckhouse/modules-actions/setup@v1

.github/workflows/build_prod.yml

@@ -25,11 +25,13 @@ jobs:
       - name: SET VAR
         run: |
           echo "MODULES_MODULE_SOURCE=$MODULES_REGISTRY/$MODULE_SOURCE_NAME/ce/modules" >> "$GITHUB_ENV"
+          echo "MODULE_EDITION=CE" >> "$GITHUB_ENV"
       - run: |
           echo $MODULES_REGISTRY
           echo $MODULES_MODULE_NAME
           echo $MODULES_MODULE_SOURCE
           echo $MODULES_MODULE_TAG
+          echo $MODULE_EDITION
         shell: bash
         name: Show vars
 
@@ -58,11 +60,13 @@ jobs:
       - name: SET VAR
         run: |
           echo "MODULES_MODULE_SOURCE=$MODULES_REGISTRY/$MODULE_SOURCE_NAME/ee/modules" >> "$GITHUB_ENV"
+          echo "MODULE_EDITION=EE" >> "$GITHUB_ENV"
       - run: |
           echo $MODULES_REGISTRY
           echo $MODULES_MODULE_NAME
           echo $MODULES_MODULE_SOURCE
           echo $MODULES_MODULE_TAG
+          echo $MODULE_EDITION
         shell: bash
         name: Show vars
 
@@ -91,11 +95,13 @@ jobs:
       - name: SET VAR
         run: |
           echo "MODULES_MODULE_SOURCE=$MODULES_REGISTRY/$MODULE_SOURCE_NAME/fe/modules" >> "$GITHUB_ENV"
+          echo "MODULE_EDITION=EE" >> "$GITHUB_ENV"
       - run: |
           echo $MODULES_REGISTRY
           echo $MODULES_MODULE_NAME
           echo $MODULES_MODULE_SOURCE
           echo $MODULES_MODULE_TAG
+          echo $MODULE_EDITION
         shell: bash
         name: Show vars
 
@@ -124,11 +130,13 @@ jobs:
       - name: SET VAR
         run: |
           echo "MODULES_MODULE_SOURCE=$MODULES_REGISTRY/$MODULE_SOURCE_NAME/se/modules" >> "$GITHUB_ENV"
+          echo "MODULE_EDITION=EE" >> "$GITHUB_ENV"
       - run: |
           echo $MODULES_REGISTRY
           echo $MODULES_MODULE_NAME
           echo $MODULES_MODULE_SOURCE
           echo $MODULES_MODULE_TAG
+          echo $MODULE_EDITION
         shell: bash
         name: Show vars
 
@@ -157,11 +165,13 @@ jobs:
       - name: SET VAR
         run: |
           echo "MODULES_MODULE_SOURCE=$MODULES_REGISTRY/$MODULE_SOURCE_NAME/se-plus/modules" >> "$GITHUB_ENV"
+          echo "MODULE_EDITION=EE" >> "$GITHUB_ENV"
       - run: |
           echo $MODULES_REGISTRY
           echo $MODULES_MODULE_NAME
           echo $MODULES_MODULE_SOURCE
           echo $MODULES_MODULE_TAG
+          echo $MODULE_EDITION
         shell: bash
         name: Show vars
 

.github/workflows/go_lint.yaml

@@ -1,5 +1,8 @@
 name: Go linter for images
 
+env:
+  GO_BUILD_TAGS: "EE CE"
+
 on:
   pull_request:
   push:
@@ -27,17 +30,18 @@ jobs:
         run: |
           basedir=$(pwd)
           failed='false'
-          for dir in $(find images -type d); do
-            if ls $dir/go.mod &> /dev/null; then
-              echo "Running linter in $dir"
-              cd $dir
-              golangci-lint run
+          for i in $(find images -type f -name go.mod);do
+            dir=$(echo $i | sed 's/go.mod$//')
+            cd $basedir/$dir
+            # check all editions
+            for edition in $GO_BUILD_TAGS ;do
+              echo "Running linter in $dir (edition: $edition)"
+              golangci-lint run --build-tags $edition
               if [ $? -ne 0 ]; then
-                echo "Linter failed in $dir"
+                echo "Linter failed in $dir (edition: $edition)"
                 failed='true'
               fi
-            cd $basedir
-            fi
+            done
           done
           if [ $failed == 'true' ]; then
             exit 1

.github/workflows/go_modules_check.yaml

@@ -44,8 +44,14 @@ jobs:
             if [[ "$line" == *github.com/deckhouse/sds-* || "$line" == *github.com/deckhouse/csi-* || "$line" == *github.com/deckhouse/virtualization ]]; then
               repository=$(echo "$line" | awk '{print $1}' | awk -F'/' '{ print "https://"$1"/"$2"/"$3".git" }')
               pseudo_tag=$(echo "$line" | awk '{print $2}')
+              
+              go_pkg=$(echo "$line" | awk '{print $1}')
+              if grep -q "^replace $go_pkg" $go_mod_file ;then
+                echo "Skipping $go_pkg check because it exists in replacement"
+                continue
+              fi
+              
               echo "Cloning repo $repository into $temp_dir"
-            
               git clone "$repository" "$temp_dir/$repository" >/dev/null 2>&1
               
               if [ -d "$temp_dir/$repository/api" ]; then
@@ -68,7 +74,7 @@ jobs:
               else
                 echo "No api directory in $repository"
               fi
-            
+
               rm -rf "$temp_dir/$repository"
             fi
             done < "$go_mod_file"

.github/workflows/go_tests.yaml

@@ -1,5 +1,8 @@
 name: Go tests for images
 
+env:
+  GO_BUILD_TAGS: "EE CE"
+
 on:
   pull_request:
   push:
@@ -24,17 +27,18 @@ jobs:
         run: |
           basedir=$(pwd)
           failed='false'
-          for dir in $(find images -type d); do
-            if ls $dir/*_test.go &> /dev/null; then
-              echo "Running tests in $dir"
-              cd $dir
-              go test -v
+          for i in $(find images -type f -name '*_test.go');do
+            dir=$(echo $i | sed 's/[a-z_A-Z0-9-]*_test.go$//')
+            cd $basedir/$dir
+            # check all editions
+            for edition in $GO_BUILD_TAGS ;do
+              echo "Running tests in $dir (edition: $edition)"
+              go test -v -tags $edition
               if [ $? -ne 0 ]; then
-                echo "Tests failed in $dir"
+                echo "Tests failed in $dir (edition: $edition)"
                 failed='true'
               fi
-            cd $basedir
-            fi
+            done
           done
           if [ $failed == 'true' ]; then
             exit 1

.github/workflows/trivy_image_check.yaml

@@ -12,9 +12,13 @@ on:
   pull_request:
 
 jobs:
+  build_dev:
+    uses: ./.github/workflows/build_dev.yml
+    secrets: inherit
   test:
     name: Trivy images check
     runs-on: [self-hosted, regular]
+    needs: [build_dev]
 
     steps:
       - uses: actions/checkout@v4

.werf/bundle.yaml

@@ -1,8 +1,7 @@
 # Bundle image, stored in your.registry.io/modules/<module-name>:<semver>
 ---
 image: bundle
-from: registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc
-fromCacheVersion: "2024-09-22.1"
+from: {{ $.BASE_SCRATCH }}
 import:
 # Rendering .werf/images-digests.yaml is required!
 - image: images-digests
@@ -14,6 +13,11 @@ import:
   add: /lib/python/dist
   to: /lib/python/dist
   after: setup
+# Rendering .werf/choise-edition.yaml is required!
+- image: choise-edition
+  add: /openapi
+  to: /openapi
+  after: setup
 git:
 - add: /
   to: /
@@ -26,6 +30,5 @@ git:
   - hooks
   - monitoring
   - module.yaml
-  - openapi
   - templates
   - Chart.yaml

.werf/choise-edition.yaml

@@ -0,0 +1,15 @@
+# TODO comment here
+---
+image: choise-edition
+from: {{ $.BASE_ALT_P11 }}
+fromCacheVersion: 2025-01-31.1
+git:
+  - add: /
+    to: /
+    includePaths:
+      - openapi
+shell:
+  setup:
+    - cd /openapi
+    - cp -v values_{{ .MODULE_EDITION }}.yaml values.yaml
+    - rm -rf values_*.yaml

.werf/consts.yaml

@@ -2,9 +2,16 @@
 {{- $_ := set . "BASE_GOLANG_1_23" "registry.deckhouse.io/base_images/golang:1.23.4-bookworm@sha256:a9147a48ac5e925a66764afae7cf4b1cfd37a6e94ad7519eca74c1fd8993ae45" }}
 {{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.io/base_images/scratch@sha256:653ae76965c98c8cd1c8c9ff7725316d2983986f896655b30e0f44d2f8b2dd7e" }}
 {{- $_ := set . "BASE_ALT_P11" "registry.deckhouse.io/base_images/alt:p11@sha256:e47d84424485d3674240cb2f67d3a1801b37d327e6d1eb8cc8d01be8ed3b34f3" }}
+{{- $_ := set . "BASE_ALPINE_3_16" "registry.deckhouse.io/base_images/alpine:3.16.3" }}
+{{- $_ := set . "BASE_ALPINE_3_20" "registry.deckhouse.io/base_images/alpine:3.20.3@sha256:41628df7c9b935d248f64542634e7a843f9bc7f2252d7f878e77f7b79a947466" }}
+
+# Edition module settings, default EE
+{{- $_ := set . "MODULE_EDITION" (env "MODULE_EDITION" "EE") }}
 
 # component versions
 {{- $versions := dict }}
+{{- $_ := set $versions "KMOD" "v33" }}
+{{- $_ := set $versions "KTLS_UTILS" "ktls-utils-0.11" }}
 {{- $_ := set $versions "CSI_DRIVER_NFS" "v4.7.0" }}
 {{- $_ := set $versions "NFS_UTILS" "nfs-utils-2-7-1" }}  # must match the nfs-utils package from BASE_ALT_P11
 {{- $_ := set $ "VERSIONS" $versions }}

.werf/images-digests.yaml

@@ -13,7 +13,7 @@
 # Images Digest: a files with all image digests to be able to use them in helm templates of a module
 ---
 image: images-digests
-from: registry.deckhouse.io/base_images/alpine:3.16.3
+from: {{ $.BASE_ALPINE_3_20 }}
 dependencies:
 {{- range $ImageID := $ImagesIDList }}
   {{- $ImageNameCamel  := $ImageID | splitList "/" | last  | camelcase | untitle }}