Merge pull request #210 from bcgov/dev

Feature upgrade deck (#203)

Author: ikethecoder

microservices/gatewayApi/Dockerfile

@@ -20,15 +20,16 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -
 
 #COPY --from=build /deck/deck /usr/local/bin
 
-# gwa api v1/v2
+# gwa api (kong 2)
 RUN curl -sL https://github.com/kong/deck/releases/download/v1.5.0/deck_1.5.0_linux_amd64.tar.gz -o deck.tar.gz && \
    tar -xf deck.tar.gz -C /tmp && \
-   cp /tmp/deck /usr/local/bin/
+   cp /tmp/deck /usr/local/bin/deck_kong2_150
 
-# gwa api v3
-RUN curl -sL https://github.com/Kong/deck/releases/download/v1.27.1/deck_1.27.1_linux_amd64.tar.gz -o deck.tar.gz && \
+# gwa api (kong 3)
+RUN curl -sL https://github.com/Kong/deck/releases/download/v1.45.0/deck_1.45.0_linux_amd64.tar.gz -o deck.tar.gz && \
    tar -xf deck.tar.gz -C /tmp && \
-   cp /tmp/deck /usr/local/bin/deck127
+   cp /tmp/deck /usr/local/bin/deck && \
+   cp /tmp/deck /usr/local/bin/deck_kong3_1450
 
 RUN python -m pip install --upgrade pip
 # FIX: No module named 'urllib3.packages.six'
@@ -50,5 +51,6 @@ RUN chmod +x entrypoint.sh
 EXPOSE 2000
 
 ENV DECK_ANALYTICS=off
+ENV DECK_CLI=deck
 
 ENTRYPOINT ["./entrypoint.sh"]

microservices/gatewayApi/config/test.json

@@ -30,5 +30,6 @@
     "kubeApiPass": "password",
     "kubeApiUser": "username"
   },
-  "compatibilityApiUrl": "http://compatibility-api"
+  "compatibilityApiUrl": "http://compatibility-api",
+   "deckCLI": "deck"
 }

microservices/gatewayApi/entrypoint.sh

@@ -61,7 +61,8 @@ cat > "${CONFIG_PATH:-./config/default.json}" <<EOF
         "kubeApiUser": "${KUBE_API_USER}",
         "kubeApiPass": "${KUBE_API_PASS}"
     },
-    "compatibilityApiUrl": "${COMPATIBILITY_API_URL}"
+    "compatibilityApiUrl": "${COMPATIBILITY_API_URL}",
+    "deckCLI": "${DECK_CLI}"
 }
 EOF
 

microservices/gatewayApi/tests/routes/v2/test_gateway.py

@@ -168,37 +168,37 @@ def test_success_mtls_reference(client):
     assert response.status_code == 200
     assert json.dumps(response.json) == '{"message": "Sync successful.", "results": "Deck reported no changes"}'
 
-def test_kong3_compatibility_warning(client):
-    """Test that Kong 3 incompatible config generates warning"""
-    configFile = '''
-        services:
-        - name: my-service
-          host: myupstream.local
-          tags: ["ns.mytest", "another"]
-          routes:
-          - name: route-1
-            hosts: [ myapi.api.gov.bc.ca ]
-            paths: [ "/path*" ]
-            tags: ["ns.mytest", "another2"]
-          - name: route-2
-            hosts: [ myapi2.api.gov.bc.ca ]
-            paths: [ "/other*" ]
-            tags: ["ns.mytest", "another2"]
-        '''
-
-    data = {
-        "configFile": configFile,
-        "dryRun": False
-    }
-    response = client.put('/v2/namespaces/mytest/gateway', json=data)
-    assert response.status_code == 200
-    response_data = response.json
-    assert response_data["message"] == "Sync successful."
+# def test_kong3_compatibility_warning(client):
+#     """Test that Kong 3 incompatible config generates warning"""
+#     configFile = '''
+#         services:
+#         - name: my-service
+#           host: myupstream.local
+#           tags: ["ns.mytest", "another"]
+#           routes:
+#           - name: route-1
+#             hosts: [ myapi.api.gov.bc.ca ]
+#             paths: [ "/path*" ]
+#             tags: ["ns.mytest", "another2"]
+#           - name: route-2
+#             hosts: [ myapi2.api.gov.bc.ca ]
+#             paths: [ "/other*" ]
+#             tags: ["ns.mytest", "another2"]
+#         '''
+
+#     data = {
+#         "configFile": configFile,
+#         "dryRun": False
+#     }
+#     response = client.put('/v2/namespaces/mytest/gateway', json=data)
+#     assert response.status_code == 200
+#     response_data = response.json
+#     assert response_data["message"] == "Sync successful."
 
-    # Verify warning message and failed routes are in results
-    results = response_data["results"]
-    assert "Kong 3 incompatible routes found" in results
-    assert "route-1" in results
-    assert "route-2" in results
-    # Routes should only be listed once even if multiple incompatible paths
-    assert results.count("route-1") == 1
+#     # Verify warning message and failed routes are in results
+#     results = response_data["results"]
+#     assert "Kong 3 incompatible routes found" in results
+#     assert "route-1" in results
+#     assert "route-2" in results
+#     # Routes should only be listed once even if multiple incompatible paths
+#     assert results.count("route-1") == 1

microservices/gatewayApi/tests/utils/test_add_version.py

@@ -0,0 +1,78 @@
+
+
+from utils.transforms import add_version_if_missing
+import yaml
+
+def test_add_version_with_format():
+    payload = '''
+_format_version: '3.0'
+services:
+- host: myservice
+  name: my-service
+  tags:
+  - ns.mytest
+  - another
+'''
+    y = yaml.load(payload, Loader=yaml.FullLoader)
+
+    add_version_if_missing(y)
+
+    result =  yaml.dump(y, indent=2)
+
+    assert payload.strip() == result.strip()
+
+def test_add_version_if_missing():
+    payload = '''
+services:
+- host: myservice
+  name: my-service
+  tags:
+  - ns.mytest
+  - another
+'''
+
+    expected = '''
+_format_version: '3.0'
+services:
+- host: myservice
+  name: my-service
+  tags:
+  - ns.mytest
+  - another
+'''
+    y = yaml.load(payload, Loader=yaml.FullLoader)
+
+    add_version_if_missing(y)
+
+    result =  yaml.dump(y, indent=2)
+
+    assert expected.strip() == result.strip()
+
+
+def test_add_version_with_v1():
+    payload = '''
+_format_version: 1.0
+services:
+- host: myservice
+  name: my-service
+  tags:
+  - ns.mytest
+  - another
+'''
+
+    expected = '''
+_format_version: '3.0'
+services:
+- host: myservice
+  name: my-service
+  tags:
+  - ns.mytest
+  - another
+'''
+    y = yaml.load(payload, Loader=yaml.FullLoader)
+
+    add_version_if_missing(y)
+
+    result =  yaml.dump(y, indent=2)
+
+    assert expected.strip() == result.strip()

microservices/gatewayApi/utils/deck.py

@@ -0,0 +1,12 @@
+
+def deck_cmd_sync_diff(deck_cli, cmd, select_tag, state):
+    if deck_cli == "deck" or deck_cli.startswith("deck_kong3_"):
+        return [ deck_cli, "gateway", cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", select_tag, state]
+    else:
+        return [ deck_cli, cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", select_tag, "--state", state]
+
+def deck_cmd_validate(deck_cli, state):
+    if deck_cli == "deck" or deck_cli.startswith("deck_kong3_"):
+        return [ deck_cli, "gateway", "validate", "--config", "/tmp/deck.yaml", state ]
+    else:
+        return [ deck_cli, "validate", "--config", "/tmp/deck.yaml", "--state", state ]

microservices/gatewayApi/utils/transforms.py

@@ -94,5 +94,6 @@ def traverse_plugins (yaml, plugin_configs = None):
                         upstream_jwt(item, plugin_configs)
                 traverse_plugins (item, plugin_configs)
 
-
-
+def add_version_if_missing(yaml):
+    if "_format_version" not in yaml or yaml["_format_version"] != "3.0":
+        yaml["_format_version"] = "3.0"

microservices/gatewayApi/v1/routes/gateway.py

@@ -26,8 +26,9 @@
 from clients.ocp_gateway_secret import prep_submitted_config, prep_and_apply_secret, write_submitted_config
 
 from utils.validators import host_valid, validate_upstream
-from utils.transforms import plugins_transformations
+from utils.transforms import plugins_transformations, add_version_if_missing
 from utils.masking import mask
+from utils.deck import deck_cmd_sync_diff, deck_cmd_validate
 
 gw = Blueprint('gwa', 'gateway')
 
@@ -67,11 +68,12 @@ def delete_config(namespace: str, qualifier="") -> object:
 
     # Call the 'deck' command
     cmd = "sync"
+    deck_cli = app.config['deckCLI']
+
+
+    log.info("[%s] (%s) %s action using %s" % (namespace, deck_cli, cmd, selectTag))
+    args = deck_cmd_sync_diff(deck_cli, cmd, selectTag, tempFolder)
 
-    log.info("[%s] %s action using %s" % (namespace, cmd, selectTag))
-    args = [
-        "deck", cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", selectTag, "--state", tempFolder
-    ]
     log.debug("[%s] Running %s" % (namespace, args))
     deck_run = Popen(args, stdout=PIPE, stderr=STDOUT)
     out, err = deck_run.communicate()
@@ -235,6 +237,9 @@ def write_config(namespace: str) -> object:
         # Enrichments
         #######################
 
+        # Add format version if its missing - needed in Kong v3+
+        add_version_if_missing(gw_config)
+
         # Transformation route hosts if in non-prod environment (HOST_TRANSFORM_ENABLED)
         host_transformation(namespace, gw_config)
 
@@ -246,19 +251,21 @@ def write_config(namespace: str) -> object:
         # Enrich the rate-limiting plugin with the appropriate Redis details
         plugins_transformations(namespace, gw_config)
 
-        # Check Kong 3 compatibility
-        is_compatible, compatibility_message, failed_routes, kong2_config = check_kong3_compatibility(namespace, gw_config)
-        if not is_compatible:
-            warning_message = compatibility_message
+        # Disabled:
+        #
+        # # Check Kong 3 compatibility
+        # is_compatible, compatibility_message, failed_routes, kong2_config = check_kong3_compatibility(namespace, gw_config)
+        # if not is_compatible:
+        #     warning_message = compatibility_message
 
-        # Track incompatible routes
-        if not is_compatible:
-            has_incompatible_routes = True
-            all_failed_routes.extend(failed_routes)
+        # # Track incompatible routes
+        # if not is_compatible:
+        #     has_incompatible_routes = True
+        #     all_failed_routes.extend(failed_routes)
 
-        # Use kong2_config (which has compatibility tags) regardless of compatibility status
-        if kong2_config:
-            gw_config = kong2_config
+        # # Use kong2_config (which has compatibility tags) regardless of compatibility status
+        # if kong2_config:
+        #     gw_config = kong2_config
 
         with open("%s/%s" % (tempFolder, 'config-%02d.yaml' % index), 'w') as file:
             yaml.dump(gw_config, file)
@@ -320,12 +327,12 @@ def write_config(namespace: str) -> object:
         selectTag = ns_qualifier
 
     # Call the 'deck' command
+    deck_cli = app.config['deckCLI']
+
+    log.info("[%s] (%s) %s action using %s" % (namespace, deck_cli, cmd, selectTag))
 
-    log.info("[%s] %s action using %s" % (namespace, cmd, selectTag))
+    args = deck_cmd_validate(deck_cli, tempFolder)
 
-    args = [
-        "deck", "validate", "--config", "/tmp/deck.yaml", "--state", tempFolder
-    ]
     log.debug("[%s] Running %s" % (namespace, args))
     deck_validate = Popen(args, stdout=PIPE, stderr=STDOUT)
     out, err = deck_validate.communicate()
@@ -335,9 +342,8 @@ def write_config(namespace: str) -> object:
         abort_early(event_id, 'validate', namespace, jsonify(
             error="Validation Failed.", results=mask(out.decode('utf-8'))))
 
-    args = [
-        "deck", cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", selectTag, "--state", tempFolder
-    ]
+    args = deck_cmd_sync_diff(deck_cli, cmd, selectTag, tempFolder)
+
     log.debug("[%s] Running %s" % (namespace, args))
     deck_run = Popen(args, stdout=PIPE, stderr=STDOUT)
     out, err = deck_run.communicate()