aws-v2-divas (#598)

Summary:

- Accomodate problematic `aws` interfaces.
- `SELECT` from `aws.s3.buckets` working in real life.
- `SELECT` on `aws.s3.objects` working.
- Bucket locations working for real.
- Added robot integration test `AWS S3 Buckets Location Constraint`.
- Added robot integration test `AWS S3 Buckets List`.
- Added robot integration test `AWS S3 Bucket Objects List`.

Author: general-kroll-4-life

.github/workflows/build.yml

@@ -742,6 +742,26 @@ jobs:
         echo "## End ##"
         python cicd/python/build.py --robot-test-integration
     
+    - name: Run traffic light robot integration tests
+      if: (startsWith(env.STATE_SOURCE_TAG, 'build-traffic-lights') || (github.repository == 'stackql/stackql' && github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.registry == 'test/registry'
+      env:
+        AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+        AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+        AZURE_INTEGRATION_TESTING_SUB_ID: ${{ secrets.AZURE_INTEGRATION_TESTING_SUB_ID }}
+        AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: |
+        echo "## Stray flask apps to be killed before robot tests ##"
+        pgrep -f flask | xargs kill -9 || true
+        echo "## End ##"
+        if "$(python cicd/python/build.py --robot-test-traffic-lights-integration)"; then
+          echo "Traffic light robot integration tests **all** passed"
+        else
+          rv="$?"
+          echo "**some** traffic light robot integration tests failed code = $rv"
+        fi
+    
     - name: Prepare Test DB
       if: success() && matrix.registry != 'test/registry'
       run: cp test/db/db.sqlite test/db/tmp/python-tests-tmp-db.sqlite

cicd/python/build.py

@@ -67,11 +67,20 @@ def run_robot_integration_tests_stackql(*args, **kwargs) -> int:
     return subprocess.call(
         'robot '
         f'{variables} ' 
-        '-d test/robot/integration '
+        '-d test/robot/reports-integration '
         'test/robot/integration',
         shell=True
     )
 
+def run_robot_integration_traffic_lights_tests_stackql(*args, **kwargs) -> int:
+    variables = ' '.join([f'--variable {key}:{sanitise_val(value)} ' for key, value in kwargs.get("variables", {}).items()])
+    return subprocess.call(
+        'robot '
+        f'{variables} ' 
+        '-d test/robot/reports-integration-traffic-lights '
+        'test/robot/integration-traffic-lights',
+        shell=True
+    )
 
 def main():
     parser = argparse.ArgumentParser()
@@ -81,6 +90,7 @@ def main():
     parser.add_argument('--test', action='store_true')
     parser.add_argument('--robot-test', action='store_true')
     parser.add_argument('--robot-test-integration', action='store_true')
+    parser.add_argument('--robot-test-traffic-lights-integration', action='store_true')
     parser.add_argument('--config', type=json.loads, default={})
     args = parser.parse_args()
     ret_code = 0
@@ -104,6 +114,10 @@ def main():
         ret_code = run_robot_integration_tests_stackql(**args.config)
         if ret_code != 0:
             exit(ret_code)
+    if args.robot_test_traffic_lights_integration:
+        ret_code = run_robot_integration_traffic_lights_tests_stackql(**args.config)
+        if ret_code != 0:
+            exit(ret_code)
     exit(ret_code)
 
 

test/python/stackql_test_tooling/flask/aws/app.py

@@ -216,6 +216,14 @@ def _extract_request_region(request: Request) -> str:
 def handle_root_requests():
     return generic_handler(request)
 
+@app.route('/stackql-trial-bucket-01', methods=["GET"])
+def handle_object_list_requests():
+    return generic_handler(request)
+
+@app.route('/stackql-trial-bucket-02', methods=["GET"])
+def handle_object_list2_requests():
+    return generic_handler(request)
+
 @app.route('/2013-04-01/hostedzone/<rrset_id>/rrset/', methods=['POST', 'GET'])
 def handle_rrset_requests(rrset_id: str):
     return generic_handler(request)

test/python/stackql_test_tooling/flask/aws/root_path_cfg.json

@@ -250,7 +250,7 @@
     },
     "GET:/:10": {
         "method": "GET",
-        "path": "/",
+        "path": "/stackql-trial-bucket-01",
         "queryStringParameters": { 
           "max-keys": [ "1000" ]
         },
@@ -286,7 +286,7 @@
     },
     "GET:/:12": {
         "method": "GET",
-        "path": "/",
+        "path": "/stackql-trial-bucket-01",
         "queryStringParameters": { 
           "location": [ "" ]
         },

test/python/stackql_test_tooling/stackql_context.py

@@ -678,10 +678,10 @@ def generate_password() -> str:
   SELECT_AZURE_COMPUTE_VIRTUAL_MACHINES_JSON_EXPECTED = get_json_from_local_file(os.path.join('test', 'assets', 'expected', 'azure', 'compute', 'vm-list.json'))
   SELECT_AZURE_COMPUTE_BILLING_ACCOUNTS_JSON_EXPECTED = get_json_from_local_file(os.path.join('test', 'assets', 'expected', 'azure', 'billing', 'billing-account-list.json'))
 
-  SELECT_AWS_S3_BUCKET_LOCATIONS = "select LocationConstraint from aws.s3.bucket_locations where region = 'ap-southeast-1' and bucket = 'stackql-trial-bucket-01';"
+  SELECT_AWS_S3_BUCKET_LOCATIONS = "select LocationConstraint from aws.s3.bucket_locations where region = 'ap-southeast-1' and Bucket = 'stackql-trial-bucket-01';"
   SELECT_AWS_S3_BUCKETS = "select Name, CreationDate from  aws.s3.buckets where region = 'ap-southeast-1' order by Name ASC;"
-  SELECT_AWS_S3_OBJECTS = "select \"Key\", Size, StorageClass from  aws.s3.objects where region = 'ap-southeast-1' and bucket = 'stackql-trial-bucket-01' order by \"Key\" ASC;"
-  SELECT_AWS_S3_OBJECTS_NULL = "select \"Key\", Size, StorageClass from  aws.s3.objects where region = 'ap-southeast-2' and bucket = 'stackql-trial-bucket-02' order by \"Key\" ASC;"
+  SELECT_AWS_S3_OBJECTS = "select \"Key\", Size, StorageClass from  aws.s3.objects where region = 'ap-southeast-1' and Bucket = 'stackql-trial-bucket-01' order by \"Key\" ASC;"
+  SELECT_AWS_S3_OBJECTS_NULL = "select \"Key\", Size, StorageClass from  aws.s3.objects where region = 'ap-southeast-2' and Bucket = 'stackql-trial-bucket-02' order by \"Key\" ASC;"
   SELECT_AWS_EC2_VPN_GATEWAYS_NULL = "select vpnGatewayId, amazonSideAsn from aws.ec2.vpn_gateways where region = 'ap-southeast-1' order by vpnGatewayId ASC;"
   SELECT_AWS_VOLUMES = "select volumeId, encrypted, size from aws.ec2.volumes where region = 'ap-southeast-1' order by volumeId asc;"
   SELECT_AWS_IAM_USERS_ASC = "select UserName, Arn from aws.iam.users WHERE region = 'us-east-1' order by UserName ASC;"