Merge pull request #98 from OpenLXP/dev

Dev

Author: sammy-sandhu

.github/workflows/cd-workflows.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches: 
       - main
-      - dev
 jobs:
   code-test:
     # Kicks off the workflow and prepares Github to run coverage test using a ubuntu-latest container.
@@ -51,50 +50,3 @@ jobs:
         echo "Docker network successfully created"
         echo "Running coverage unit test"
         docker compose --env-file ./.env run app_xis sh -c "python manage.py waitdb && coverage run manage.py test --tag=unit && flake8 && coverage report && coverage report --fail-under=80"
-
-  sonarcloud:
-    name: SonarCloud
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-      - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-    
-  build:
-    # requires dependency from step above
-    needs: code-test
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: ${{ secrets.AWS_REGION }}
-
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v1
-      with:
-        mask-password: 'true'
-
-    - name: Build, tag, and push image to Amazon ECR
-      env:
-        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        ECR_REPOSITORY: ${{ secrets.ECR_REPO }}
-        IMAGE_TAG: xis
-      run: |
-        echo "Starting docker build"
-        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-        echo "Pushing image to ECR..."
-        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

Dockerfile

@@ -1,6 +1,6 @@
 # Dockerfile
 
-FROM python:3.9-buster
+FROM python:3.9-bookworm
 
 # install nginx
 

app/api/management/utils/api_helper_functions.py

@@ -31,7 +31,10 @@ def add_metadata_ledger(data, experience_id):
         data['unique_record_identifier'] = str(uuid.uuid4())
 
     # sorting the metadata for consistency
-    data['metadata'] = multi_dict_sort(data['metadata'])
+    if 'metadata' in data:
+        data['metadata'] = multi_dict_sort(data['metadata'])
+    else:
+        data['metadata'] = {}
 
     # create hash values of metadata and supplemental data
     metadata_hash = hashlib.sha512(str(data['metadata']).encode(
@@ -78,7 +81,10 @@ def add_supplemental_ledger(data, experience_id):
         data['unique_record_identifier'] = str(uuid.uuid4())
 
     # sorting the metadata for consistency
-    data['metadata'] = multi_dict_sort(data['metadata'])
+    if 'metadata' in data:
+        data['metadata'] = multi_dict_sort(data['metadata'])
+    else:
+        data['metadata'] = {}
 
     # create hash values of metadata and supplemental data
     supplemental_hash = hashlib.sha512(str(data['metadata'])

app/api/serializers.py

@@ -102,7 +102,6 @@ def validate(self, data):
         # validate for recommended values in data
         validate_recommended(
             data, recommended_column_list, flattened_source_data)
-
         # Type checking for values in metadata
         for item in flattened_source_data:
             # check if datatype has been assigned to field
@@ -139,6 +138,7 @@ def validate(self, data):
             raise serializers.ValidationError("Metadata has missing fields. "
                                               "Data did not pass validation."
                                               "Check logs for more details")
+
         return data
 
     def update(self, instance, validated_data):

app/api/tests/test_setup.py

@@ -12,13 +12,19 @@
 from core.models import (CompositeLedger, MetadataLedger, SupplementalLedger,
                          XISConfiguration)
 
+from django.test import override_settings
+
 
 class TestSetUp(APITestCase):
     """Class with setup and teardown for tests in XIS"""
 
     def setUp(self):
         """Function to set up necessary data for testing"""
 
+        settings_manager = override_settings(SECURE_SSL_REDIRECT=False)
+        settings_manager.enable()
+        self.addCleanup(settings_manager.disable)
+
         self.su_username = "super@test.com"
         self.su_password = "1234"
 

app/api/views.py

@@ -113,6 +113,8 @@ def filter_queryset(self, queryset):
     def post(self, request):
         """This method defines the API's to save data to the
         metadata ledger in the XIS"""
+        logger.info("Start processing")
+        logger.error("Incoming experience")
 
         # Add optional/recommended fields to the metadata
         extra_fields = get_optional_and_recommended_fields_for_validation()
@@ -346,31 +348,31 @@ def post(self, request, provider_id, experience_id):
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_workflow_api(request):
     logger.info('XIS workflow api')
     task = xis_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_downstream_workflow_api(request):
     logger.info('Downstream workflow api')
     task = xis_downstream_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_upstream_workflow_api(request):
     logger.info('Upstream workflow api')
     task = xis_upstream_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def get_status(request, task_id):
     task_result = AsyncResult(task_id)
     result = {

app/core/fixtures/admin_theme_data.json

@@ -41,7 +41,7 @@
             "css_delete_button_background_color": "#BA2121",
             "css_delete_button_background_hover_color": "#A41515",
             "css_delete_button_text_color": "#FFFFFF",
-            "related_modal_active": true,
+            "related_modal_active": false,
             "related_modal_background_color": "#000000",
             "related_modal_background_opacity": "0.3",
             "related_modal_rounded_corners": true,

app/core/management/commands/load_metadata_from_xis.py

@@ -35,20 +35,21 @@ def retrieve_records(self, upstream):
 
         xis_response = requests.get(
             url=upstream.xis_api_endpoint + 'metadata/',
-            headers=headers)
+            headers=headers, timeout=3.0)
 
         while (xis_response.status_code//10 == 20):
             for record in xis_response.json()['results']:
                 self.save_record(upstream, bleach_data_to_json(record))
 
             if (xis_response.json()['next'] is not None):
                 xis_response = requests.get(
-                    url=xis_response.json()['next'], headers=headers)
+                    url=xis_response.json()['next'], headers=headers,
+                    timeout=3.0)
             else:
                 return
 
         logger.error(
-            f"HTTP Error {xis_response.status_code} from {upstream}")
+            "HTTP Error %s from %s", xis_response.status_code, upstream)
 
     def save_record(self, upstream, record):
         """saves record to metadata and supplemental ledgers as needed"""

app/core/management/commands/load_metadata_into_xis.py

@@ -76,9 +76,9 @@ def send_record(self, downstream, record):
             url=f'{downstream.xis_api_endpoint}managed-data/catalogs/'
             f'{record["provider_name"]}/{record["metadata_key_hash"]}',
             data=json.dumps(CompositeLedgerSerializer(record).data),
-            headers=headers)
+            headers=headers, timeout=3.0)
 
-        if (xis_response.status_code//10 == 20):
+        if(xis_response.status_code//10 == 20):
             downstream.composite_experiences.add(
                 record['unique_record_identifier'])
         else:
@@ -96,10 +96,10 @@ def handle(self, *args, **options):
         downstream_apis = XISDownstream.objects.all().filter(
             xis_api_endpoint_status=XISDownstream.ACTIVE)
         # if there are ids as an arg, filter to only those ids
-        if ('id' in options and options['id']):
+        if('id' in options and options['id']):
             downstream_apis = downstream_apis.filter(pk__in=options['id'])
         # if there are apis as an arg, filter to only those apis
-        if ('api' in options and options['api']):
+        if('api' in options and options['api']):
             downstream_apis = downstream_apis.filter(
                 xis_api_endpoint__in=options['api'])
 
@@ -109,7 +109,7 @@ def handle(self, *args, **options):
             queryset = ds.apply_filter().values()
             # get the fields that should be included/excluded in records
             include, exclude = ds.determine_fields()
-            if (include):
+            if(include):
                 for record in queryset:
                     metadata = self.__add_fields(include, record)
                     metadata = self.__remove_fields(exclude, metadata)

app/core/management/utils/xis_internal.py

@@ -1,4 +1,3 @@
-import html
 import logging
 
 import bleach
@@ -11,35 +10,44 @@
 def required_recommended_logs(id_num, category, field):
     """logs the missing required and recommended """
 
+    RECORD = "Record"
+
     # Logs the missing required columns
     if category == 'Required':
         logger.error(
-            "Record " + str(
-                id_num) + " does not have all " + category +
-            " fields. "
-            + field + " field is empty")
-
+            "%s %s does not have all %s fields. %s field is empty",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
     # Logs the missing recommended columns
     if category == 'Recommended':
         logger.warning(
-            "Record " + str(
-                id_num) + " does not have all " + category +
-            " fields. "
-            + field + " field is empty")
-
+            "%s %s does not have all %s fields. %s field is empty",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
     # Logs the inaccurate datatype columns
     if category == 'datatype':
         logger.warning(
-            "Record " + str(
-                id_num) + " does not have the expected " + category +
-            " for the field " + field)
-
+            "%s %s does not have the expected %s for the field %s",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
+    # Logs the prefered alias during homoglyph check
     if category == 'homoglyphs':
         logger.error(
-            "Record " + str(
-                id_num) + " does not have the expected " + "preferred aliases "
-                                                           "for the field " +
-                                                           field)
+            "%s %s does not have the expected "
+            "preferred aliases for the field %s",
+            RECORD,
+            id_num,
+            field
+        )
 
 
 def dict_flatten(data_dict, required_column_list):
@@ -213,10 +221,10 @@ def bleach_data_to_json(rdata):
         # if string, clean
         if isinstance(rdata[key], str):
             rdata[key] = bleach.clean(rdata[key], tags={}, strip=True)
-            rdata[key] = html.unescape(rdata[key])
         # if dict, enter dict
         if isinstance(rdata[key], dict):
             rdata[key] = bleach_data_to_json(rdata[key])
+
     return rdata