Merge pull request #99 from OpenLXP/code-sync

Latest sync with P1 master branch

Author: sammy-sandhu

.github/workflows/cd-workflows.yml

@@ -51,50 +51,3 @@ jobs:
         echo "Docker network successfully created"
         echo "Running coverage unit test"
         docker compose --env-file ./.env run app_xis sh -c "python manage.py waitdb && coverage run manage.py test --tag=unit && flake8 && coverage report && coverage report --fail-under=80"
-
-  # sonarcloud:
-  #   name: SonarCloud
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-  #     - name: SonarCloud Scan
-  #       uses: SonarSource/sonarcloud-github-action@master
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-  #         SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-    
-  build:
-    # requires dependency from step above
-    needs: code-test
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: ${{ secrets.AWS_REGION }}
-
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v1
-      with:
-        mask-password: 'true'
-
-    - name: Build, tag, and push image to Amazon ECR
-      env:
-        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        ECR_REPOSITORY: ${{ secrets.ECR_REPO }}
-        IMAGE_TAG: latest
-      run: |
-        echo "Starting docker build"
-        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-        echo "Pushing image to ECR..."
-        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

app/api/serializers.py

@@ -102,7 +102,6 @@ def validate(self, data):
         # validate for recommended values in data
         validate_recommended(
             data, recommended_column_list, flattened_source_data)
-
         # Type checking for values in metadata
         for item in flattened_source_data:
             # check if datatype has been assigned to field
@@ -139,6 +138,7 @@ def validate(self, data):
             raise serializers.ValidationError("Metadata has missing fields. "
                                               "Data did not pass validation."
                                               "Check logs for more details")
+
         return data
 
     def update(self, instance, validated_data):

app/api/views.py

@@ -113,6 +113,8 @@ def filter_queryset(self, queryset):
     def post(self, request):
         """This method defines the API's to save data to the
         metadata ledger in the XIS"""
+        logger.info("Start processing")
+        logger.error("Incoming experience")
 
         # Add optional/recommended fields to the metadata
         extra_fields = get_optional_and_recommended_fields_for_validation()
@@ -346,31 +348,31 @@ def post(self, request, provider_id, experience_id):
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_workflow_api(request):
     logger.info('XIS workflow api')
     task = xis_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_downstream_workflow_api(request):
     logger.info('Downstream workflow api')
     task = xis_downstream_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def xis_upstream_workflow_api(request):
     logger.info('Upstream workflow api')
     task = xis_upstream_workflow.delay()
     return JsonResponse({"task_id": task.id}, status=status.HTTP_202_ACCEPTED)
 
 
 @api_view(['GET'])
-@permission_classes((permissions.AllowAny,))
+@permission_classes((permissions.IsAdminUser,))
 def get_status(request, task_id):
     task_result = AsyncResult(task_id)
     result = {

app/core/fixtures/admin_theme_data.json

@@ -41,7 +41,7 @@
             "css_delete_button_background_color": "#BA2121",
             "css_delete_button_background_hover_color": "#A41515",
             "css_delete_button_text_color": "#FFFFFF",
-            "related_modal_active": true,
+            "related_modal_active": false,
             "related_modal_background_color": "#000000",
             "related_modal_background_opacity": "0.3",
             "related_modal_rounded_corners": true,

app/core/management/commands/load_metadata_from_xis.py

@@ -35,20 +35,21 @@ def retrieve_records(self, upstream):
 
         xis_response = requests.get(
             url=upstream.xis_api_endpoint + 'metadata/',
-            headers=headers)
+            headers=headers, timeout=3.0)
 
         while (xis_response.status_code//10 == 20):
             for record in xis_response.json()['results']:
                 self.save_record(upstream, bleach_data_to_json(record))
 
             if (xis_response.json()['next'] is not None):
                 xis_response = requests.get(
-                    url=xis_response.json()['next'], headers=headers)
+                    url=xis_response.json()['next'], headers=headers,
+                    timeout=3.0)
             else:
                 return
 
         logger.error(
-            f"HTTP Error {xis_response.status_code} from {upstream}")
+            "HTTP Error %s from %s", xis_response.status_code, upstream)
 
     def save_record(self, upstream, record):
         """saves record to metadata and supplemental ledgers as needed"""

app/core/management/commands/load_metadata_into_xis.py

@@ -76,9 +76,9 @@ def send_record(self, downstream, record):
             url=f'{downstream.xis_api_endpoint}managed-data/catalogs/'
             f'{record["provider_name"]}/{record["metadata_key_hash"]}',
             data=json.dumps(CompositeLedgerSerializer(record).data),
-            headers=headers)
+            headers=headers, timeout=3.0)
 
-        if (xis_response.status_code//10 == 20):
+        if(xis_response.status_code//10 == 20):
             downstream.composite_experiences.add(
                 record['unique_record_identifier'])
         else:
@@ -96,10 +96,10 @@ def handle(self, *args, **options):
         downstream_apis = XISDownstream.objects.all().filter(
             xis_api_endpoint_status=XISDownstream.ACTIVE)
         # if there are ids as an arg, filter to only those ids
-        if ('id' in options and options['id']):
+        if('id' in options and options['id']):
             downstream_apis = downstream_apis.filter(pk__in=options['id'])
         # if there are apis as an arg, filter to only those apis
-        if ('api' in options and options['api']):
+        if('api' in options and options['api']):
             downstream_apis = downstream_apis.filter(
                 xis_api_endpoint__in=options['api'])
 
@@ -109,7 +109,7 @@ def handle(self, *args, **options):
             queryset = ds.apply_filter().values()
             # get the fields that should be included/excluded in records
             include, exclude = ds.determine_fields()
-            if (include):
+            if(include):
                 for record in queryset:
                     metadata = self.__add_fields(include, record)
                     metadata = self.__remove_fields(exclude, metadata)

app/core/management/utils/xis_internal.py

@@ -1,4 +1,3 @@
-import html
 import logging
 
 import bleach
@@ -11,35 +10,44 @@
 def required_recommended_logs(id_num, category, field):
     """logs the missing required and recommended """
 
+    RECORD = "Record"
+
     # Logs the missing required columns
     if category == 'Required':
         logger.error(
-            "Record " + str(
-                id_num) + " does not have all " + category +
-            " fields. "
-            + field + " field is empty")
-
+            "%s %s does not have all %s fields. %s field is empty",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
     # Logs the missing recommended columns
     if category == 'Recommended':
         logger.warning(
-            "Record " + str(
-                id_num) + " does not have all " + category +
-            " fields. "
-            + field + " field is empty")
-
+            "%s %s does not have all %s fields. %s field is empty",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
     # Logs the inaccurate datatype columns
     if category == 'datatype':
         logger.warning(
-            "Record " + str(
-                id_num) + " does not have the expected " + category +
-            " for the field " + field)
-
+            "%s %s does not have the expected %s for the field %s",
+            RECORD,
+            id_num,
+            category,
+            field
+        )
+    # Logs the prefered alias during homoglyph check
     if category == 'homoglyphs':
         logger.error(
-            "Record " + str(
-                id_num) + " does not have the expected " + "preferred aliases "
-                                                           "for the field " +
-                                                           field)
+            "%s %s does not have the expected "
+            "preferred aliases for the field %s",
+            RECORD,
+            id_num,
+            field
+        )
 
 
 def dict_flatten(data_dict, required_column_list):
@@ -213,10 +221,10 @@ def bleach_data_to_json(rdata):
         # if string, clean
         if isinstance(rdata[key], str):
             rdata[key] = bleach.clean(rdata[key], tags={}, strip=True)
-            rdata[key] = html.unescape(rdata[key])
         # if dict, enter dict
         if isinstance(rdata[key], dict):
             rdata[key] = bleach_data_to_json(rdata[key])
+
     return rdata
 
 

app/core/management/utils/xss_client.py

@@ -39,10 +39,10 @@ def read_json_data(schema_ref):
 
 def get_target_validation_schema():
     """Retrieve target validation schema from XIS configuration """
-    logger.info("Configuration of schemas and files")
+    logger.error("Configuration of schemas and files")
     data = XISConfiguration.objects.first()
     target_validation_schema = data.target_schema
-    logger.info("Reading schema for validation")
+    logger.error("Reading schema for validation")
     # Read source validation schema as dictionary
     schema_data_dict = read_json_data(target_validation_schema)
     return schema_data_dict

app/core/migrations/0012_auto_20230517_1537.py

@@ -1,8 +1,8 @@
 # Generated by Django 3.2.19 on 2023-05-17 15:37
 
-import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
+import django.db.models.deletion
 
 
 class Migration(migrations.Migration):

app/core/tests/test_commands_integration.py

@@ -3,14 +3,12 @@
 
 from ddt import ddt
 from django.test import tag
-from elasticsearch import Elasticsearch
 
 from core.management.commands.consolidate_ledgers import (
     check_metadata_ledger_transmission_ready_record,
     put_metadata_ledger_into_composite_ledger)
 from core.management.commands.load_metadata_into_xse import (
     check_records_to_load_into_xse, post_data_to_xse)
-from core.management.utils.xse_client import get_elasticsearch_endpoint
 from core.models import CompositeLedger, MetadataLedger
 
 from .test_setup import TestSetUp
@@ -103,11 +101,6 @@ def test_post_data_to_xse_created(self):
         """Test for POSTing XIS composite_ledger to XSE in JSON format when
             record gets created in XSE"""
 
-        es = Elasticsearch(get_elasticsearch_endpoint())
-
-        if(es.indices.exists('testing_index')):
-            es.indices.delete('testing_index')
-
         self.composite_ledger.save()
 
         data = CompositeLedger.objects.filter(