Latest sync with P1 master branch

.github/workflows/cd-workflow.yml

@@ -47,46 +47,3 @@ jobs:
         echo "Docker network successfully created"
         echo "Running coverage unit test"
         docker compose --env-file ./.env run app sh -c "python manage.py waitdb && coverage run manage.py test --tag=unit && flake8 && coverage report && coverage report --fail-under=80"
-
-  # sonarcloud:
-  #     name: SonarCloud
-  #     runs-on: ubuntu-latest
-  #     steps:
-  #       - uses: actions/checkout@v2
-  #         with:
-  #           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-  #       - name: SonarCloud Scan
-  #         uses: SonarSource/sonarcloud-github-action@master
-  #         env:
-  #           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-  #           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-
-  build:
-    # require dependency from step above
-    needs: code-test
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: ${{ secrets.AWS_REGION }}
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v1
-      with:
-        mask-password: 'true'
-    - name: Build, tag, and push image to Amazon ECR
-      env:
-        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        ECR_REPOSITORY: ${{ secrets.ECR_REPO }}
-        IMAGE_TAG: latest
-      run: |
-        echo "Starting docker build"
-        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-        echo "Pushing image to ECR..."
-        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

README.md

@@ -43,6 +43,7 @@ git clone https://github.com/OpenLXP/openlxp-xss.git
 | AWS_ACCESS_KEY_ID         | The Access Key ID for AWS  |
 | AWS_SECRET_ACCESS_KEY     | The Secret Access Key for AWS  |
 | AWS_DEFAULT_REGION        | The region for AWS |
+| CORS_ALLOWED_ORIGINS      | List of trusted origins that are allowed to make cross-origin requests to the server |
 | DB_HOST                   | The host name, IP, or docker container name of the database |
 | DB_NAME                   | The name to give the database |
 | DB_PASSWORD               | The password for the user to access the database |

app/core/admin.py

@@ -90,7 +90,7 @@ class TermAdmin(admin.ModelAdmin):
                     'modified', )
     fieldsets = (
         (None, {'fields': ('iri', 'name', 'uuid', 'description', 'status',)}),
-        ('Info', {'fields': ('data_type', 'use',
+        ('Info', {'fields': ('data_type', 'use', 'learning_type',
                              'multiple_expected',
                              'source',)}),
         ('Connections', {'fields': ('term_set', 'mapping',)}),

app/core/models.py

@@ -34,6 +34,10 @@
                r'| \xF1-\xF3{3} # planes 4-15 '
                r'| \xF4\x80-\x8F{2} # plane 16 )*\Z))')
 
+source_tag = 'ldss:'
+graph_tag = '@graph'
+context_tag = '@context'
+
 
 def validate_version(value):
     check = re.fullmatch('[0-9]*[.][0-9]*[.][0-9]*', value)
@@ -79,33 +83,33 @@ def json_ld(self):
         graph = {}
         context = {}
         # add elements to graph and context
-        graph['@id'] = 'ldss:' + self.iri
+        graph['@id'] = source_tag + self.iri
         graph['@type'] = 'rdfs:Class'
         graph['rdfs:label'] = self.name
         context['rdfs'] = 'http://www.w3.org/2000/01/rdf-schema#'
         if hasattr(self, 'childtermset'):
             graph['schema:domainIncludes'] = {
-                '@id': 'ldss:' +
+                '@id': source_tag +
                 self.childtermset.parent_term_set.iri}
             context['schema'] = 'https://schema.org/'
         # iterate over child term sets and collect their graphs and contexts
         children = []
         for kid in self.children.filter(status='published'):
             kid_ld = kid.json_ld()
-            children.extend(kid_ld['@graph'])
+            children.extend(kid_ld[graph_tag])
             # add children's context to current context, but current has
             # higher priority
-            context = {**kid_ld['@context'], **context}
+            context = {**kid_ld[context_tag], **context}
         # iterate over terms and collect their graphs and contexts
         terms = []
         for term in self.terms.filter(status='published'):
             term_ld = term.json_ld()
-            terms.extend(term_ld['@graph'])
+            terms.extend(term_ld[graph_tag])
             # add terms' context to current context, but current has higher
             # priority
-            context = {**term_ld['@context'], **context}
+            context = {**term_ld[context_tag], **context}
         # return the graph and context
-        return {'@context': context, '@graph': [graph, *children, *terms]}
+        return {context_tag: context, graph_tag: [graph, *children, *terms]}
 
     def mapped_to(self, target_root):
         """Return dict of Terms mapped to anything in target_root string"""
@@ -149,13 +153,18 @@ class Term(TimeStampedModel):
                    ('Optional', 'Optional'),
                    ('Recommended', 'Recommended'),
                    ]
+    TYPE_CHOICES = [('Learning Resource', 'Learning Resource'),
+                    ('Learning Event', 'Learning Event'),
+                    ('Both', 'Both'),
+                    ]
     name = models.SlugField(max_length=255, allow_unicode=True)
     description = models.TextField(null=True, blank=True)
     iri = models.SlugField(max_length=255, unique=True,
                            allow_unicode=True, primary_key=True)
     uuid = models.UUIDField(default=uuid4, editable=False, unique=True)
     data_type = models.CharField(max_length=255, null=True, blank=True)
     use = models.CharField(max_length=255, choices=USE_CHOICES)
+    # multiple_expected fields added in migration 0008
     multiple_expected = models.BooleanField(default=True,
                                             help_text="Whether multiple"
                                             " values "
@@ -203,7 +212,7 @@ def json_ld(self):
         graph = {}
         context = {}
         # add elements to graph and context
-        graph['@id'] = 'ldss:' + self.iri
+        graph['@id'] = source_tag + self.iri
         graph['@type'] = 'rdf:Property'
         if self.description is not None and len(self.description.strip()) > 0:
             graph['rdfs:comment'] = self.description
@@ -213,15 +222,15 @@ def json_ld(self):
                 '@id': data_type_matching[self.data_type]}
         if self.mapping.exists():
             graph['owl:equivalentProperty'] = [
-                {'@id': 'ldss:' + alt.iri} for alt in self.mapping.all()]
+                {'@id': source_tag + alt.iri} for alt in self.mapping.all()]
             context['owl'] = 'http://www.w3.org/2002/07/owl#'
         graph['rdfs:label'] = self.name
         graph['schema:domainIncludes'] = {'@id': 'ldss:' + self.term_set.iri}
         context['schema'] = 'https://schema.org/'
         context['rdfs'] = 'http://www.w3.org/2000/01/rdf-schema#'
         context['rdf'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
         # return the graph and context
-        return {'@context': context, '@graph': [graph, ]}
+        return {context_tag: context, graph_tag: [graph, ]}
 
     def path(self):
         """Get the path of the Term"""
@@ -294,7 +303,8 @@ def clean(self):
 
         if self.schema_file:
             # scan file for malicious payloads
-            cd = clamd.ClamdUnixSocket()
+            cd = clamd.ClamdNetworkSocket(host="clamd.clamav", port=3310,
+                                          timeout=10)
             json_file = self.schema_file
             scan_results = cd.instream(json_file)['stream']
             if 'OK' not in scan_results:
@@ -391,13 +401,15 @@ def clean(self):
         if self.schema_mapping_file:
             json_file = self.schema_mapping_file
             # scan file for malicious payloads
-            cd = clamd.ClamdUnixSocket()
+            cd = clamd.ClamdNetworkSocket(host="clamd.clamav",
+                                          port=3310, timeout=10)
             scan_results = cd.instream(json_file)['stream']
             if 'OK' not in scan_results:
                 for issue_type, issue in [scan_results, ]:
                     logger.error(
                         '%s %s in transform %s to %s',
-                        issue_type, issue, self.source_schema.iri, self.target_schema.iri  # noqa: E501
+                        issue_type, issue, self.source_schema.iri,
+                        self.target_schema.iri  # noqa: E501
                     )
             # only load json if no issues found
             else:

app/core/tests/test_models_unit.py

@@ -71,6 +71,7 @@ def test_schema_ledger_virus(self):
                 patch('core.models.clamd') as clam:
             clam.instream.return_value = {'stream': ('BAD', 'EICAR')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(schema.version, '')
             self.assertEqual(schema.schema_file.size, len(EICAR))
@@ -112,6 +113,7 @@ def test_schema_ledger_non_json(self):
             magic.from_file.return_value = 'text/plain'
             clam.instream.return_value = {'stream': ('OK', 'OKAY')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(schema.version, '')
             self.assertEqual(schema.schema_file.size, len(file_contents))
@@ -153,6 +155,7 @@ def test_schema_ledger_bleach(self):
             magic.from_file.return_value = 'application/json'
             clam.instream.return_value = {'stream': ('OK', 'OKAY')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(schema.version, '')
             self.assertEqual(schema.schema_file.size, len(file_contents))
@@ -207,6 +210,7 @@ def test_transformation_ledger_virus(self):
                 patch('core.models.clamd') as clam:
             clam.instream.return_value = {'stream': ('BAD', 'EICAR')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(mapping.schema_mapping_file.size, len(EICAR))
             mapping.clean()
@@ -242,6 +246,7 @@ def test_transformation_ledger_non_json(self):
             magic.from_file.return_value = 'text/plain'
             clam.instream.return_value = {'stream': ('OK', 'OKAY')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(mapping.schema_mapping_file.size,
                              len(file_contents))
@@ -279,6 +284,7 @@ def test_transformation_ledger_bleach(self):
             magic.from_file.return_value = 'application/json'
             clam.instream.return_value = {'stream': ('OK', 'OKAY')}
             clam.ClamdUnixSocket.return_value = clam
+            clam.ClamdNetworkSocket.return_value = clam
 
             self.assertEqual(mapping.schema_mapping_file.size,
                              len(file_contents))

app/openlxp_xss_project/settings.py

@@ -26,11 +26,23 @@
 SECRET_KEY = os.environ.get('SECRET_KEY_VAL')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 mimetypes.add_type("text/css", ".css", True)
 
-ALLOWED_HOSTS = ['*']
+ALLOWED_HOSTS = os.environ.get('HOSTS').split(';')
+
+# Content Security Policy (CSP)
+SELF_VALUE = "'self'"  # defining a constant
+IMG_DATA_VALUE = "data:"
+
+CSP_DEFAULT_SRC = (SELF_VALUE)
+CSP_SCRIPT_SRC = (SELF_VALUE,)
+CSP_IMG_SRC = (SELF_VALUE, IMG_DATA_VALUE)
+CSP_STYLE_SRC = (SELF_VALUE)
+CSP_FRAME_SRC = (SELF_VALUE,)
+CSP_FONT_SRC = (SELF_VALUE,)
+
 
 # Application definition
 
@@ -49,7 +61,7 @@
     'rest_framework.authtoken',
     'core.apps.CoreConfig',
     'api',
-    'health_check', 
+    'health_check',
     'users',
     'social_django',
     'openlxp_authentication',
@@ -64,8 +76,19 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.contrib.admindocs.middleware.XViewMiddleware',
+    'csp.middleware.CSPMiddleware',
 ]
 
+
+# CORS_ALLOWED_ORIGINS = [os.environ.get('CORS_ALLOWED_ORIGINS')]
+# CORS_ALLOW_CREDENTIALS = True
+
+SESSION_COOKIE_SECURE = True
+SECURE_BROWSER_XSS_FILTER = True
+SECURE_HSTS_SECONDS = 31536000
+SECURE_HSTS_INCLUDE_SUBDOMAINS = True
+
+
 ROOT_URLCONF = 'openlxp_xss_project.urls'
 
 TEMPLATES = [
@@ -188,10 +211,14 @@
     ],
 }
 
-CSRF_COOKIE_DOMAIN = '.deloitteopenlxp.com'
-CSRF_TRUSTED_ORIGINS = ['https://dev-ldss.deloitteopenlxp.com', ]
+CSRF_COOKIE_HTTPONLY = True
+CSRF_COOKIE_SECURE = True
+if os.environ.get('CSRF_TRUSTED_ORIGINS'):
+    CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS').split(';')
+else:
+    CSRF_TRUSTED_ORIGINS = ['http://localhost:8010']
 
-SECURE_SSL_REDIRECT = False
+# SECURE_SSL_REDIRECT = True
 
 AUTHENTICATION_BACKENDS = (
     'django.contrib.auth.backends.ModelBackend',