feat!: HDRN implementation & associated features with breaking changes to groups & perms

.github/workflows/testing-pipline.yml

@@ -43,9 +43,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: |
-          sudo apt-get update
-          sudo apt-get install -y libgconf-2-4 libatk1.0-0 libatk-bridge2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev libasound2 
-          wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+            sudo apt-get update
+            sudo apt-get install -y libgconf-2-4 libatk1.0-0 libatk-bridge2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev libasound2
       - uses: browser-actions/setup-chrome@v1
       - uses: actions/cache@v3
         with:
@@ -65,6 +64,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
+          sudo apt-get install -y -q dirmngr
           python -m pip install --upgrade pip
           pip install --upgrade --upgrade-strategy eager --default-timeout 100  -r docker/requirements/test.txt
 

CodeListLibrary_project/clinicalcode/admin.py

@@ -8,9 +8,12 @@
 from .models.GenericEntity import GenericEntity
 from .models.Template import Template
 from .models.OntologyTag import OntologyTag
+from .models.Organisation import Organisation
 from .models.DMD_CODES import DMD_CODES
+
 from .forms.TemplateForm import TemplateAdminForm
 from .forms.EntityClassForm import EntityAdminForm
+from .forms.OrganisationForms import OrganisationAdminForm, OrganisationMembershipInline, OrganisationAuthorityInline
 
 @admin.register(OntologyTag)
 class OntologyTag(admin.ModelAdmin):
@@ -51,10 +54,9 @@ def save_model(self, request, obj, form, change):
 
 @admin.register(Brand)
 class BrandAdmin(admin.ModelAdmin):
-    list_display = ['name', 'id', 'logo_path', 'owner', 'description']
-    list_filter = ['name', 'description', 'created', 'modified', 'owner']
+    list_filter = ['name', 'description', 'created', 'modified']
+    list_display = ['name', 'id', 'logo_path', 'description']
     search_fields = ['name', 'id', 'description']
-    exclude = ['created_by', 'updated_by']
 
 
 @admin.register(DataSource)
@@ -72,6 +74,47 @@ class CodingSystemAdmin(admin.ModelAdmin):
     search_fields = ['name', 'codingsystem_id', 'description']
     exclude = []
 
+@admin.register(Organisation)
+class OrganisationAdmin(admin.ModelAdmin):
+  """
+    Organisation admin representation
+  """
+  form = OrganisationAdminForm
+  inlines = [OrganisationMembershipInline, OrganisationAuthorityInline]
+  #exclude = ['created', 'owner', 'members', 'brands']
+
+  list_filter = ['id', 'name']
+  search_fields = ['id', 'name']
+  list_display = ['id', 'name', 'slug']
+  prepopulated_fields = {'slug': ['name']}
+
+  def get_form(self, request, obj=None, **kwargs):
+    """
+      Responsible for pre-populating form data & resolving the associated model form
+
+      Args:
+        request (RequestContext): the request context of the form
+        obj (dict|None): an Organisation model instance (optional; defaults to `None`)
+        **kwargs (**kwargs): arbitrary form key-value pair data
+
+      Returns:
+        (OrganisationModelForm) - the prepared ModelForm instance
+    """
+    form = super(OrganisationAdmin, self).get_form(request, obj, **kwargs)
+
+    if obj is None:
+      form.base_fields['slug'].initial = ''
+      form.base_fields['created'].initial = timezone.now()
+    else:
+      form.base_fields['slug'].initial = obj.slug
+      form.base_fields['created'].initial = obj.created
+
+    form.base_fields['slug'].disabled = True
+    form.base_fields['slug'].help_text = 'This field is not editable'
+    form.base_fields['created'].disabled = True
+    form.base_fields['created'].help_text = 'This field is not editable'
+
+    return form
 
 @admin.register(Template)
 class TemplateAdmin(admin.ModelAdmin):

CodeListLibrary_project/clinicalcode/api/urls.py

@@ -139,6 +139,9 @@ def get_schema(self, request=None, public=False):
     url(r'^data-sources/$', 
         DataSource.get_datasources, 
         name='data_sources'),
+    url(r'^data-sources/(?P<datasource_id>[\d-]+)/export/$',
+        DataSource.get_datasource_internal_detail,
+        name='data_source_by_internal_id'),
     url(r'^data-sources/(?P<datasource_id>[\w-]+)/detail/$',
         DataSource.get_datasource_detail,
         name='data_source_by_id'),

CodeListLibrary_project/clinicalcode/api/views/Collection.py

@@ -1,31 +1,46 @@
+from rest_framework import status
+from django.db.models import F, Q
+from rest_framework.response import Response
 from rest_framework.decorators import (api_view, permission_classes)
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
-from rest_framework.response import Response
-from rest_framework import status
-from django.db.models import F
+from django.contrib.postgres.search import TrigramWordSimilarity
 
 from ...models import Tag, GenericEntity
-from ...entity_utils import api_utils
-from ...entity_utils import constants
+from ...entity_utils import constants, gen_utils, api_utils
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticatedOrReadOnly])
 def get_collections(request):
     """
-        Get all collections
+        Get all Collections
+
+        Available parameters:
+
+        | Param         | Type            | Default | Desc                                                          |
+        |---------------|-----------------|---------|---------------------------------------------------------------|
+        | search        | `str`           | `NULL`  | Full-text search across _name_ field                   |
+        | id            | `int/list[int]` | `NULL`  | Match by a single `int` _id_ field, or match by array overlap |
     """
-    collections = Tag.objects.filter(
-        tag_type=constants.TAG_TYPE.COLLECTION.value
-    ) \
-    .order_by('id')
-
-    result = collections.annotate(
-        name=F('description')
-    ) \
-    .values('id', 'name')
+    search = request.query_params.get('search', '')
+
+    collections = Tag.get_brand_records_by_request(request, params={ 'tag_type': 2 })
+    if collections is not None:
+        if not gen_utils.is_empty_string(search) and len(search.strip()) > 1:
+            collections = collections.annotate(
+                    similarity=TrigramWordSimilarity(search, 'description')
+                ) \
+                .filter(Q(similarity__gte=0.7)) \
+                .order_by('-similarity')
+        else:
+            collections = collections.order_by('id')
+
+        collections = collections.annotate(
+            name=F('description')
+        ) \
+        .values('id', 'name')
 
     return Response(
-        data=list(result),
+        data=collections.values('id', 'name'),
         status=status.HTTP_200_OK
     )
 
@@ -36,8 +51,11 @@ def get_collection_detail(request, collection_id):
         Get detail of specified collection by collection_id, including associated
           published entities
     """
-    collection = Tag.objects.filter(id=collection_id)
-    if not collection.exists():
+    collection = Tag.get_brand_assoc_queryset(request.BRAND_OBJECT, 'collections')
+    if collection is not None:
+        collection = collection.filter(id=collection_id)
+
+    if not collection or not collection.exists():
         return Response(
             data={
                 'message': 'Collection with id does not exist'

CodeListLibrary_project/clinicalcode/api/views/Concept.py

@@ -119,7 +119,7 @@ def get_concepts(request):
         query_clauses.append(psycopg2.sql.SQL('''(
             setweight(to_tsvector('pg_catalog.english', coalesce(historical.name,'')), 'A') ||
             setweight(to_tsvector('pg_catalog.english', coalesce(historical.description,'')), 'B')
-        ) @@ to_tsquery('pg_catalog.english', replace(websearch_to_tsquery('pg_catalog.english', %(search_query)s)::text || ':*', '<->', '|'))
+        ) @@ to_tsquery('pg_catalog.english', replace(to_tsquery('pg_catalog.english', concat(regexp_replace(trim(%(search_query)s), '\W+', ':* & ', 'gm'), ':*'))::text, '<->', '|'))
         '''))
 
     # Resolve pagination behaviour
@@ -422,7 +422,7 @@ def get_concept_detail(request, concept_id, version_id=None, export_codes=False,
     if not user_can_access:
         return Response(
             data={
-                'message': 'Concept version must be published or you must have permission to access it'
+                'message': 'Entity version must be published or you must have permission to access it'
             }, 
             content_type='json',
             status=status.HTTP_401_UNAUTHORIZED

CodeListLibrary_project/clinicalcode/api/views/DataSource.py

@@ -1,39 +1,159 @@
-from rest_framework.decorators import (api_view, permission_classes)
-from rest_framework.permissions import IsAuthenticatedOrReadOnly
-from rest_framework.response import Response
 from rest_framework import status
-from django.db.models import Subquery, OuterRef
+from django.db.models import Q, Subquery, OuterRef
+from rest_framework.response import Response
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
+from django.contrib.postgres.search import TrigramWordSimilarity
 
 from ...models import DataSource, Template, GenericEntity
-from ...entity_utils import api_utils
-from ...entity_utils import gen_utils
-from ...entity_utils import constants
+from ...entity_utils import api_utils, gen_utils, constants
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticatedOrReadOnly])
 def get_datasources(request):
     """
-        Get all datasources
+        Get all DataSources
+
+        Available parameters:
+
+        | Param         | Type            | Default | Desc                                                          |
+        |---------------|-----------------|---------|---------------------------------------------------------------|
+        | search        | `str`           | `NULL`  | Full-text search across _name_ and _description_ fields       |
+        | id            | `int/list[int]` | `NULL`  | Match by a single `int` _id_ field, or match by array overlap |
+        | name          | `str`           | `NULL`  | Case insensitive direct match of _name_ field                 |
+        | uid           | `str/uuid`      | `NULL`  | Case insensitive direct match of _uid_ field                  |
+        | datasource_id | `int`           | `NULL`  | Match by exact _datasource_id_                                |
+        | url           | `str`           | `NULL`  | Case insensitive direct match of _url_ field                  |
+        | source        | `str`           | `NULL`  | Case insensitive direct match of _source_ field               |
     """
-    datasources = DataSource.objects.all().order_by('id')
-    datasources = list(datasources.values('id', 'name', 'url', 'uid', 'source'))
+    params = gen_utils.parse_model_field_query(DataSource, request, ignored_fields=['description'])
+    if params is not None:
+        datasources = DataSource.objects.filter(**params)
+    else:
+        datasources = DataSource.objects.all()
+
+    search = request.query_params.get('search')
+    if not gen_utils.is_empty_string(search) and len(search.strip()) > 3:
+        datasources = datasources.annotate(
+                similarity=(
+                    TrigramWordSimilarity(search, 'name') + \
+                    TrigramWordSimilarity(search, 'description')
+                )
+            ) \
+            .filter(Q(similarity__gte=0.7)) \
+            .order_by('-similarity')
+    else:
+        datasources = datasources.order_by('id')
+
     return Response(
-        data=datasources,
+        data=datasources.values('id', 'name', 'description', 'url', 'uid', 'datasource_id', 'source'),
+        status=status.HTTP_200_OK
+    )
+
+@api_view(['GET'])
+@permission_classes([IsAuthenticatedOrReadOnly])
+def get_datasource_internal_detail(request, datasource_id):
+    """
+        Get detail of specified datasource by by its internal Id
+    """
+    query = None
+    if gen_utils.parse_int(datasource_id, default=None) is not None:
+        query = { 'id': int(datasource_id) }
+
+    if not query:
+        return Response(
+            data={
+                'message': 'Invalid id, expected int-like value'
+            },
+            content_type='json',
+            status=status.HTTP_400_BAD_REQUEST
+        )
+
+    datasource = DataSource.objects.filter(**query)
+    if not datasource.exists():
+        return Response(
+            data={
+                'message': 'Datasource with this internal Id does not exist'
+            },
+            content_type='json',
+            status=status.HTTP_404_NOT_FOUND
+        )
+
+    datasource = datasource.first()
+
+    # Get all templates and their versions where data_sources exist
+    templates = Template.history.filter(
+        definition__fields__has_key='data_sources'
+    ) \
+    .annotate(
+        was_deleted=Subquery(
+            Template.history.filter(
+                id=OuterRef('id'),
+                history_date__gte=OuterRef('history_date'),
+                history_type='-'
+            )
+            .order_by('id', '-history_id')
+            .distinct('id')
+            .values('id')
+        )
+    ) \
+    .exclude(was_deleted__isnull=False) \
+    .order_by('id', '-template_version', '-history_id') \
+    .distinct('id', 'template_version')
+
+    template_ids = list(templates.values_list('id', flat=True))
+    template_versions = list(templates.values_list('template_version', flat=True))
+
+    # Get all published entities with this datasource
+    entities = GenericEntity.history.filter(
+        template_id__in=template_ids,
+        template_version__in=template_versions,
+        publish_status=constants.APPROVAL_STATUS.APPROVED.value
+    ) \
+    .extra(where=[f"""
+        exists(
+            select 1
+            from jsonb_array_elements(
+                case jsonb_typeof(template_data->'data_sources') when 'array' 
+                    then template_data->'data_sources' 
+                    else '[]' 
+                end
+            ) as val
+            where val in ('{datasource.id}')
+        )"""
+    ]) \
+    .order_by('id', '-history_id') \
+    .distinct('id')
+
+    # Format results
+    entities = api_utils.annotate_linked_entities(entities)
+
+    result = {
+        'id': datasource.id,
+        'name': datasource.name,
+        'url': datasource.url,
+        'uid': datasource.uid,
+        'description': datasource.description,
+        'source': datasource.source,
+        'phenotypes': list(entities)
+    }
+
+    return Response(
+        data=result,
         status=status.HTTP_200_OK
     )
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticatedOrReadOnly])
 def get_datasource_detail(request, datasource_id):
     """
-        Get detail of specified datasource by datasource_id (id or HDRUK UUID for
-          linkage between applications), including associated published entities
+        Get detail of specified datasource by `datasource_id`, _i.e._ the HDRUK DataSource `pid` or its `UUID` for linkage between applications, including associated published entities.
     """
     query = None
     if gen_utils.is_valid_uuid(datasource_id):
         query = { 'uid': datasource_id }
     elif gen_utils.parse_int(datasource_id, default=None) is not None:
-        query = { 'id': int(datasource_id) }
+        query = { 'datasource_id': int(datasource_id) }
 
     if not query:
         return Response(