Skip to content

Commit 0cab0c6

Browse files
Deepthi-ChandDeepthi-Chand
authored
Merge pull request #12 from CivicDataLab/dev
Update main to latest
2 parents 9a2adb5 + 9c595a9 commit 0cab0c6

29 files changed

+1493
-51
lines changed

‎.env.example‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
DB_ENGINE=django.db.backends.postgresql
2+
DB_NAME=postgres
3+
DB_USER=postgres
4+
DB_PASSWORD=postgres
5+
DB_HOST=backend_db
6+
DB_PORT=5432
7+
TELEMETRY_URL=http://otel-collector:4317
8+
ELASTICSEARCH_INDEX=http://elasticsearch:9200
9+
ELASTICSEARCH_USERNAME=elastic
10+
ELASTICSEARCH_PASS=changeme
11+
URL_WHITELIST=http://localhost:8000,http://localhost,http://localhost:3000
12+
DEBUG=True
13+
SECRET_KEY=your-secret-key
14+
REDIS_URL=redis://redis:6379/1

‎.github/workflows/deploy-to-ecs.yml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
name: Deploy to Amazon ECS
2+
3+
on:
4+
push:
5+
branches:
6+
- dev
7+
workflow_dispatch:
8+
9+
env:
10+
AWS_REGION: ${{ secrets.AWS_REGION }}
11+
ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }}
12+
ECS_CLUSTER: ${{ secrets.ECS_CLUSTER }}
13+
ECS_EXECUTION_ROLE_ARN: ${{ secrets.ECS_EXECUTION_ROLE_ARN }}
14+
APP_NAME: dataspace
15+
APP_PORT: 8000
16+
DB_ENGINE: django.db.backends.postgresql
17+
DB_PORT: 5432
18+
DEBUG_MODE: "False"
19+
TELEMETRY_URL: http://otel-collector:4317
20+
CPU_UNITS: 256
21+
MEMORY_UNITS: 512
22+
SSM_PATH_PREFIX: /dataspace
23+
ENVIRONMENT: ${{ secrets.ENVIRONMENT || 'dev' }}
24+
25+
jobs:
26+
deploy-infrastructure:
27+
name: Deploy Infrastructure
28+
runs-on: ubuntu-latest
29+
environment: development
30+
if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.modified, 'aws/cloudformation')
31+
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@v3
35+
36+
- name: Configure AWS credentials
37+
uses: aws-actions/configure-aws-credentials@v1
38+
with:
39+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
40+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
41+
aws-region: ${{ env.AWS_REGION }}
42+
43+
- name: Deploy CloudFormation stack
44+
run: |
45+
aws cloudformation deploy \
46+
--template-file aws/cloudformation/dataspace-infrastructure.yml \
47+
--stack-name dataspace-${{ env.ENVIRONMENT }}-infrastructure \
48+
--parameter-overrides \
49+
Environment=${{ env.ENVIRONMENT }} \
50+
VpcId=${{ secrets.VPC_ID }} \
51+
SubnetIds=${{ secrets.SUBNET_IDS }} \
52+
DBUsername=${{ secrets.DB_USERNAME }} \
53+
DBPassword=${{ secrets.DB_PASSWORD }} \
54+
DBName=${{ secrets.DB_NAME }} \
55+
ElasticsearchPassword=${{ secrets.ELASTICSEARCH_PASSWORD }} \
56+
DjangoSecretKey=${{ secrets.DJANGO_SECRET_KEY }} \
57+
--capabilities CAPABILITY_IAM \
58+
--no-fail-on-empty-changeset
59+
60+
deploy-app:
61+
name: Deploy Application
62+
runs-on: ubuntu-latest
63+
environment: development
64+
needs: deploy-infrastructure
65+
if: always() # Run even if infrastructure deployment is skipped
66+
67+
steps:
68+
- name: Checkout
69+
uses: actions/checkout@v3
70+
71+
- name: Configure AWS credentials
72+
uses: aws-actions/configure-aws-credentials@v1
73+
with:
74+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
75+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
76+
aws-region: ${{ env.AWS_REGION }}
77+
78+
- name: Login to Amazon ECR
79+
id: login-ecr
80+
uses: aws-actions/amazon-ecr-login@v1
81+
82+
- name: Build, tag, and push image to Amazon ECR
83+
id: build-image
84+
env:
85+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
86+
IMAGE_TAG: ${{ github.sha }}
87+
run: |
88+
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
89+
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
90+
echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
91+
92+
- name: Download task definition and get EFS ID
93+
run: |
94+
aws ecs describe-task-definition --task-definition dataspace --query taskDefinition > aws/current-task-definition.json
95+
aws ecs describe-task-definition --task-definition dataspace-otel-collector --query taskDefinition > aws/current-otel-task-definition.json
96+
# Get the EFS ID from CloudFormation export
97+
EFS_ID=$(aws cloudformation list-exports --query "Exports[?Name=='dataspace-${{ env.ENVIRONMENT }}-MigrationsFileSystemId'].Value" --output text)
98+
echo "EFS_ID=$EFS_ID" >> $GITHUB_ENV
99+
100+
- name: Update container image only
101+
id: task-def-app
102+
uses: aws-actions/amazon-ecs-render-task-definition@v1
103+
with:
104+
task-definition: aws/current-task-definition.json
105+
container-name: dataspace
106+
image: ${{ steps.build-image.outputs.image }}
107+
108+
- name: Deploy main application ECS task definition
109+
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
110+
with:
111+
task-definition: ${{ steps.task-def-app.outputs.task-definition }}
112+
service: ${{ secrets.ECS_SERVICE }}
113+
cluster: ${{ env.ECS_CLUSTER }}
114+
wait-for-service-stability: true
115+
116+
deploy-otel:
117+
name: Deploy OpenTelemetry Collector
118+
runs-on: ubuntu-latest
119+
environment: development
120+
needs: deploy-app
121+
122+
steps:
123+
- name: Checkout
124+
uses: actions/checkout@v3
125+
126+
- name: Configure AWS credentials
127+
uses: aws-actions/configure-aws-credentials@v1
128+
with:
129+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
130+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
131+
aws-region: ${{ env.AWS_REGION }}
132+
133+
- name: Download current OpenTelemetry task definition
134+
id: download-otel-taskdef
135+
run: |
136+
aws ecs describe-task-definition \
137+
--task-definition dataspace-otel-collector \
138+
--query taskDefinition > aws/current-otel-task-definition.json
139+
cat aws/current-otel-task-definition.json
140+
141+
- name: Deploy OpenTelemetry ECS task definition
142+
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
143+
with:
144+
task-definition: aws/current-otel-task-definition.json
145+
service: ${{ secrets.ECS_OTEL_SERVICE }}
146+
cluster: ${{ env.ECS_CLUSTER }}
147+
wait-for-service-stability: true

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,7 @@ resources/
155155
.env
156156
api/migrations/*
157157
authorization/migrations/*
158+
159+
160+
# AWS files
161+
aws/.env.*

‎.pre-commit-config.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ repos:
55
- id: trailing-whitespace
66
- id: end-of-file-fixer
77
- id: check-yaml
8+
exclude: ^aws/cloudformation/.*\.yml$
89
- id: check-added-large-files
910
- id: debug-statements
1011

@@ -20,6 +21,17 @@ repos:
2021
- id: isort
2122
args: ["--profile", "black"]
2223

24+
- repo: local
25+
hooks:
26+
- id: cloudformation-validate
27+
name: AWS CloudFormation Validation
28+
description: Validates CloudFormation templates using AWS CLI
29+
entry: bash -c 'aws cloudformation validate-template --template-body file://$0 || exit 1'
30+
language: system
31+
files: ^aws/cloudformation/.*\.yml$
32+
require_serial: true
33+
pass_filenames: true
34+
2335
- repo: https://github.com/pre-commit/mirrors-mypy
2436
rev: v1.9.0
2537
hooks:

‎Dockerfile‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,18 @@ RUN echo 'deb http://archive.debian.org/debian stretch main contrib non-free' >>
1212
WORKDIR /code
1313
COPY . /code/
1414

15-
RUN pip install psycopg2-binary
15+
RUN pip install psycopg2-binary uvicorn
1616
RUN pip install -r requirements.txt
17-
#RUN python manage.py migrate
17+
18+
# Create healthcheck script
19+
RUN echo '#!/bin/bash\nset -e\npython -c "import sys; import django; django.setup(); sys.exit(0)"' > /code/healthcheck.sh \
20+
&& chmod +x /code/healthcheck.sh
1821

1922

2023
EXPOSE 8000
21-
#CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
24+
25+
# Make entrypoint script executable
26+
RUN chmod +x /code/docker-entrypoint.sh
27+
28+
ENTRYPOINT ["/code/docker-entrypoint.sh"]
29+
CMD ["uvicorn", "DataSpace.asgi:application", "--host", "0.0.0.0", "--port", "8000"]

‎api/activities/decorators.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def decorator(func: F) -> F:
3939
def wrapper(*args: Any, **kwargs: Any) -> Any:
4040
# Extract request from args (typically the first or second argument in view functions)
4141
request = None
42-
for arg in args:
42+
for arg in list(args) + list(kwargs.values()):
4343
if isinstance(arg, HttpRequest):
4444
request = arg
4545
break

‎api/models/UseCase.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class UseCase(models.Model):
5454
)
5555
started_on = models.DateField(blank=True, null=True)
5656
completed_on = models.DateField(blank=True, null=True)
57+
platform_url = models.URLField(blank=True, null=True)
5758

5859
def save(self, *args: Any, **kwargs: Any) -> None:
5960
if self.title and not self.slug:

‎api/schema/dataset_schema.py‎

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
ResourceChartDetails,
1919
ResourceChartImage,
2020
Sector,
21+
UseCase,
2122
)
2223
from api.models.Dataset import Tag
2324
from api.models.DatasetMetadata import DatasetMetadata
@@ -31,7 +32,12 @@
3132
from api.types.type_organization import TypeOrganization
3233
from api.types.type_resource_chart import TypeResourceChart
3334
from api.types.type_resource_chart_image import TypeResourceChartImage
34-
from api.utils.enums import DatasetAccessType, DatasetLicense, DatasetStatus
35+
from api.utils.enums import (
36+
DatasetAccessType,
37+
DatasetLicense,
38+
DatasetStatus,
39+
UseCaseStatus,
40+
)
3541
from api.utils.graphql_telemetry import trace_resolver
3642
from authorization.models import DatasetPermission, OrganizationMembership, Role, User
3743
from authorization.permissions import (
@@ -469,20 +475,30 @@ def get_publishers(self, info: Info) -> List[Union[TypeOrganization, TypeUser]]:
469475
published_datasets = Dataset.objects.filter(
470476
status=DatasetStatus.PUBLISHED.value
471477
)
478+
published_ds_organizations = published_datasets.values_list(
479+
"organization_id", flat=True
480+
)
481+
published_usecases = UseCase.objects.filter(
482+
status=UseCaseStatus.PUBLISHED.value
483+
)
484+
published_uc_organizations = published_usecases.values_list(
485+
"organization_id", flat=True
486+
)
487+
published_organizations = set(published_ds_organizations) | set(
488+
published_uc_organizations
489+
)
472490

473491
# Get unique organizations that have published datasets
474492
org_publishers = Organization.objects.filter(
475-
id__in=published_datasets.filter(organization__isnull=False).values_list(
476-
"organization_id", flat=True
477-
)
493+
id__in=published_organizations
478494
).distinct()
479495

496+
published_ds_users = published_datasets.values_list("user_id", flat=True)
497+
published_uc_users = published_usecases.values_list("user_id", flat=True)
498+
published_users = set(published_ds_users) | set(published_uc_users)
499+
480500
# Get unique individual users who have published datasets without an organization
481-
individual_publishers = User.objects.filter(
482-
id__in=published_datasets.filter(organization__isnull=True).values_list(
483-
"user_id", flat=True
484-
)
485-
).distinct()
501+
individual_publishers = User.objects.filter(id__in=published_users).distinct()
486502

487503
# Convert to GraphQL types
488504
org_types = [TypeOrganization.from_django(org) for org in org_publishers]
@@ -564,6 +580,10 @@ def add_update_dataset_metadata(
564580
dataset = Dataset.objects.get(id=dataset_id)
565581
except Dataset.DoesNotExist as e:
566582
raise DjangoValidationError(f"Dataset with ID {dataset_id} does not exist.")
583+
if dataset.status != DatasetStatus.DRAFT.value:
584+
raise DjangoValidationError(
585+
f"Dataset with ID {dataset_id} is not in draft status."
586+
)
567587

568588
if update_metadata_input.description:
569589
dataset.description = update_metadata_input.description
@@ -616,11 +636,14 @@ def update_dataset(
616636
dataset = Dataset.objects.get(id=dataset_id)
617637
except Dataset.DoesNotExist as e:
618638
raise ValueError(f"Dataset with ID {dataset_id} does not exist.")
619-
639+
if dataset.status != DatasetStatus.DRAFT.value:
640+
raise ValueError(f"Dataset with ID {dataset_id} is not in draft status.")
641+
if update_dataset_input.title.strip() == "":
642+
raise ValueError("Title cannot be empty.")
620643
if update_dataset_input.title:
621-
dataset.title = update_dataset_input.title
644+
dataset.title = update_dataset_input.title.strip()
622645
if update_dataset_input.description:
623-
dataset.description = update_dataset_input.description
646+
dataset.description = update_dataset_input.description.strip()
624647
if update_dataset_input.access_type:
625648
dataset.access_type = update_dataset_input.access_type
626649
if update_dataset_input.license:

‎api/schema/organization_data_schema.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import strawberry
66
import strawberry_django
7+
from django.db.models import Q
78
from strawberry.types import Info
89

910
from api.models import Dataset, Organization, Sector, UseCase
@@ -58,7 +59,12 @@ def organization_published_use_cases(
5859
try:
5960
# Get published use cases for this organization
6061
queryset = UseCase.objects.filter(
61-
usecaseorganizationrelationship__organization_id=organization_id,
62+
(
63+
Q(organization__id=organization_id)
64+
| Q(
65+
usecaseorganizationrelationship__organization_id=organization_id
66+
)
67+
),
6268
status=UseCaseStatus.PUBLISHED.value,
6369
).distinct()
6470
return TypeUseCase.from_django_list(queryset)

‎api/schema/organization_schema.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,15 @@ def organizations(
8585

8686
return [TypeOrganization.from_django(org) for org in queryset]
8787

88+
@strawberry_django.field(permission_classes=[IsAuthenticated])
89+
def all_organizations(self, info: Info) -> List[TypeOrganization]:
90+
"""Get all organizations."""
91+
user = info.context.user
92+
if not user or getattr(user, "is_anonymous", True):
93+
logging.warning("Anonymous user or no user found in context")
94+
return []
95+
return [TypeOrganization.from_django(org) for org in Organization.objects.all()]
96+
8897
@strawberry_django.field
8998
def organization(self, info: Info, id: str) -> Optional[TypeOrganization]:
9099
"""Get organization by ID."""

0 commit comments

Comments
 (0)