Skip to content

Commit ebfad3e

Browse files
Ken LippoldKen Lippold
Ken Lippold
authored and
Ken Lippold
committed
Updated ETL models and endpoints
1 parent f192cb8 commit ebfad3e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+2488
-2928
lines changed

‎etl/admin.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,7 @@
11
from django.contrib import admin
2-
from etl.models import (
3-
OrchestrationSystem,
4-
DataConnector,
5-
LinkedDatastream,
6-
)
2+
from etl.models import OrchestrationSystem, DataSource, DataArchive
73

84

95
admin.site.register(OrchestrationSystem)
10-
admin.site.register(DataConnector)
11-
admin.site.register(LinkedDatastream)
6+
admin.site.register(DataSource)
7+
admin.site.register(DataArchive)

‎etl/fixtures/hydroshare_archival.yaml‎

Lines changed: 2 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,5 @@
1-
- model: etl.etlsystemplatform
2-
pk: eb1a26e0-75c9-415d-9895-8b0be8466139
3-
fields:
4-
name: HydroShare Archival
5-
interval_schedule_supported: False
6-
crontab_schedule_supported: False
7-
- model: etl.etlsystem
1+
- model: etl.orchestrationsystem
82
pk: 77380670-5d80-4ffc-86d0-7522e95bfbe7
93
fields:
104
name: HydroShare Archival Manager
11-
etl_system_platform: eb1a26e0-75c9-415d-9895-8b0be8466139
12-
- model: etl.etlconfiguration
13-
pk: 529b62fc-84bf-4173-8add-c9fc9126d3db
14-
fields:
15-
name: HydroShare Archive Loader Settings
16-
etl_system_platform: eb1a26e0-75c9-415d-9895-8b0be8466139
17-
etl_configuration_model: datasource
18-
etl_configuration_type: loader
19-
etl_configuration_schema:
20-
type: object
21-
properties:
22-
thingId:
23-
type: string
24-
format: uuid
25-
link:
26-
type: string
27-
path:
28-
type: string
29-
datastreamIds:
30-
type: array
31-
items:
32-
type: string
33-
format: uuid
34-
required:
35-
- thingId
36-
- link
37-
- path
38-
- datastreamIds
39-
additionalProperties: false
5+
orchestration_system_type: HydroShare

‎etl/management/commands/load_etl_test_data.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ def handle(self, *args, **kwargs):
1818
"tests/fixtures/test_sensors.yaml",
1919
"tests/fixtures/test_units.yaml",
2020
"tests/fixtures/test_datastreams.yaml",
21-
"tests/fixtures/test_etl_system_platforms.yaml",
22-
"tests/fixtures/test_etl_configurations.yaml",
23-
"tests/fixtures/test_etl_systems.yaml",
21+
"tests/fixtures/test_orchestration_systems.yaml",
2422
"tests/fixtures/test_data_sources.yaml",
23+
"tests/fixtures/test_data_archives.yaml",
24+
"tests/fixtures/test_etl_datastreams.yaml",
2525
]
2626

2727
for fixture in fixtures:

‎etl/migrations/0001_initial.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Generated by Django 5.2b1 on 2025-04-10 16:24
2+
3+
import django.db.models.deletion
4+
import iam.models.utils
5+
import uuid
6+
from django.db import migrations, models
7+
8+
9+
class Migration(migrations.Migration):
10+
11+
initial = True
12+
13+
dependencies = [
14+
("iam", "0001_initial"),
15+
]
16+
17+
operations = [
18+
migrations.CreateModel(
19+
name="OrchestrationSystem",
20+
fields=[
21+
(
22+
"id",
23+
models.UUIDField(
24+
default=uuid.uuid4,
25+
editable=False,
26+
primary_key=True,
27+
serialize=False,
28+
),
29+
),
30+
("name", models.CharField(max_length=255)),
31+
("orchestration_system_type", models.CharField(max_length=255)),
32+
(
33+
"workspace",
34+
models.ForeignKey(
35+
blank=True,
36+
null=True,
37+
on_delete=django.db.models.deletion.DO_NOTHING,
38+
related_name="orchestration_systems",
39+
to="iam.workspace",
40+
),
41+
),
42+
],
43+
bases=(models.Model, iam.models.utils.PermissionChecker),
44+
),
45+
migrations.CreateModel(
46+
name="DataSource",
47+
fields=[
48+
("interval", models.PositiveIntegerField(blank=True, null=True)),
49+
(
50+
"interval_units",
51+
models.CharField(blank=True, max_length=255, null=True),
52+
),
53+
("crontab", models.CharField(blank=True, max_length=255, null=True)),
54+
("start_time", models.DateTimeField(blank=True, null=True)),
55+
("end_time", models.DateTimeField(blank=True, null=True)),
56+
("paused", models.BooleanField(default=False)),
57+
("last_run_successful", models.BooleanField(blank=True, null=True)),
58+
("last_run_message", models.TextField(blank=True, null=True)),
59+
("last_run", models.DateTimeField(blank=True, null=True)),
60+
("next_run", models.DateTimeField(blank=True, null=True)),
61+
(
62+
"id",
63+
models.UUIDField(
64+
default=uuid.uuid4,
65+
editable=False,
66+
primary_key=True,
67+
serialize=False,
68+
),
69+
),
70+
("name", models.CharField(max_length=255)),
71+
("settings", models.JSONField(blank=True, null=True)),
72+
(
73+
"workspace",
74+
models.ForeignKey(
75+
on_delete=django.db.models.deletion.DO_NOTHING,
76+
related_name="data_sources",
77+
to="iam.workspace",
78+
),
79+
),
80+
(
81+
"orchestration_system",
82+
models.ForeignKey(
83+
on_delete=django.db.models.deletion.DO_NOTHING,
84+
related_name="data_sources",
85+
to="etl.orchestrationsystem",
86+
),
87+
),
88+
],
89+
options={
90+
"abstract": False,
91+
},
92+
bases=(models.Model, iam.models.utils.PermissionChecker),
93+
),
94+
migrations.CreateModel(
95+
name="DataArchive",
96+
fields=[
97+
("interval", models.PositiveIntegerField(blank=True, null=True)),
98+
(
99+
"interval_units",
100+
models.CharField(blank=True, max_length=255, null=True),
101+
),
102+
("crontab", models.CharField(blank=True, max_length=255, null=True)),
103+
("start_time", models.DateTimeField(blank=True, null=True)),
104+
("end_time", models.DateTimeField(blank=True, null=True)),
105+
("paused", models.BooleanField(default=False)),
106+
("last_run_successful", models.BooleanField(blank=True, null=True)),
107+
("last_run_message", models.TextField(blank=True, null=True)),
108+
("last_run", models.DateTimeField(blank=True, null=True)),
109+
("next_run", models.DateTimeField(blank=True, null=True)),
110+
(
111+
"id",
112+
models.UUIDField(
113+
default=uuid.uuid4,
114+
editable=False,
115+
primary_key=True,
116+
serialize=False,
117+
),
118+
),
119+
("name", models.CharField(max_length=255)),
120+
("settings", models.JSONField(blank=True, null=True)),
121+
(
122+
"workspace",
123+
models.ForeignKey(
124+
on_delete=django.db.models.deletion.DO_NOTHING,
125+
related_name="data_archives",
126+
to="iam.workspace",
127+
),
128+
),
129+
(
130+
"orchestration_system",
131+
models.ForeignKey(
132+
on_delete=django.db.models.deletion.DO_NOTHING,
133+
related_name="data_archives",
134+
to="etl.orchestrationsystem",
135+
),
136+
),
137+
],
138+
options={
139+
"abstract": False,
140+
},
141+
bases=(models.Model, iam.models.utils.PermissionChecker),
142+
),
143+
]

‎etl/models/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1-
from .data_connector import DataConnector, LinkedDatastream
1+
from .data_source import DataSource
2+
from .data_archive import DataArchive
23
from .orchestration_system import OrchestrationSystem

etl/models/data_connector.py renamed to etl/models/data_archive.py

Lines changed: 27 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from django.db import models
55
from django.db.models import Q
66
from iam.models.utils import PermissionChecker
7+
from .orchestration_configuration import OrchestrationConfiguration
78

89
if typing.TYPE_CHECKING:
910
from django.contrib.auth import get_user_model
@@ -12,8 +13,10 @@
1213
User = get_user_model()
1314

1415

15-
class DataConnectorQuerySet(models.QuerySet):
16+
class DataArchiveQuerySet(models.QuerySet):
1617
def visible(self, user: "User"):
18+
if not user:
19+
return self.none()
1720
if user.account_type == "admin":
1821
return self
1922
else:
@@ -23,7 +26,7 @@ def visible(self, user: "User"):
2326
workspace__collaborators__user=user,
2427
workspace__collaborators__role__permissions__resource_type__in=[
2528
"*",
26-
"DataConnector",
29+
"DataArchive",
2730
],
2831
workspace__collaborators__role__permissions__permission_type__in=[
2932
"*",
@@ -33,40 +36,38 @@ def visible(self, user: "User"):
3336
)
3437

3538

36-
class DataConnector(models.Model, PermissionChecker):
39+
class DataArchive(OrchestrationConfiguration, PermissionChecker):
3740
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
3841
workspace = models.ForeignKey(
39-
"iam.Workspace", related_name="data_connectors", on_delete=models.DO_NOTHING
42+
"iam.Workspace", related_name="data_archives", on_delete=models.DO_NOTHING
4043
)
41-
name = models.CharField(max_length=255)
4244
orchestration_system = models.ForeignKey(
43-
"OrchestrationSystem", related_name="data_connectors", on_delete=models.DO_NOTHING
45+
"OrchestrationSystem", related_name="data_archives", on_delete=models.DO_NOTHING
4446
)
45-
interval = models.PositiveIntegerField(blank=True, null=True)
46-
interval_units = models.CharField(max_length=255, blank=True, null=True)
47-
crontab = models.CharField(max_length=255, blank=True, null=True)
48-
start_time = models.DateTimeField(blank=True, null=True)
49-
end_time = models.DateTimeField(blank=True, null=True)
50-
paused = models.BooleanField(default=False)
51-
last_run_successful = models.BooleanField(blank=True, null=True)
52-
last_run_message = models.TextField(blank=True, null=True)
53-
last_run = models.DateTimeField(blank=True, null=True)
54-
next_run = models.DateTimeField(blank=True, null=True)
47+
name = models.CharField(max_length=255)
5548
settings = models.JSONField(blank=True, null=True)
5649

57-
objects = DataConnectorQuerySet.as_manager()
50+
objects = DataArchiveQuerySet.as_manager()
51+
52+
@property
53+
def status(self):
54+
return self
55+
56+
@property
57+
def schedule(self):
58+
return self
5859

5960
@classmethod
6061
def can_user_create(cls, user: Optional["User"], workspace: "Workspace"):
6162
return cls.check_create_permissions(
62-
user=user, workspace=workspace, resource_type="DataConnector"
63+
user=user, workspace=workspace, resource_type="DataArchive"
6364
)
6465

6566
def get_user_permissions(
6667
self, user: Optional["User"]
6768
) -> list[Literal["edit", "delete", "view"]]:
6869
user_permissions = self.check_object_permissions(
69-
user=user, workspace=self.workspace, resource_type="DataConnector"
70+
user=user, workspace=self.workspace, resource_type="DataArchive"
7071
)
7172

7273
return user_permissions
@@ -77,19 +78,12 @@ def delete(self, *args, **kwargs):
7778

7879
@staticmethod
7980
def delete_contents(filter_arg: models.Model, filter_suffix: Optional[str]):
80-
data_connector_relation_filter = (
81-
f"data_connector__{filter_suffix}" if filter_suffix else "data_connector"
82-
)
83-
LinkedDatastream.objects.filter(
84-
**{data_connector_relation_filter: filter_arg}
85-
).delete()
81+
from sta.models import Datastream
8682

83+
data_archive_relation_filter = (
84+
f"dataarchive__{filter_suffix}" if filter_suffix else "dataarchive"
85+
)
8786

88-
class LinkedDatastream(models.Model):
89-
data_connector = models.ForeignKey(
90-
DataConnector, related_name="linked_datastreams", on_delete=models.DO_NOTHING
91-
)
92-
datastream = models.ForeignKey(
93-
"sta.Datastream", related_name="data_connectors", on_delete=models.DO_NOTHING
94-
)
95-
is_data_source = models.BooleanField(default=False)
87+
Datastream.data_archives.through.objects.filter(
88+
**{data_archive_relation_filter: filter_arg}
89+
).delete()

0 commit comments

Comments
 (0)