Skip to content

Commit 339740a

Browse files
kjlippoldkjlippold
authored
Merge pull request #232 from hydroserver2/240-etl
240 etl
2 parents 3a1edb4 + bcaafc5 commit 339740a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+4449
-125
lines changed

‎etl/admin.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
from django.contrib import admin
2+
from etl.models import OrchestrationSystem, DataSource, DataArchive
23

3-
# Register your models here.
4+
5+
admin.site.register(OrchestrationSystem)
6+
admin.site.register(DataSource)
7+
admin.site.register(DataArchive)

‎etl/fixtures/hydroshare_archival.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
- model: etl.orchestrationsystem
2+
pk: 77380670-5d80-4ffc-86d0-7522e95bfbe7
3+
fields:
4+
name: HydroShare Archival Manager
5+
orchestration_system_type: HydroShare

‎etl/management/__init__.py‎

Whitespace-only changes.

‎etl/management/commands/__init__.py‎

Whitespace-only changes.

‎etl/management/commands/load_etl_test_data.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
from django.core.management.base import BaseCommand
2+
from django.core.management import call_command
3+
4+
5+
class Command(BaseCommand):
6+
help = "Loads test data from fixtures"
7+
8+
def handle(self, *args, **kwargs):
9+
fixtures = [
10+
"tests/fixtures/test_users.yaml",
11+
"tests/fixtures/test_workspaces.yaml",
12+
"tests/fixtures/test_roles.yaml",
13+
"tests/fixtures/test_collaborators.yaml",
14+
"tests/fixtures/test_things.yaml",
15+
"tests/fixtures/test_observed_properties.yaml",
16+
"tests/fixtures/test_processing_levels.yaml",
17+
"tests/fixtures/test_result_qualifiers.yaml",
18+
"tests/fixtures/test_sensors.yaml",
19+
"tests/fixtures/test_units.yaml",
20+
"tests/fixtures/test_datastreams.yaml",
21+
"tests/fixtures/test_orchestration_systems.yaml",
22+
"tests/fixtures/test_data_sources.yaml",
23+
"tests/fixtures/test_data_archives.yaml",
24+
"tests/fixtures/test_etl_datastreams.yaml",
25+
]
26+
27+
for fixture in fixtures:
28+
self.stdout.write(self.style.NOTICE(f"Loading fixture: {fixture}"))
29+
try:
30+
call_command("loaddata", fixture)
31+
self.stdout.write(self.style.SUCCESS(f"Successfully loaded {fixture}"))
32+
except Exception as e:
33+
self.stderr.write(self.style.ERROR(f"Failed to load {fixture}: {e}"))

‎etl/migrations/0001_initial.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Generated by Django 5.2b1 on 2025-04-10 16:24
2+
3+
import django.db.models.deletion
4+
import iam.models.utils
5+
import uuid
6+
from django.db import migrations, models
7+
8+
9+
class Migration(migrations.Migration):
10+
11+
initial = True
12+
13+
dependencies = [
14+
("iam", "0001_initial"),
15+
]
16+
17+
operations = [
18+
migrations.CreateModel(
19+
name="OrchestrationSystem",
20+
fields=[
21+
(
22+
"id",
23+
models.UUIDField(
24+
default=uuid.uuid4,
25+
editable=False,
26+
primary_key=True,
27+
serialize=False,
28+
),
29+
),
30+
("name", models.CharField(max_length=255)),
31+
("orchestration_system_type", models.CharField(max_length=255)),
32+
(
33+
"workspace",
34+
models.ForeignKey(
35+
blank=True,
36+
null=True,
37+
on_delete=django.db.models.deletion.DO_NOTHING,
38+
related_name="orchestration_systems",
39+
to="iam.workspace",
40+
),
41+
),
42+
],
43+
bases=(models.Model, iam.models.utils.PermissionChecker),
44+
),
45+
migrations.CreateModel(
46+
name="DataSource",
47+
fields=[
48+
("interval", models.PositiveIntegerField(blank=True, null=True)),
49+
(
50+
"interval_units",
51+
models.CharField(blank=True, max_length=255, null=True),
52+
),
53+
("crontab", models.CharField(blank=True, max_length=255, null=True)),
54+
("start_time", models.DateTimeField(blank=True, null=True)),
55+
("end_time", models.DateTimeField(blank=True, null=True)),
56+
("paused", models.BooleanField(default=False)),
57+
("last_run_successful", models.BooleanField(blank=True, null=True)),
58+
("last_run_message", models.TextField(blank=True, null=True)),
59+
("last_run", models.DateTimeField(blank=True, null=True)),
60+
("next_run", models.DateTimeField(blank=True, null=True)),
61+
(
62+
"id",
63+
models.UUIDField(
64+
default=uuid.uuid4,
65+
editable=False,
66+
primary_key=True,
67+
serialize=False,
68+
),
69+
),
70+
("name", models.CharField(max_length=255)),
71+
("settings", models.JSONField(blank=True, null=True)),
72+
(
73+
"workspace",
74+
models.ForeignKey(
75+
on_delete=django.db.models.deletion.DO_NOTHING,
76+
related_name="data_sources",
77+
to="iam.workspace",
78+
),
79+
),
80+
(
81+
"orchestration_system",
82+
models.ForeignKey(
83+
on_delete=django.db.models.deletion.DO_NOTHING,
84+
related_name="data_sources",
85+
to="etl.orchestrationsystem",
86+
),
87+
),
88+
],
89+
options={
90+
"abstract": False,
91+
},
92+
bases=(models.Model, iam.models.utils.PermissionChecker),
93+
),
94+
migrations.CreateModel(
95+
name="DataArchive",
96+
fields=[
97+
("interval", models.PositiveIntegerField(blank=True, null=True)),
98+
(
99+
"interval_units",
100+
models.CharField(blank=True, max_length=255, null=True),
101+
),
102+
("crontab", models.CharField(blank=True, max_length=255, null=True)),
103+
("start_time", models.DateTimeField(blank=True, null=True)),
104+
("end_time", models.DateTimeField(blank=True, null=True)),
105+
("paused", models.BooleanField(default=False)),
106+
("last_run_successful", models.BooleanField(blank=True, null=True)),
107+
("last_run_message", models.TextField(blank=True, null=True)),
108+
("last_run", models.DateTimeField(blank=True, null=True)),
109+
("next_run", models.DateTimeField(blank=True, null=True)),
110+
(
111+
"id",
112+
models.UUIDField(
113+
default=uuid.uuid4,
114+
editable=False,
115+
primary_key=True,
116+
serialize=False,
117+
),
118+
),
119+
("name", models.CharField(max_length=255)),
120+
("settings", models.JSONField(blank=True, null=True)),
121+
(
122+
"workspace",
123+
models.ForeignKey(
124+
on_delete=django.db.models.deletion.DO_NOTHING,
125+
related_name="data_archives",
126+
to="iam.workspace",
127+
),
128+
),
129+
(
130+
"orchestration_system",
131+
models.ForeignKey(
132+
on_delete=django.db.models.deletion.DO_NOTHING,
133+
related_name="data_archives",
134+
to="etl.orchestrationsystem",
135+
),
136+
),
137+
],
138+
options={
139+
"abstract": False,
140+
},
141+
bases=(models.Model, iam.models.utils.PermissionChecker),
142+
),
143+
]

‎etl/models/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from .data_source import DataSource
2+
from .data_archive import DataArchive
3+
from .orchestration_system import OrchestrationSystem

‎etl/models/data_archive.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import uuid
2+
import typing
3+
from typing import Literal, Optional
4+
from django.db import models
5+
from django.db.models import Q
6+
from iam.models.utils import PermissionChecker
7+
from .orchestration_configuration import OrchestrationConfiguration
8+
9+
if typing.TYPE_CHECKING:
10+
from django.contrib.auth import get_user_model
11+
from iam.models import Workspace
12+
13+
User = get_user_model()
14+
15+
16+
class DataArchiveQuerySet(models.QuerySet):
17+
def visible(self, user: "User"):
18+
if not user:
19+
return self.none()
20+
if user.account_type == "admin":
21+
return self
22+
else:
23+
return self.filter(
24+
Q(workspace__owner=user)
25+
| Q(
26+
workspace__collaborators__user=user,
27+
workspace__collaborators__role__permissions__resource_type__in=[
28+
"*",
29+
"DataArchive",
30+
],
31+
workspace__collaborators__role__permissions__permission_type__in=[
32+
"*",
33+
"view",
34+
],
35+
)
36+
)
37+
38+
39+
class DataArchive(OrchestrationConfiguration, PermissionChecker):
40+
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
41+
workspace = models.ForeignKey(
42+
"iam.Workspace", related_name="data_archives", on_delete=models.DO_NOTHING
43+
)
44+
orchestration_system = models.ForeignKey(
45+
"OrchestrationSystem", related_name="data_archives", on_delete=models.DO_NOTHING
46+
)
47+
name = models.CharField(max_length=255)
48+
settings = models.JSONField(blank=True, null=True)
49+
50+
objects = DataArchiveQuerySet.as_manager()
51+
52+
@property
53+
def status(self):
54+
return self
55+
56+
@property
57+
def schedule(self):
58+
return self
59+
60+
@classmethod
61+
def can_user_create(cls, user: Optional["User"], workspace: "Workspace"):
62+
return cls.check_create_permissions(
63+
user=user, workspace=workspace, resource_type="DataArchive"
64+
)
65+
66+
def get_user_permissions(
67+
self, user: Optional["User"]
68+
) -> list[Literal["edit", "delete", "view"]]:
69+
user_permissions = self.check_object_permissions(
70+
user=user, workspace=self.workspace, resource_type="DataArchive"
71+
)
72+
73+
return user_permissions
74+
75+
def delete(self, *args, **kwargs):
76+
self.delete_contents(filter_arg=self, filter_suffix="")
77+
super().delete(*args, **kwargs)
78+
79+
@staticmethod
80+
def delete_contents(filter_arg: models.Model, filter_suffix: Optional[str]):
81+
from sta.models import Datastream
82+
83+
data_archive_relation_filter = (
84+
f"dataarchive__{filter_suffix}" if filter_suffix else "dataarchive"
85+
)
86+
87+
Datastream.data_archives.through.objects.filter(
88+
**{data_archive_relation_filter: filter_arg}
89+
).delete()

‎etl/models/data_source.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import uuid
2+
import typing
3+
from typing import Literal, Optional
4+
from django.db import models
5+
from django.db.models import Q
6+
from iam.models.utils import PermissionChecker
7+
from .orchestration_configuration import OrchestrationConfiguration
8+
9+
if typing.TYPE_CHECKING:
10+
from django.contrib.auth import get_user_model
11+
from iam.models import Workspace
12+
13+
User = get_user_model()
14+
15+
16+
class DataSourceQuerySet(models.QuerySet):
17+
def visible(self, user: "User"):
18+
if not user:
19+
return self.none()
20+
if user.account_type == "admin":
21+
return self
22+
else:
23+
return self.filter(
24+
Q(workspace__owner=user)
25+
| Q(
26+
workspace__collaborators__user=user,
27+
workspace__collaborators__role__permissions__resource_type__in=[
28+
"*",
29+
"DataSource",
30+
],
31+
workspace__collaborators__role__permissions__permission_type__in=[
32+
"*",
33+
"view",
34+
],
35+
)
36+
)
37+
38+
39+
class DataSource(OrchestrationConfiguration, PermissionChecker):
40+
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
41+
workspace = models.ForeignKey(
42+
"iam.Workspace", related_name="data_sources", on_delete=models.DO_NOTHING
43+
)
44+
orchestration_system = models.ForeignKey(
45+
"OrchestrationSystem", related_name="data_sources", on_delete=models.DO_NOTHING
46+
)
47+
name = models.CharField(max_length=255)
48+
settings = models.JSONField(blank=True, null=True)
49+
50+
objects = DataSourceQuerySet.as_manager()
51+
52+
@property
53+
def status(self):
54+
return self
55+
56+
@property
57+
def schedule(self):
58+
return self
59+
60+
@classmethod
61+
def can_user_create(cls, user: Optional["User"], workspace: "Workspace"):
62+
return cls.check_create_permissions(
63+
user=user, workspace=workspace, resource_type="DataSource"
64+
)
65+
66+
def get_user_permissions(
67+
self, user: Optional["User"]
68+
) -> list[Literal["edit", "delete", "view"]]:
69+
user_permissions = self.check_object_permissions(
70+
user=user, workspace=self.workspace, resource_type="DataSource"
71+
)
72+
73+
return user_permissions
74+
75+
def delete(self, *args, **kwargs):
76+
self.delete_contents(filter_arg=self, filter_suffix="")
77+
super().delete(*args, **kwargs)
78+
79+
@staticmethod
80+
def delete_contents(filter_arg: models.Model, filter_suffix: Optional[str]):
81+
from sta.models import Datastream
82+
83+
data_source_relation_filter = (
84+
f"data_source__{filter_suffix}" if filter_suffix else "data_source"
85+
)
86+
Datastream.objects.filter(**{data_source_relation_filter: filter_arg}).update(
87+
data_source=None
88+
)

‎etl/models/orchestration_configuration.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from django.db import models
2+
3+
4+
class OrchestrationConfiguration(models.Model):
5+
interval = models.PositiveIntegerField(blank=True, null=True)
6+
interval_units = models.CharField(max_length=255, blank=True, null=True)
7+
crontab = models.CharField(max_length=255, blank=True, null=True)
8+
start_time = models.DateTimeField(blank=True, null=True)
9+
end_time = models.DateTimeField(blank=True, null=True)
10+
paused = models.BooleanField(default=False)
11+
last_run_successful = models.BooleanField(blank=True, null=True)
12+
last_run_message = models.TextField(blank=True, null=True)
13+
last_run = models.DateTimeField(blank=True, null=True)
14+
next_run = models.DateTimeField(blank=True, null=True)
15+
16+
class Meta:
17+
abstract = True

0 commit comments

Comments
 (0)