Skip to content

Commit 3083fef

Browse files
KIRA009abrichr
KIRA009
and
abrichr
authored
feat(dashboard): Dashboard scrubbing (#672)
* feat: Add ability to download spacy model from within pyinstaller build * refactor: Add id column as foreign keys instead of timestamp * feat(scrubbing): Add api endpoints and scrubbing feature * fix: Fix code after merge with main * lint: flake8 lint * refactor: Cleanup code and add comments * chore: Fix comments * chore: Add comments on crud.py --------- Co-authored-by: Richard Abrich <[email protected]>
1 parent 4475e9d commit 3083fef

36 files changed

+1998
-336
lines changed

‎openadapt/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,7 @@
22
33
This package contains modules for the OpenAdapt project.
44
"""
5-
__version__ = "0.5.1"
5+
6+
from pathlib import Path
7+
8+
PROJECT_DIR = Path(__file__).parent

‎openadapt/alembic/versions/186316d4f3ca_add_scrubbed_columns.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""add_scrubbed_columns
2+
3+
Revision ID: 186316d4f3ca
4+
Revises: 2a8a241785f2
5+
Create Date: 2024-04-26 02:23:19.422051
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "186316d4f3ca"
13+
down_revision = "2a8a241785f2"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
# ### commands auto generated by Alembic - please adjust! ###
20+
with op.batch_alter_table("action_event", schema=None) as batch_op:
21+
batch_op.add_column(sa.Column("scrubbed_text", sa.String(), nullable=True))
22+
batch_op.add_column(
23+
sa.Column("scrubbed_canonical_text", sa.String(), nullable=True)
24+
)
25+
26+
# ### end Alembic commands ###
27+
28+
29+
def downgrade() -> None:
30+
# ### commands auto generated by Alembic - please adjust! ###
31+
with op.batch_alter_table("action_event", schema=None) as batch_op:
32+
batch_op.drop_column("scrubbed_canonical_text")
33+
batch_op.drop_column("scrubbed_text")
34+
35+
# ### end Alembic commands ###

‎openadapt/alembic/versions/2a8a241785f2_add_original_recording_id.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"""add_original_recording_id
2+
3+
Revision ID: 2a8a241785f2
4+
Revises: 87a78a84a8bf
5+
Create Date: 2024-04-25 10:25:54.685135
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "2a8a241785f2"
13+
down_revision = "87a78a84a8bf"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
# ### commands auto generated by Alembic - please adjust! ###
20+
with op.batch_alter_table("recording", schema=None) as batch_op:
21+
batch_op.add_column(
22+
sa.Column("original_recording_id", sa.Integer(), nullable=True)
23+
)
24+
batch_op.create_foreign_key(
25+
batch_op.f("fk_recording_original_recording_id_recording"),
26+
"recording",
27+
["original_recording_id"],
28+
["id"],
29+
)
30+
31+
# ### end Alembic commands ###
32+
33+
34+
def downgrade() -> None:
35+
# ### commands auto generated by Alembic - please adjust! ###
36+
with op.batch_alter_table("recording", schema=None) as batch_op:
37+
batch_op.drop_constraint(
38+
batch_op.f("fk_recording_original_recording_id_recording"),
39+
type_="foreignkey",
40+
)
41+
batch_op.drop_column("original_recording_id")
42+
43+
# ### end Alembic commands ###

‎openadapt/alembic/versions/87a78a84a8bf_remove_recording_timestamp_fks.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""remove_recording_timestamp_fks
2+
3+
Revision ID: 87a78a84a8bf
4+
Revises: f9586c10a561
5+
Create Date: 2024-04-24 20:16:31.970666
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "87a78a84a8bf"
13+
down_revision = "f9586c10a561"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
# ### commands auto generated by Alembic - please adjust! ###
20+
with op.batch_alter_table("action_event", schema=None) as batch_op:
21+
batch_op.drop_constraint(
22+
"fk_input_event_recording_timestamp_recording", type_="foreignkey"
23+
)
24+
batch_op.drop_constraint(
25+
"fk_input_event_window_event_timestamp_window_event", type_="foreignkey"
26+
)
27+
batch_op.drop_constraint(
28+
"fk_input_event_screenshot_timestamp_screenshot", type_="foreignkey"
29+
)
30+
31+
with op.batch_alter_table("screenshot", schema=None) as batch_op:
32+
batch_op.drop_constraint(
33+
"fk_screenshot_recording_timestamp_recording", type_="foreignkey"
34+
)
35+
36+
with op.batch_alter_table("window_event", schema=None) as batch_op:
37+
batch_op.drop_constraint(
38+
"fk_window_event_recording_timestamp_recording", type_="foreignkey"
39+
)
40+
41+
# ### end Alembic commands ###
42+
43+
44+
def downgrade() -> None:
45+
# ### commands auto generated by Alembic - please adjust! ###
46+
with op.batch_alter_table("window_event", schema=None) as batch_op:
47+
batch_op.create_foreign_key(
48+
"fk_window_event_recording_timestamp_recording",
49+
"recording",
50+
["recording_timestamp"],
51+
["timestamp"],
52+
)
53+
54+
with op.batch_alter_table("screenshot", schema=None) as batch_op:
55+
batch_op.create_foreign_key(
56+
"fk_screenshot_recording_timestamp_recording",
57+
"recording",
58+
["recording_timestamp"],
59+
["timestamp"],
60+
)
61+
62+
with op.batch_alter_table("action_event", schema=None) as batch_op:
63+
batch_op.create_foreign_key(
64+
"fk_input_event_screenshot_timestamp_screenshot",
65+
"screenshot",
66+
["screenshot_timestamp"],
67+
["timestamp"],
68+
)
69+
batch_op.create_foreign_key(
70+
"fk_input_event_window_event_timestamp_window_event",
71+
"window_event",
72+
["window_event_timestamp"],
73+
["timestamp"],
74+
)
75+
batch_op.create_foreign_key(
76+
"fk_input_event_recording_timestamp_recording",
77+
"recording",
78+
["recording_timestamp"],
79+
["timestamp"],
80+
)
81+
82+
# ### end Alembic commands ###

‎openadapt/alembic/versions/c24abb5455d3_create_new_recording_fks.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
"""create_new_recording_fks
2+
3+
Revision ID: c24abb5455d3
4+
Revises: 30a5ba9d6453
5+
Create Date: 2024-04-24 19:32:59.011079
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "c24abb5455d3"
13+
down_revision = "8495f5471e23"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
# ### commands auto generated by Alembic - please adjust! ###
20+
with op.batch_alter_table("action_event", schema=None) as batch_op:
21+
batch_op.add_column(sa.Column("recording_id", sa.Integer(), nullable=True))
22+
batch_op.add_column(sa.Column("screenshot_id", sa.Integer(), nullable=True))
23+
batch_op.add_column(sa.Column("window_event_id", sa.Integer(), nullable=True))
24+
batch_op.create_foreign_key(
25+
batch_op.f("fk_action_event_recording_id_recording"),
26+
"recording",
27+
["recording_id"],
28+
["id"],
29+
)
30+
batch_op.create_foreign_key(
31+
batch_op.f("fk_action_event_screenshot_id_screenshot"),
32+
"screenshot",
33+
["screenshot_id"],
34+
["id"],
35+
)
36+
batch_op.create_foreign_key(
37+
batch_op.f("fk_action_event_window_event_id_window_event"),
38+
"window_event",
39+
["window_event_id"],
40+
["id"],
41+
)
42+
43+
with op.batch_alter_table("performance_stat", schema=None) as batch_op:
44+
batch_op.add_column(sa.Column("recording_id", sa.Integer(), nullable=True))
45+
batch_op.create_foreign_key(
46+
batch_op.f("fk_performance_stat_recording_id_recording"),
47+
"recording",
48+
["recording_id"],
49+
["id"],
50+
)
51+
52+
with op.batch_alter_table("memory_stat", schema=None) as batch_op:
53+
batch_op.add_column(sa.Column("recording_id", sa.Integer(), nullable=True))
54+
batch_op.create_foreign_key(
55+
batch_op.f("fk_memory_stat_recording_id_recording"),
56+
"recording",
57+
["recording_id"],
58+
["id"],
59+
)
60+
61+
with op.batch_alter_table("screenshot", schema=None) as batch_op:
62+
batch_op.add_column(sa.Column("recording_id", sa.Integer(), nullable=True))
63+
batch_op.create_foreign_key(
64+
batch_op.f("fk_screenshot_recording_id_recording"),
65+
"recording",
66+
["recording_id"],
67+
["id"],
68+
)
69+
70+
with op.batch_alter_table("window_event", schema=None) as batch_op:
71+
batch_op.add_column(sa.Column("recording_id", sa.Integer(), nullable=True))
72+
batch_op.create_foreign_key(
73+
batch_op.f("fk_window_event_recording_id_recording"),
74+
"recording",
75+
["recording_id"],
76+
["id"],
77+
)
78+
79+
# ### end Alembic commands ###
80+
81+
82+
def downgrade() -> None:
83+
# ### commands auto generated by Alembic - please adjust! ###
84+
with op.batch_alter_table("window_event", schema=None) as batch_op:
85+
batch_op.drop_constraint(
86+
batch_op.f("fk_window_event_recording_id_recording"), type_="foreignkey"
87+
)
88+
batch_op.drop_column("recording_id")
89+
90+
with op.batch_alter_table("screenshot", schema=None) as batch_op:
91+
batch_op.drop_constraint(
92+
batch_op.f("fk_screenshot_recording_id_recording"), type_="foreignkey"
93+
)
94+
batch_op.drop_column("recording_id")
95+
96+
with op.batch_alter_table("memory_stat", schema=None) as batch_op:
97+
batch_op.drop_column("recording_id")
98+
99+
with op.batch_alter_table("performance_stat", schema=None) as batch_op:
100+
batch_op.drop_column("recording_id")
101+
102+
with op.batch_alter_table("action_event", schema=None) as batch_op:
103+
batch_op.drop_constraint(
104+
batch_op.f("fk_action_event_window_event_id_window_event"),
105+
type_="foreignkey",
106+
)
107+
batch_op.drop_constraint(
108+
batch_op.f("fk_action_event_screenshot_id_screenshot"), type_="foreignkey"
109+
)
110+
batch_op.drop_constraint(
111+
batch_op.f("fk_action_event_recording_id_recording"), type_="foreignkey"
112+
)
113+
batch_op.drop_column("window_event_id")
114+
batch_op.drop_column("screenshot_id")
115+
batch_op.drop_column("recording_id")
116+
117+
# ### end Alembic commands ###

‎openadapt/alembic/versions/d714cc86fce8_add_scrubbed_recording_model.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
"""add_scrubbed_recording_model
2+
3+
Revision ID: d714cc86fce8
4+
Revises: 186316d4f3ca
5+
Create Date: 2024-04-29 09:17:04.237108
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
from openadapt.models import ForceFloat
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "d714cc86fce8"
15+
down_revision = "186316d4f3ca"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade() -> None:
21+
# ### commands auto generated by Alembic - please adjust! ###
22+
op.create_table(
23+
"scrubbed_recording",
24+
sa.Column("id", sa.Integer(), nullable=False),
25+
sa.Column(
26+
"timestamp",
27+
ForceFloat(precision=10, scale=2, asdecimal=False),
28+
nullable=True,
29+
),
30+
sa.Column("recording_id", sa.Integer(), nullable=True),
31+
sa.Column("provider", sa.String(), nullable=True),
32+
sa.Column("scrubbed", sa.Boolean(), nullable=True),
33+
sa.ForeignKeyConstraint(
34+
["recording_id"],
35+
["recording.id"],
36+
name=op.f("fk_scrubbed_recording_recording_id_recording"),
37+
),
38+
sa.PrimaryKeyConstraint("id", name=op.f("pk_scrubbed_recording")),
39+
)
40+
# ### end Alembic commands ###
41+
42+
43+
def downgrade() -> None:
44+
# ### commands auto generated by Alembic - please adjust! ###
45+
op.drop_table("scrubbed_recording")
46+
# ### end Alembic commands ###

‎openadapt/alembic/versions/f9586c10a561_migrate_data_to_new_fks.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
"""migrate_data_to_new_fks
2+
3+
Revision ID: f9586c10a561
4+
Revises: c24abb5455d3
5+
Create Date: 2024-04-24 19:34:00.000152
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "f9586c10a561"
13+
down_revision = "c24abb5455d3"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
bind = op.get_bind()
20+
session = sa.orm.Session(bind=bind)
21+
22+
for table in [
23+
"action_event",
24+
"window_event",
25+
"screenshot",
26+
"memory_stat",
27+
"performance_stat",
28+
]:
29+
session.execute(
30+
f"UPDATE {table} SET recording_id = (SELECT id FROM recording WHERE"
31+
f" recording.timestamp = {table}.recording_timestamp)"
32+
)
33+
34+
session.execute(
35+
"UPDATE action_event SET window_event_id = (SELECT id FROM window_event WHERE"
36+
" window_event.timestamp = action_event.window_event_timestamp)"
37+
)
38+
session.execute(
39+
"UPDATE action_event SET screenshot_id = (SELECT id FROM screenshot WHERE"
40+
" screenshot.timestamp = action_event.screenshot_timestamp)"
41+
)
42+
43+
44+
def downgrade() -> None:
45+
pass

0 commit comments

Comments
 (0)