Skip to content

Commit 2b5c3a2

Browse files
JWittmeyerJWittmeyer
authored
Release v1.3.0 (#53)
Release v1.3.0
1 parent 86960dd commit 2b5c3a2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+2280
-186
lines changed

‎.gitmodules‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[submodule "submodules/model"]
22
path = submodules/model
3-
url = https://github.com/code-kern-ai/refinery-submodule-model.git
3+
url = git@github.com:code-kern-ai/refinery-submodule-model.git
44
[submodule "submodules/s3"]
55
path = submodules/s3
6-
url = https://github.com/code-kern-ai/refinery-submodule-s3.git
6+
url = git@github.com:code-kern-ai/refinery-submodule-s3.git

‎alembic/versions/87f463aa5112_adds_created_attributes_columns.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
"""Adds created attributes columns
2+
3+
Revision ID: 87f463aa5112
4+
Revises: 9618924f9679
5+
Create Date: 2022-09-13 08:22:52.181427
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
from sqlalchemy.dialects import postgresql
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "87f463aa5112"
15+
down_revision = "9618924f9679"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade():
21+
# ### commands auto generated by Alembic - please adjust! ###
22+
op.add_column("attribute", sa.Column("user_created", sa.Boolean(), nullable=True))
23+
op.add_column("attribute", sa.Column("source_code", sa.String(), nullable=True))
24+
op.add_column("attribute", sa.Column("state", sa.String(), nullable=True))
25+
op.add_column("attribute", sa.Column("logs", sa.ARRAY(sa.String()), nullable=True))
26+
op.add_column(
27+
"embedding",
28+
sa.Column("attribute_id", postgresql.UUID(as_uuid=True), nullable=True),
29+
)
30+
op.create_foreign_key(
31+
"embedding_attribute_id_fkey",
32+
"embedding",
33+
"attribute",
34+
["attribute_id"],
35+
["id"],
36+
ondelete="CASCADE",
37+
),
38+
# ### end Alembic commands ###
39+
40+
41+
def downgrade():
42+
# ### commands auto generated by Alembic - please adjust! ###
43+
op.drop_column("attribute", "user_created")
44+
op.drop_column("attribute", "source_code")
45+
op.drop_column("attribute", "state")
46+
op.drop_column("attribute", "logs")
47+
op.drop_column("embedding", "attribute_id")
48+
op.drop_constraint("embedding_attribute_id_fkey", "embedding", type_="foreignkey")
49+
# ### end Alembic commands ###

‎alembic/versions/9618924f9679_adds_comment_link_tables_some_org_.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
"""Adds comment & link tables & some org management
2+
3+
Revision ID: 9618924f9679
4+
Revises: 5b3a4deea1c4
5+
Create Date: 2022-09-15 07:15:02.934928
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
from sqlalchemy.dialects import postgresql
11+
12+
# revision identifiers, used by Alembic.
13+
revision = "9618924f9679"
14+
down_revision = "5b3a4deea1c4"
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
# ### commands auto generated by Alembic - please adjust! ###
21+
op.create_table(
22+
"comment_data",
23+
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
24+
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True),
25+
sa.Column("xfkey", postgresql.UUID(as_uuid=True), nullable=True),
26+
sa.Column("xftype", sa.String(), nullable=True),
27+
sa.Column("order_key", sa.Integer(), autoincrement=True, nullable=True),
28+
sa.Column("comment", sa.String(), nullable=True),
29+
sa.Column("is_markdown", sa.Boolean(), nullable=True),
30+
sa.Column("is_private", sa.Boolean(), nullable=True),
31+
sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True),
32+
sa.Column("created_at", sa.DateTime(), nullable=True),
33+
sa.ForeignKeyConstraint(
34+
["created_by"],
35+
["user.id"],
36+
),
37+
sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
38+
sa.PrimaryKeyConstraint("id"),
39+
)
40+
op.create_index(
41+
op.f("ix_comment_data_created_by"), "comment_data", ["created_by"], unique=False
42+
)
43+
op.create_index(
44+
op.f("ix_comment_data_project_id"), "comment_data", ["project_id"], unique=False
45+
)
46+
op.create_index(
47+
op.f("ix_comment_data_xfkey"), "comment_data", ["xfkey"], unique=False
48+
)
49+
op.create_index(
50+
op.f("ix_comment_data_xftype"), "comment_data", ["xftype"], unique=False
51+
)
52+
op.create_table(
53+
"labeling_access_link",
54+
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
55+
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True),
56+
sa.Column("link", sa.String(), nullable=True),
57+
sa.Column("data_slice_id", postgresql.UUID(as_uuid=True), nullable=True),
58+
sa.Column("heuristic_id", postgresql.UUID(as_uuid=True), nullable=True),
59+
sa.Column("link_type", sa.String(), nullable=True),
60+
sa.Column("created_at", sa.DateTime(), nullable=True),
61+
sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True),
62+
sa.Column("is_locked", sa.Boolean(), nullable=True),
63+
sa.Column("changed_at", sa.DateTime(), nullable=True),
64+
sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
65+
sa.ForeignKeyConstraint(["created_by"], ["user.id"], ondelete="CASCADE"),
66+
sa.ForeignKeyConstraint(
67+
["data_slice_id"], ["data_slice.id"], ondelete="CASCADE"
68+
),
69+
sa.ForeignKeyConstraint(
70+
["heuristic_id"], ["information_source.id"], ondelete="CASCADE"
71+
),
72+
sa.PrimaryKeyConstraint("id"),
73+
)
74+
op.create_index(
75+
op.f("ix_labeling_access_link_project_id"),
76+
"labeling_access_link",
77+
["project_id"],
78+
unique=False,
79+
)
80+
op.create_index(
81+
op.f("ix_labeling_access_link_created_by"),
82+
"labeling_access_link",
83+
["created_by"],
84+
unique=False,
85+
)
86+
op.create_index(
87+
op.f("ix_labeling_access_link_data_slice_id"),
88+
"labeling_access_link",
89+
["data_slice_id"],
90+
unique=False,
91+
)
92+
op.create_index(
93+
op.f("ix_labeling_access_link_heuristic_id"),
94+
"labeling_access_link",
95+
["heuristic_id"],
96+
unique=False,
97+
)
98+
op.add_column("organization", sa.Column("started_at", sa.DateTime(), nullable=True))
99+
op.add_column("organization", sa.Column("is_paying", sa.Boolean(), nullable=True))
100+
op.add_column("organization", sa.Column("created_at", sa.DateTime(), nullable=True))
101+
op.add_column("user", sa.Column("role", sa.String(), nullable=True))
102+
# ### end Alembic commands ###
103+
104+
105+
def downgrade():
106+
# ### commands auto generated by Alembic - please adjust! ###
107+
op.drop_column("user", "role")
108+
op.drop_column("organization", "created_at")
109+
op.drop_column("organization", "is_paying")
110+
op.drop_column("organization", "started_at")
111+
op.drop_index(
112+
op.f("ix_labeling_access_link_project_id"), table_name="labeling_access_link"
113+
)
114+
op.drop_index(
115+
op.f("ix_labeling_access_link_heuristic_id"), table_name="labeling_access_link"
116+
)
117+
op.drop_index(
118+
op.f("ix_labeling_access_link_data_slice_id"), table_name="labeling_access_link"
119+
)
120+
op.drop_index(
121+
op.f("ix_labeling_access_link_created_by"), table_name="labeling_access_link"
122+
)
123+
op.drop_table("labeling_access_link")
124+
op.drop_index(op.f("ix_comment_data_xftype"), table_name="comment_data")
125+
op.drop_index(op.f("ix_comment_data_xfkey"), table_name="comment_data")
126+
op.drop_index(op.f("ix_comment_data_project_id"), table_name="comment_data")
127+
op.drop_index(op.f("ix_comment_data_created_by"), table_name="comment_data")
128+
op.drop_table("comment_data")
129+
# ### end Alembic commands ###

‎api/project.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from controller.auth import manager as auth_manager
66
from controller.project import manager as project_manager
77
from controller.attribute import manager as attribute_manager
8+
from submodules.model import exceptions
89

910

1011
logging.basicConfig(level=logging.DEBUG)
@@ -14,9 +15,16 @@ class ProjectDetails(HTTPEndpoint):
1415
def get(self, request) -> JSONResponse:
1516
project_id = request.path_params["project_id"]
1617
user_id = request.query_params["user_id"]
17-
auth_manager.check_project_access_from_user_id(user_id, project_id)
18+
try:
19+
auth_manager.check_project_access_from_user_id(
20+
user_id, project_id, from_api=True
21+
)
22+
except exceptions.EntityNotFoundException:
23+
return JSONResponse({"error": "Could not find project"}, status_code=404)
24+
except exceptions.AccessDeniedException:
25+
return JSONResponse({"error": "Access denied"}, status_code=403)
1826
project = project_manager.get_project(project_id)
19-
attributes = attribute_manager.get_all_attributes(project_id)
27+
attributes = attribute_manager.get_all_attributes(project_id, ["ALL"])
2028
result = {
2129
"name": project.name,
2230
"description": project.description,
@@ -26,6 +34,7 @@ def get(self, request) -> JSONResponse:
2634
"name": attribute.name,
2735
"data_type": attribute.data_type,
2836
"is_primary_key": attribute.is_primary_key,
37+
"state": attribute.state,
2938
}
3039
for attribute in attributes
3140
],

‎api/transfer.py‎

Lines changed: 83 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
from submodules.model.business_objects import organization
99

1010
from controller.transfer import manager as upload_manager
11-
from controller.upload_task import manager as task_manager
11+
from controller.upload_task import manager as upload_task_manager
1212
from controller.auth import manager as auth_manager
1313
from controller.transfer import manager as transfer_manager
14+
from controller.transfer import association_transfer_manager
1415
from controller.auth import manager as auth
1516

16-
from submodules.model import enums
17+
from submodules.model import enums, exceptions
1718
from util.notification import create_notification
1819
from submodules.model.enums import NotificationType
1920
from submodules.model.models import UploadTask
@@ -43,7 +44,7 @@ async def post(self, request) -> PlainTextResponse:
4344
if org_id == "archive":
4445
return PlainTextResponse("OK")
4546

46-
task = task_manager.get_upload_task_secure(
47+
task = upload_task_manager.get_upload_task_secure(
4748
upload_task_id=upload_task_id,
4849
project_id=project_id,
4950
file_name=file_name,
@@ -62,7 +63,14 @@ def get(self, request) -> JSONResponse:
6263
project_id = request.path_params["project_id"]
6364
user_id = request.query_params["user_id"]
6465
num_samples = request.query_params.get("num_samples")
65-
auth_manager.check_project_access_from_user_id(user_id, project_id)
66+
try:
67+
auth_manager.check_project_access_from_user_id(
68+
user_id, project_id, from_api=True
69+
)
70+
except exceptions.EntityNotFoundException:
71+
return JSONResponse({"error": "Could not find project"}, status_code=404)
72+
except exceptions.AccessDeniedException:
73+
return JSONResponse({"error": "Access denied"}, status_code=403)
6674
result = transfer_manager.export_records(project_id, num_samples)
6775
return JSONResponse(result)
6876

@@ -72,23 +80,37 @@ def get(self, request) -> JSONResponse:
7280
project_id = request.path_params["project_id"]
7381
list_id = request.path_params["knowledge_base_id"]
7482
user_id = request.query_params["user_id"]
75-
auth_manager.check_project_access_from_user_id(user_id, project_id)
83+
try:
84+
auth_manager.check_project_access_from_user_id(
85+
user_id, project_id, from_api=True
86+
)
87+
except exceptions.EntityNotFoundException:
88+
return JSONResponse({"error": "Could not find project"}, status_code=404)
89+
except exceptions.AccessDeniedException:
90+
return JSONResponse({"error": "Access denied"}, status_code=403)
7691
result = transfer_manager.export_knowledge_base(project_id, list_id)
7792
return JSONResponse(result)
7893

7994

80-
class PrepareImport(HTTPEndpoint):
95+
class PrepareFileImport(HTTPEndpoint):
8196
async def post(self, request) -> JSONResponse:
8297
auth.check_is_demo_without_info()
8398
project_id = request.path_params["project_id"]
8499
request_body = await request.json()
85100

86101
user_id = request_body["user_id"]
87-
auth_manager.check_project_access_from_user_id(user_id, project_id)
102+
try:
103+
auth_manager.check_project_access_from_user_id(
104+
user_id, project_id, from_api=True
105+
)
106+
except exceptions.EntityNotFoundException:
107+
return JSONResponse({"error": "Could not find project"}, status_code=404)
108+
except exceptions.AccessDeniedException:
109+
return JSONResponse({"error": "Access denied"}, status_code=403)
88110
file_name = request_body["file_name"]
89111
file_type = request_body["file_type"]
90112
file_import_options = request_body.get("file_import_options")
91-
task = task_manager.create_upload_task(
113+
task = upload_task_manager.create_upload_task(
92114
user_id, project_id, file_name, file_type, file_import_options
93115
)
94116
org_id = organization.get_id_by_project_id(project_id)
@@ -98,14 +120,63 @@ async def post(self, request) -> JSONResponse:
98120
return JSONResponse(credentials_and_id)
99121

100122

123+
class JSONImport(HTTPEndpoint):
124+
async def post(self, request) -> JSONResponse:
125+
auth.check_is_demo_without_info()
126+
project_id = request.path_params["project_id"]
127+
request_body = await request.json()
128+
user_id = request_body["user_id"]
129+
auth_manager.check_project_access_from_user_id(user_id, project_id)
130+
transfer_manager.import_records_from_json(
131+
project_id,
132+
user_id,
133+
request_body["records"],
134+
request_body["request_uuid"],
135+
request_body["is_last"],
136+
)
137+
return JSONResponse({"success": True})
138+
139+
140+
class AssociationsImport(HTTPEndpoint):
141+
async def post(self, request) -> JSONResponse:
142+
project_id = request.path_params["project_id"]
143+
request_body = await request.json()
144+
user_id = request_body["user_id"]
145+
try:
146+
auth_manager.check_project_access_from_user_id(
147+
user_id, project_id, from_api=True
148+
)
149+
except exceptions.EntityNotFoundException:
150+
return JSONResponse({"error": "Could not find project"}, status_code=404)
151+
except exceptions.AccessDeniedException:
152+
return JSONResponse({"error": "Access denied"}, status_code=403)
153+
new_associations_added = association_transfer_manager.import_associations(
154+
project_id,
155+
user_id,
156+
request_body["name"],
157+
request_body["label_task_name"],
158+
request_body["associations"],
159+
request_body["indices"],
160+
request_body["source_type"],
161+
)
162+
return JSONResponse(new_associations_added)
163+
164+
101165
class UploadTask(HTTPEndpoint):
102166
def get(self, request) -> JSONResponse:
103167
auth.check_is_demo_without_info()
104168
project_id = request.path_params["project_id"]
105169
task_id = request.path_params["task_id"]
106170
user_id = request.query_params["user_id"]
107-
auth_manager.check_project_access_from_user_id(user_id, project_id)
108-
task = task_manager.get_upload_task(project_id, task_id)
171+
try:
172+
auth_manager.check_project_access_from_user_id(
173+
user_id, project_id, from_api=True
174+
)
175+
except exceptions.EntityNotFoundException:
176+
return JSONResponse({"error": "Could not find project"}, status_code=404)
177+
except exceptions.AccessDeniedException:
178+
return JSONResponse({"error": "Access denied"}, status_code=403)
179+
task = upload_task_manager.get_upload_task(project_id, task_id)
109180
task_dict = {
110181
"id": str(task.id),
111182
"file_name": str(task.file_name),
@@ -119,7 +190,7 @@ def get(self, request) -> JSONResponse:
119190

120191
def init_file_import(task: UploadTask, project_id: str, is_global_update: bool) -> None:
121192
if "records" in task.file_type:
122-
upload_manager.import_records(project_id, task)
193+
upload_manager.import_records_from_file(project_id, task)
123194
elif "project" in task.file_type:
124195
upload_manager.import_project(project_id, task)
125196
elif "knowledge_base" in task.file_type:
@@ -145,7 +216,7 @@ def file_import_error_handling(
145216
task.file_type,
146217
)
147218
logger.error(
148-
task_manager.get_upload_task_message(
219+
upload_task_manager.get_upload_task_message(
149220
task,
150221
)
151222
)

0 commit comments

Comments
 (0)