add pdp stuff;s/_/- for routes

Author: CrysR

src/webapp/database.py

@@ -116,7 +116,7 @@ class InstTable(Base):
     schemas = Column(MutableList.as_mutable(JSON))
     state = Column(String(VAR_CHAR_LENGTH), nullable=True)
     # Only populated for PDP schools. Uncomment once logic available.
-    # pdp_id: Mapped[int] = mapped_column(nullable=True)
+    pdp_id: Mapped[int] = mapped_column(nullable=True)
     # A short description or note on this inst.
     description = Column(String(VAR_CHAR_LONGER_LENGTH))
     created_at = Column(DateTime(timezone=True), server_default=func.now())

src/webapp/main.py

@@ -127,7 +127,7 @@ async def login_for_access_token(
 
 
 # Get users that don't have institution specifications. (either datakinders or people who haven't set their institution yet)
-@app.get("/non_inst_users", response_model=list[users.UserAccount])
+@app.get("/non-inst-users", response_model=list[users.UserAccount])
 async def read_cross_inst_users(
     current_user: Annotated[BaseUser, Depends(get_current_active_user)],
     sql_session: Annotated[Session, Depends(get_session)],
@@ -206,7 +206,7 @@ async def set_datakinders(
     return res
 
 
-@app.get("/check_self", response_model=SelfInfo)
+@app.get("/check-self", response_model=SelfInfo)
 def read_self_info(
     current_user: Annotated[BaseUser, Depends(get_current_active_user)],
     sql_session: Annotated[Session, Depends(get_session)],

src/webapp/main_test.py

@@ -138,7 +138,7 @@ def test_retrieve_token(client: TestClient):
 
 def test_get_cross_isnt_users(client: TestClient):
     """Test GET /non_inst_users."""
-    response = client.get("/non_inst_users")
+    response = client.get("/non-inst-users")
     assert response.status_code == 200
     assert response.json() == [
         {
@@ -167,7 +167,7 @@ def test_set_datakinders(client: TestClient):
 
 def test_check_self_datakinder(client: TestClient):
     """Test GET /check_self."""
-    response = client.get("/check_self")
+    response = client.get("/check-self")
     assert response.status_code == 200
     assert response.json() == {
         "access_type": "DATAKINDER",
@@ -179,7 +179,7 @@ def test_check_self_datakinder(client: TestClient):
 
 def test_check_self(user_client: TestClient):
     """Test GET /check_self."""
-    response = user_client.get("/check_self")
+    response = user_client.get("/check-self")
     assert response.status_code == 200
     assert response.json() == {
         "access_type": "VIEWER",

src/webapp/routers/data.py

@@ -117,6 +117,7 @@ class ValidationResult(BaseModel):
     name: str
     inst_id: str
     file_types: set[SchemaType]
+    source: str
 
 
 class DataOverview(BaseModel):
@@ -261,7 +262,7 @@ def read_inst_all_input_files(
     }
 
 
-@router.get("/{inst_id}/input_debugging", response_model=list[str])
+@router.get("/{inst_id}/input-debugging", response_model=list[str])
 def get_all_files_in_bucket(
     inst_id: str,
     current_user: Annotated[BaseUser, Depends(get_current_active_user)],
@@ -549,7 +550,6 @@ def update_batch(
     model_owner_and_higher_or_err(current_user, "modify batch")
 
     update_data = request.model_dump(exclude_unset=True)
-    print("aaaaaaaaaaaaaaaaaaaaaaaa:" + str(update_data))
     local_session.set(sql_session)
     # Check that the batch exists.
     query_result = (
@@ -676,7 +676,7 @@ def update_batch(
 # TODO: check expiration of files and batches
 
 
-@router.get("/{inst_id}/file_id/{file_id}", response_model=DataInfo)
+@router.get("/{inst_id}/file-id/{file_id}", response_model=DataInfo)
 def read_file_id_info(
     inst_id: str,
     file_id: str,
@@ -796,7 +796,7 @@ def read_file_info(
 
 
 # TODO: ADD TESTS for the below
-@router.get("/{inst_id}/download_url/{file_name}", response_model=str)
+@router.get("/{inst_id}/download-url/{file_name}", response_model=str)
 def download_url_inst_file(
     inst_id: str,
     file_name: str,
@@ -867,15 +867,14 @@ def download_url_inst_file(
 # 3. Validate the file
 
 
-@router.post("/{inst_id}/input/validate/{file_name}", response_model=ValidationResult)
-def validate_file(
+def validation_helper(
+    source_str: str,
     inst_id: str,
     file_name: str,
-    current_user: Annotated[BaseUser, Depends(get_current_active_user)],
-    storage_control: Annotated[StorageControl, Depends(StorageControl)],
-    sql_session: Annotated[Session, Depends(get_session)],
+    current_user: BaseUser,
+    storage_control: StorageControl,
+    sql_session: Session,
 ) -> Any:
-    """Validate a given file. The file_name should not contain slashes."""
     has_access_to_inst_or_err(inst_id, current_user)
     if file_name.find("/") != -1:
         raise HTTPException(
@@ -916,7 +915,7 @@ def validate_file(
         name=file_name,
         inst_id=str_to_uuid(inst_id),
         uploader=str_to_uuid(current_user.user_id),
-        source="MANUAL_UPLOAD",
+        source=source_str,
         sst_generated=False,
         schemas=list(inferred_schemas),
         valid=True,
@@ -926,10 +925,48 @@ def validate_file(
         "name": file_name,
         "inst_id": inst_id,
         "file_types": inferred_schemas,
+        "source": source_str,
     }
 
 
-@router.get("/{inst_id}/upload_url/{file_name}", response_model=str)
+@router.post(
+    "/{inst_id}/input/validate-sftp/{file_name}", response_model=ValidationResult
+)
+def validate_file_sftp(
+    inst_id: str,
+    file_name: str,
+    current_user: Annotated[BaseUser, Depends(get_current_active_user)],
+    storage_control: Annotated[StorageControl, Depends(StorageControl)],
+    sql_session: Annotated[Session, Depends(get_session)],
+) -> Any:
+    """Validate a given file pulled from SFTP. The file_name should not contain slashes."""
+    if not current_user.is_datakinder:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="SFTP validation needs to be done by a datakinder.",
+        )
+    return validation_helper(
+        "PDP_SFTP", inst_id, file_name, current_user, storage_control, sql_session
+    )
+
+
+@router.post(
+    "/{inst_id}/input/validate-upload/{file_name}", response_model=ValidationResult
+)
+def validate_file_manual_upload(
+    inst_id: str,
+    file_name: str,
+    current_user: Annotated[BaseUser, Depends(get_current_active_user)],
+    storage_control: Annotated[StorageControl, Depends(StorageControl)],
+    sql_session: Annotated[Session, Depends(get_session)],
+) -> Any:
+    """Validate a given file. The file_name should not contain slashes."""
+    return validation_helper(
+        "MANUAL_UPLOAD", inst_id, file_name, current_user, storage_control, sql_session
+    )
+
+
+@router.get("/{inst_id}/upload-url/{file_name}", response_model=str)
 def get_upload_url(
     inst_id: str,
     file_name: str,

src/webapp/routers/data_test.py

@@ -404,7 +404,7 @@ def test_read_file_id_info(client: TestClient):
     response = client.get(
         "/institutions/"
         + uuid_to_str(UUID_INVALID)
-        + "/file_id/"
+        + "/file-id/"
         + uuid_to_str(FILE_UUID_1)
     )
 
@@ -417,7 +417,7 @@ def test_read_file_id_info(client: TestClient):
     response = client.get(
         "/institutions/"
         + uuid_to_str(USER_VALID_INST_UUID)
-        + "/file_id/"
+        + "/file-id/"
         + uuid_to_str(FILE_UUID_1)
     )
     assert response.status_code == 200
@@ -527,39 +527,81 @@ def test_update_batch(client: TestClient):
 def test_validate_success_batch(client: TestClient):
     """Test PATCH /institutions/<uuid>/batch."""
     MOCK_STORAGE.validate_file.return_value = {SchemaType.UNKNOWN}
-    response = client.post(
-        "/institutions/" + uuid_to_str(UUID_INVALID) + "/input/validate/file_name.csv",
+
+    # Use validate for manual upload
+    response_upload = client.post(
+        "/institutions/"
+        + uuid_to_str(UUID_INVALID)
+        + "/input/validate-upload/file_name.csv",
     )
-    assert str(response) == "<Response [401 Unauthorized]>"
+    assert str(response_upload) == "<Response [401 Unauthorized]>"
     assert (
-        response.text
+        response_upload.text
         == '{"detail":"Not authorized to read this institution\'s resources."}'
     )
     # Authorized.
-    response = client.post(
+    response_upload = client.post(
         "/institutions/"
         + uuid_to_str(USER_VALID_INST_UUID)
-        + "/input/validate/file_name.csv",
+        + "/input/validate-upload/file_name.csv",
     )
-    assert response.status_code == 200
-    assert response.json()["name"] == "file_name.csv"
-    assert response.json()["file_types"] == ["UNKNOWN"]
-    assert response.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_upload.status_code == 200
+    assert response_upload.json()["name"] == "file_name.csv"
+    assert response_upload.json()["file_types"] == ["UNKNOWN"]
+    assert response_upload.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_upload.json()["source"] == "MANUAL_UPLOAD"
+
+    # Use validate for SFTP
+    response_sftp = client.post(
+        "/institutions/"
+        + uuid_to_str(UUID_INVALID)
+        + "/input/validate-sftp/file_name.csv",
+    )
+    assert str(response_sftp) == "<Response [401 Unauthorized]>"
+    assert (
+        response_sftp.text
+        == '{"detail":"Not authorized to read this institution\'s resources."}'
+    )
+    # Authorized.
+    response_sftp = client.post(
+        "/institutions/"
+        + uuid_to_str(USER_VALID_INST_UUID)
+        + "/input/validate-sftp/file_name.csv",
+    )
+    assert response_sftp.status_code == 200
+    assert response_sftp.json()["name"] == "file_name.csv"
+    assert response_sftp.json()["file_types"] == ["UNKNOWN"]
+    assert response_sftp.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_sftp.json()["source"] == "PDP_SFTP"
 
 
 def test_validate_failure_batch(client: TestClient):
     """Test PATCH /institutions/<uuid>/batch."""
     MOCK_STORAGE.validate_file.return_value = {SchemaType.PDP_COHORT}
     # Authorized.
-    response = client.post(
+    # Use validate upload
+    response_upload = client.post(
         "/institutions/"
         + uuid_to_str(USER_VALID_INST_UUID)
-        + "/input/validate/file_name.csv",
+        + "/input/validate-upload/file_name.csv",
     )
-    assert response.status_code == 200
-    assert response.json()["name"] == "file_name.csv"
-    assert response.json()["file_types"] == ["PDP_COHORT"]
-    assert response.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_upload.status_code == 200
+    assert response_upload.json()["name"] == "file_name.csv"
+    assert response_upload.json()["file_types"] == ["PDP_COHORT"]
+    assert response_upload.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_upload.json()["source"] == "MANUAL_UPLOAD"
+
+    # Use valiate sftp
+    response_sftp = client.post(
+        "/institutions/"
+        + uuid_to_str(USER_VALID_INST_UUID)
+        + "/input/validate-upload/file_name.csv",
+    )
+    assert response_sftp.status_code == 200
+    assert response_sftp.json()["name"] == "file_name.csv"
+    assert response_sftp.json()["file_types"] == ["PDP_COHORT"]
+    assert response_sftp.json()["inst_id"] == uuid_to_str(USER_VALID_INST_UUID)
+    assert response_sftp.json()["source"] == "MANUAL_UPLOAD"
 
 
 def test_pull_pdp_sftp(client: TestClient):

src/webapp/routers/institutions.py

@@ -70,6 +70,7 @@ class InstitutionCreationRequest(BaseModel):
     allowed_emails: Dict[str, AccessType] | None = None
     # The following is a shortcut to specifying the allowed_schemas list and will mean that the allowed_schemas will be augmented with the PDP_SCHEMA_GROUP.
     is_pdp: bool | None = None
+    pdp_id: int | None = None
     retention_days: int | None = None
 
 
@@ -83,6 +84,7 @@ class Institution(BaseModel):
     # The following are characteristics of an institution set at institution creation time.
     # If zero, it follows DK defaults (deletion after completion).
     retention_days: int | None = None  # In Days
+    pdp_id: int | None = None
 
 
 @router.get("/institutions", response_model=list[Institution])
@@ -143,6 +145,12 @@ def create_institution(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Description length too long.",
         )
+    if (req.is_pdp and not req.pdp_id) or (req.pdp_id and not req.is_pdp):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Please set the PDP's Institution ID for PDP schools and check PDP as a schema type.",
+        )
+
     local_session.set(sql_session)
     query_result = (
         local_session.get()
@@ -168,6 +176,7 @@ def create_institution(
                 name=req.name,
                 retention_days=req.retention_days,
                 description=req.description,
+                pdp_id=req.pdp_id,
                 # Sets aren't json serializable, so turn them into lists first
                 schemas=list(set(requested_schemas)),
                 allowed_emails=req.allowed_emails,
@@ -210,6 +219,7 @@ def create_institution(
         "inst_id": uuid_to_str(query_result[0][0].id),
         "name": query_result[0][0].name,
         "state": query_result[0][0].state,
+        "pdp_id": query_result[0][0].pdp_id,
         "description": query_result[0][0].description,
         "retention_days": query_result[0][0].retention_days,
     }
@@ -253,6 +263,48 @@ def read_inst_name(
         "description": query_result[0][0].description,
         "retention_days": query_result[0][0].retention_days,
         "state": query_result[0][0].state,
+        "pdp_id": query_result[0][0].pdp_id,
+    }
+
+
+# All other API transactions require the UUID as an identifier, this allows the UUID lookup by PDP ID.
+@router.get("/institutions/pdp-id/{pdp_id}", response_model=Institution)
+def read_inst_pdp_id(
+    pdp_id: int,
+    current_user: Annotated[BaseUser, Depends(get_current_active_user)],
+    sql_session: Annotated[Session, Depends(get_session)],
+) -> Any:
+    """Returns overview data on a specific institution.
+
+    The root-level API view. Only visible to users of that institution or Datakinder access types.
+
+    Args:
+        current_user: the user making the request.
+    """
+    local_session.set(sql_session)
+    query_result = (
+        local_session.get()
+        .execute(select(InstTable).where(InstTable.pdp_id == pdp_id))
+        .all()
+    )
+    if len(query_result) == 0:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Institution not found.",
+        )
+    if len(query_result) > 1:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Institution duplicates found.",
+        )
+    has_access_to_inst_or_err(uuid_to_str(query_result[0][0].id), current_user)
+    return {
+        "inst_id": uuid_to_str(query_result[0][0].id),
+        "name": query_result[0][0].name,
+        "description": query_result[0][0].description,
+        "retention_days": query_result[0][0].retention_days,
+        "state": query_result[0][0].state,
+        "pdp_id": query_result[0][0].pdp_id,
     }
 
 
@@ -292,4 +344,5 @@ def read_inst_id(
         "description": query_result[0][0].description,
         "retention_days": query_result[0][0].retention_days,
         "state": query_result[0][0].state,
+        "pdp_id": query_result[0][0].pdp_id,
     }