diff --git a/api/src/api/common_grants/COMMON_GRANTS_INTEGRATION.md b/api/src/api/common_grants/COMMON_GRANTS_INTEGRATION.md index 1b76032019..c7dd32c55d 100644 --- a/api/src/api/common_grants/COMMON_GRANTS_INTEGRATION.md +++ b/api/src/api/common_grants/COMMON_GRANTS_INTEGRATION.md @@ -288,3 +288,31 @@ make lint-ruff # Type checking make lint-mypy ``` + + +## Adding Custom Fields + +To add custom fields to the openAPI sample documentation follow these steps: +1. Add any new custom fields to the marshmallow schemas in `common_grants_custom_fields.py` this file is the container for all existing custom fields +2. Inside of `common_grants_schemas.py` update the `CustomFields` class to have an additional field named after the field that has been added and set it's value equal to the newly imported custom field. Like so +```python +class CustomFields(Schema): + legacyId = fields.Nested(LegacyId, allow_none=True) + federalOpportunityNumber = fields.Nested(FederalOpportunityNumber, allow_none=True) + assistanceListing = fields.Nested(AssistanceListing, allow_none=True) + agency = fields.Nested(Agency, allow_none=True) + attachments = fields.Nested(Attachments, allow_none=True) + category = fields.Nested(Category, allow_none=True) + fiscalYear = fields.Nested(FiscalYear, allow_none=True) + costSharing = fields.Nested(CostSharing, allow_none=True) + additionalInfo = fields.Nested(AdditionalInfo, allow_none=True) + AgencyContact = fields.Nested(AgencyContact, allow_none=True) + yourNewestField = fields.Nexted(YourNewestField, allow_none=True) + +``` + + +3. To verify the changes run `make openapi-spec-common-grants` at the first `/api` directory in the repository and then run `make init make db-seed-local && make populate-search-opportunities make run-logs` to stand up the openApi endpoints + +4. After the Docker containers have started go to `localhost:8080/docs` and enter the user key for the `common grants` endpoints. +5. Run the endpoints to validate responses. \ No newline at end of file diff --git a/api/src/api/common_grants/common_grants_custom_fields.py b/api/src/api/common_grants/common_grants_custom_fields.py new file mode 100644 index 0000000000..66e94bdd64 --- /dev/null +++ b/api/src/api/common_grants/common_grants_custom_fields.py @@ -0,0 +1,231 @@ +""" Marshmallow schemas for CommonGrants Protocol customFields + +This file contains Marshmallow schemas that are part of the custom field portion of the specification. + +NOTE: Once added here the fields should be imported into another Marshmallow file to register them under the CustomFields class there. +At this time there is only one such file and that is common_grants_schemas.py which only supports Opportunity.py + +This pattern allows for simple re-use of custom fields across different base objects since if the fields already exist +it should be as simple as importing the already existing field into the file that requires it and adding it to the CustomFields class +as a new property. +""" + +from typing import Any + +from src.api.schemas.extension import Schema, fields +from src.api.schemas.extension import validators as validate + + +class CustomFieldType(fields.String): + """Enum field for custom field types.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__( + validate=validate.OneOf(["string", "number", "integer", "boolean", "object", "array"]), + metadata={ + "description": "The JSON schema type to use when de-serializing the value field" + }, + **kwargs + ) + + +class CustomField(Schema): + """Schema for defining custom fields on a record.""" + + fieldType: fields.String + value: fields.MixinField + + name = fields.String(required=True, metadata={"example": "eligible_applicants"}) + fieldType = CustomFieldType(required=True) + schema = fields.URL(allow_none=True, metadata={"example": "https://example.com/schema"}) + value = fields.Raw(required=True, metadata={"example": "nonprofits, state governments"}) + description = fields.String( + allow_none=True, + metadata={"example": "The types of organizations eligible to apply"}, + ) + + +# =========================================================================================== +# Custom Field Implementations +# =========================================================================================== + + +class legacySerialId(CustomField): + """Storing legacy id for compatibility iwth legacy systems""" + + name = fields.String(required=True, metadata={"example": "legacySerialId"}) + fieldType = fields.String(required=True, metadata={"example": "integer"}) + value = fields.Integer(required=True, metadata={"example": "12345"}) + description = fields.String( + allow_none=True, + metadata={ + "example": "An integer ID for the opportunity, needed for compatibility with legacy systems" + }, + ) + + +class federalOpportunityNumber(CustomField): + """Federal Opportunity Number assigned to this grant opportunity""" + + name = fields.String(required=True, metadata={"example": "federalOpportunityNumber"}) + fieldType = fields.String(required=True, metadata={"example": "string"}) + value = fields.String(required=True, metadata={"example": "ABC-123-XYZ-001"}) + description = fields.String( + allow_none=True, + metadata={"example": "The federal opportunity number assigned to this grant opportunity"}, + ) + + +class AssistanceListingValue(Schema): + """Schema for populating the AssistanceListing value field""" + + assistanceListingNumber = fields.String(required=True, metadata={"example": "43.012"}) + programTitle = fields.String(required=True, metadata={"example": "Space Technology"}) + + +class assistanceListings(CustomField): + """The assistance listing number and program title for this opportunity""" + + name = fields.String(required=True, metadata={"example": "assistanceListings"}) + fieldType = fields.String(required=True, metadata={"example": "array"}) + value = fields.List(fields.Nested(AssistanceListingValue), required=True) + description = fields.String( + allow_none=True, + metadata={ + "example": "The assistance listing number and program title for this opportunity" + }, + ) + + +class AgencyValue(Schema): + """Schema for populating the Agency value field""" + + agencyCode = fields.String(required=True, metadata={"example": "US-ABC"}) + agencyName = fields.String(allow_none=True, metadata={"example": "Department of Examples"}) + topLevelAgencyName = fields.String( + allow_none=True, metadata={"example": "Department of Examples"} + ) + + +class agency(CustomField): + """Information about the agency offering this opportunity""" + + name = fields.String(required=True, metadata={"example": "agency"}) + fieldType = fields.String(required=True, metadata={"example": "object"}) + value = fields.Nested(AgencyValue, required=True) + description = fields.String( + allow_none=True, + metadata={"example": "Information about the agency offering this opportunity"}, + ) + + +class AttachmentValue(Schema): + downloadUrl = fields.URL(allow_none=True, metadata={"example": "https://example.com/file.pdf"}) + name = fields.String(required=True, metadata={"example": "example.pdf"}) + description = fields.String( + allow_none=True, metadata={"example": "A PDF file with instructions"} + ) + sizeInBytes = fields.Integer(required=True, metadata={"example": 1000}) + mimeType = fields.String(required=True, metadata={"example": "application/pdf"}) + createdAt = fields.DateTime(required=True, metadata={"example": "2025-01-01T17:01:01.000Z"}) + lastModifiedAt = fields.DateTime( + required=True, metadata={"example": "2025-01-02T17:30:00.000Z"} + ) + + +class attachments(CustomField): + """Attachments such as NOFOs or other supplemental documents""" + + name = fields.String(required=True, metadata={"example": "attachments"}) + fieldType = fields.String(required=True, metadata={"example": "array"}) + value = fields.List(fields.Nested(AttachmentValue), required=True) + description = fields.String( + allow_none=True, + metadata={ + "example": "Attachments such as NOFOs and supplemental documents for the opportunity" + }, + ) + + +class federalFundingSource(CustomField): + """The category type of the grant opportunity""" + + name = fields.String(required=True, metadata={"example": "federalFundingSource"}) + fieldType = fields.String(required=True, metadata={"example": "string"}) + value = fields.String(required=True, metadata={"example": "discretionary"}) + description = fields.String( + allow_none=True, + metadata={"example": "The category type of the grant opportunity"}, + ) + + +class AgencyContactValue(Schema): + """Schema for populating the AgencyContact value field""" + + description = fields.String( + allow_none=True, + metadata={"example": "For more information, reach out to Jane Smith at agency US-ABC"}, + ) + emailAddress = fields.String(allow_none=True, metadata={"example": "fake_email@grants.gov"}) + emailDescription = fields.String( + allow_none=True, metadata={"example": "Click me to email the agency"} + ) + + +class contactInfo(CustomField): + """Contact information for the agency managing this opportunity""" + + name = fields.String(required=True, metadata={"example": "contactInfo"}) + fieldType = fields.String(required=True, metadata={"example": "object"}) + value = fields.Nested(AgencyContactValue, required=True) + description = fields.String( + allow_none=True, + metadata={"example": "Contact information for the agency managing this opportunity"}, + ) + + +class AdditionalInfoValue(Schema): + """Schema for populating the AdditionalInfo value field""" + + url = fields.String(allow_none=True, metadata={"example": "grants.gov"}) + description = fields.String(allow_none=True, metadata={"example": "Click me for more info"}) + + +class additionalInfo(CustomField): + """URL and description for additional information about the opportunity""" + + name = fields.String(required=True, metadata={"example": "additionalInfo"}) + fieldType = fields.String(required=True, metadata={"example": "object"}) + value = fields.Nested(AdditionalInfoValue, required=True) + description = fields.String( + allow_none=True, + metadata={ + "example": "URL and description for additional information about the opportunity" + }, + ) + + +class costSharing(CustomField): + """Whether cost sharing or matching funds are required for this opportunity""" + + name = fields.String(required=True, metadata={"example": "costSharing"}) + fieldType = fields.String(required=True, metadata={"example": "boolean"}) + value = fields.Boolean(required=True, metadata={"example": True}) + description = fields.String( + allow_none=True, + metadata={ + "example": "Whether cost sharing or matching funds are required for this opportunity" + }, + ) + + +class fiscalYear(CustomField): + """The fiscal year associated with this opportunity""" + + name = fields.String(required=True, metadata={"example": "fiscalYear"}) + fieldType = fields.String(required=True, metadata={"example": "number"}) + value = fields.Integer(required=True, metadata={"example": 2026}) + description = fields.String( + allow_none=True, + metadata={"example": "The fiscal year associated with this opportunity"}, + ) diff --git a/api/src/api/common_grants/common_grants_schemas.py b/api/src/api/common_grants/common_grants_schemas.py index 3be18dca5e..a5281e16c9 100644 --- a/api/src/api/common_grants/common_grants_schemas.py +++ b/api/src/api/common_grants/common_grants_schemas.py @@ -3,18 +3,22 @@ This file contains all Marshmallow schemas that correspond to Pydantic models, organized by category for better maintainability. -NOTICE: This file is COPIED directly from -simpler-grants-protocol/lib/python-sdk/common_grants_sdk/schemas/marshmallow/ -and then MODIFIED to change imports as described below. - - ORIGINAL: - from marshmallow import Schema, fields, validate - MODIFIED - from src.api.schemas.extension import Schema, fields, validators as validate """ from typing import Any +from src.api.common_grants.common_grants_custom_fields import ( + additionalInfo, + agency, + assistanceListings, + attachments, + contactInfo, + costSharing, + federalFundingSource, + federalOpportunityNumber, + fiscalYear, + legacySerialId, +) from src.api.schemas.extension import Schema, fields from src.api.schemas.extension import validators as validate @@ -215,32 +219,6 @@ class SystemMetadata(Schema): ) -class CustomFieldType(fields.String): - """Enum field for custom field types.""" - - def __init__(self, **kwargs: Any) -> None: - super().__init__( - validate=validate.OneOf(["string", "number", "integer", "boolean", "object", "array"]), - metadata={ - "description": "The JSON schema type to use when de-serializing the value field" - }, - **kwargs - ) - - -class CustomField(Schema): - """Schema for defining custom fields on a record.""" - - name = fields.String(required=True, metadata={"example": "eligible_applicants"}) - fieldType = CustomFieldType(required=True) - schema = fields.URL(allow_none=True, metadata={"example": "https://example.com/schema"}) - value = fields.Raw(required=True, metadata={"example": "nonprofits, state governments"}) - description = fields.String( - allow_none=True, - metadata={"example": "The types of organizations eligible to apply"}, - ) - - # ============================================================================= # FILTER INFO MODELS # ============================================================================= @@ -356,6 +334,24 @@ class OppTimeline(Schema): ) +class OpportunityCustomFields(Schema): + """ + This class serves as the collection point for all custom fields on the OpportunityBase schema. + It is passed to (camelCase) customFields on OpportunityBase with the fields.nested property + """ + + legacySerialId = fields.Nested(legacySerialId, allow_none=True) + federalOpportunityNumber = fields.Nested(federalOpportunityNumber, allow_none=True) + assistanceListings = fields.Nested(assistanceListings, allow_none=True) + agency = fields.Nested(agency, allow_none=True) + attachments = fields.Nested(attachments, allow_none=True) + federalFundingSource = fields.Nested(federalFundingSource, allow_none=True) + fiscalYear = fields.Nested(fiscalYear, allow_none=True) + costSharing = fields.Nested(costSharing, allow_none=True) + additionalInfo = fields.Nested(additionalInfo, allow_none=True) + contactInfo = fields.Nested(contactInfo, allow_none=True) + + class OpportunityBase(Schema): """Base opportunity model.""" @@ -415,9 +411,8 @@ class OpportunityBase(Schema): "example": "https://grants.gov/web/grants/view-opportunity.html?oppId=12345", }, ) - customFields = fields.Dict( - keys=fields.String(), - values=fields.Nested(CustomField), + customFields = fields.Nested( + OpportunityCustomFields, allow_none=True, metadata={"description": "Additional custom fields specific to this opportunity"}, ) diff --git a/api/src/services/common_grants/transformation.py b/api/src/services/common_grants/transformation.py index 03816adae1..cbc9941a91 100644 --- a/api/src/services/common_grants/transformation.py +++ b/api/src/services/common_grants/transformation.py @@ -4,6 +4,8 @@ from datetime import date, datetime from common_grants_sdk.schemas.pydantic import ( + CustomField, + CustomFieldType, FilterInfo, Money, MoneyRangeFilter, @@ -194,6 +196,19 @@ def transform_opportunity_to_cg(v1_opportunity: Opportunity) -> OpportunityBase "opportunity_id": v1_opportunity.opportunity_id, "opportunity_title": v1_opportunity.opportunity_title or "Untitled Opportunity", "opportunity_status": v1_opportunity.opportunity_status, + "legacy_opportunity_id": v1_opportunity.legacy_opportunity_id, + "opportunity_number": v1_opportunity.opportunity_number, + "category": v1_opportunity.category, + "agency_code": v1_opportunity.agency_code, + "agency_name": v1_opportunity.agency_name, + "top_level_agency_name": v1_opportunity.top_level_agency_name, + "opportunity_assistance_listings": [ + { + "assistance_listing_number": listing.assistance_listing_number, + "program_title": listing.program_title, + } + for listing in v1_opportunity.opportunity_assistance_listings + ], "created_at": v1_opportunity.created_at, "updated_at": v1_opportunity.updated_at, "summmary": {}, @@ -209,10 +224,29 @@ def transform_opportunity_to_cg(v1_opportunity: Opportunity) -> OpportunityBase "award_ceiling": v1_opportunity.summary.award_ceiling, "award_floor": v1_opportunity.summary.award_floor, "additional_info_url": v1_opportunity.summary.additional_info_url, + "additional_info_url_description": v1_opportunity.summary.additional_info_url_description, + "agency_contact_description": v1_opportunity.summary.agency_contact_description, + "agency_email_address": v1_opportunity.summary.agency_email_address, + "agency_email_address_description": v1_opportunity.summary.agency_email_address_description, + "fiscal_year": v1_opportunity.summary.fiscal_year, + "is_cost_sharing": v1_opportunity.summary.is_cost_sharing, "created_at": v1_opportunity.summary.created_at, "updated_at": v1_opportunity.summary.updated_at, } + opp_data["opportunity_attachments"] = [ + { + "download_path": attachment.download_path, + "file_name": attachment.file_name, + "file_description": attachment.file_description, + "file_size_bytes": attachment.file_size_bytes, + "mime_type": attachment.mime_type, + "created_at": attachment.created_at, + "updated_at": attachment.updated_at, + } + for attachment in v1_opportunity.opportunity_attachments + ] + return transform_search_result_to_cg(opp_data) @@ -295,7 +329,7 @@ def transform_search_result_to_cg(opp_data: dict) -> OpportunityBase | None: minAwardAmount=min_award_money, ), source=validate_url(summary.get("additional_info_url")), - custom_fields={}, + customFields=populate_custom_fields(opp_data), createdAt=summary.get("created_at") or datetime_util.utcnow(), lastModifiedAt=summary.get("updated_at") or datetime_util.utcnow(), ) @@ -310,6 +344,152 @@ def transform_search_result_to_cg(opp_data: dict) -> OpportunityBase | None: return None +def populate_custom_fields(opp_data: dict) -> dict[str, CustomField] | None: + """ + Helper function to assemble custom fields from the data and pass them back as part of the response. + + Args: + opp_data: Opportunity data from the SGG database + + Returns: + custom_fields: A dict with the values recovered from the input and stored in the appropriate field + """ + + custom_fields: dict[str, CustomField] = {} + summary = opp_data.get("summary") + + attachments = opp_data.get("opportunity_attachments") + if attachments: + attachment_values = [ + { + "downloadUrl": attachment.get("download_path"), + "name": attachment.get("file_name"), + "description": attachment.get("file_description"), + "sizeInBytes": attachment.get("file_size_bytes"), + "mimeType": attachment.get("mime_type"), + "createdAt": attachment.get("created_at"), + "lastModifiedAt": attachment.get("updated_at"), + } + for attachment in attachments + ] + custom_fields["attachments"] = CustomField( + name="attachments", + fieldType=CustomFieldType.ARRAY, + value=attachment_values, + description="Attachments such as NOFOs and supplemental documents for the opportunity", + ) + + legacy_opportunity_id = opp_data.get("legacy_opportunity_id") + if legacy_opportunity_id is not None: + custom_fields["legacySerialId"] = CustomField( + name="legacySerialId", + fieldType=CustomFieldType.INTEGER, + value=legacy_opportunity_id, + description="An integer ID for the opportunity, needed for compatibility with legacy systems", + ) + + federal_opportunity_number = opp_data.get("opportunity_number") + if federal_opportunity_number is not None: + custom_fields["federalOpportunityNumber"] = CustomField( + name="federalOpportunityNumber", + fieldType=CustomFieldType.STRING, + value=federal_opportunity_number, + description="The federal opportunity number assigned to this grant opportunity", + ) + + listings = opp_data.get("opportunity_assistance_listings") + if listings: + listing_values = [ + { + "assistanceListingNumber": listing.get("assistance_listing_number"), + "programTitle": listing.get("program_title"), + } + for listing in listings + ] + custom_fields["assistanceListings"] = CustomField( + name="assistanceListings", + fieldType=CustomFieldType.ARRAY, + value=listing_values, + description="The assistance listing number and program title for this opportunity", + ) + + category = opp_data.get("category") + if category is not None: + custom_fields["federalFundingSource"] = CustomField( + name="federalFundingSource", + fieldType=CustomFieldType.STRING, + value=str(category), + description="The category type of the grant opportunity", + ) + + agency = opp_data.get("agency_code") + if agency is not None: + custom_fields["agency"] = CustomField( + name="agency", + fieldType=CustomFieldType.OBJECT, + value={ + "agencyCode": agency, + "agencyName": opp_data.get("agency_name"), + "topLevelAgencyName": opp_data.get("top_level_agency_name"), + }, + description="Information about the agency offering this opportunity", + ) + + if summary: + agency_contact_description = summary.get("agency_contact_description") + agency_email_address = summary.get("agency_email_address") + agency_email_address_description = summary.get("agency_email_address_description") + if any( + [agency_contact_description, agency_email_address, agency_email_address_description] + ): + custom_fields["contactInfo"] = CustomField( + name="contactInfo", + fieldType=CustomFieldType.OBJECT, + value={ + "description": agency_contact_description, + "emailAddress": agency_email_address, + "emailDescription": agency_email_address_description, + }, + description="Contact information for the agency managing this opportunity", + ) + + additional_info_url = summary.get("additional_info_url") + additional_info_url_description = summary.get("additional_info_url_description") + if additional_info_url is not None: + custom_fields["additionalInfo"] = CustomField( + name="additionalInfo", + fieldType=CustomFieldType.OBJECT, + value={ + "url": additional_info_url, + "description": additional_info_url_description, + }, + description="URL and description for additional information about the opportunity", + ) + + fiscal_year = summary.get("fiscal_year") + if fiscal_year is not None: + custom_fields["fiscalYear"] = CustomField( + name="fiscalYear", + fieldType=CustomFieldType.NUMBER, + value=fiscal_year, + description="The fiscal year associated with this opportunity", + ) + + is_cost_sharing = summary.get("is_cost_sharing") + if is_cost_sharing is not None: + custom_fields["costSharing"] = CustomField( + name="costSharing", + fieldType=CustomFieldType.BOOLEAN, + value=is_cost_sharing, + description="Whether cost sharing or matching funds are required for this opportunity", + ) + + if custom_fields: + return custom_fields + else: + return None + + def build_money_range_filter( money_range_filter: MoneyRangeFilter | None, v1_field_name: str, v1_filters: dict ) -> None: diff --git a/api/tests/src/api/common_grants/test_opportunities_routes.py b/api/tests/src/api/common_grants/test_opportunities_routes.py index 2bd96f4102..807862d446 100644 --- a/api/tests/src/api/common_grants/test_opportunities_routes.py +++ b/api/tests/src/api/common_grants/test_opportunities_routes.py @@ -3,6 +3,7 @@ import uuid from datetime import datetime +import marshmallow from flask.testing import FlaskClient from tests.src.db.models.factories import OpportunityFactory @@ -394,3 +395,57 @@ def test_search_opportunities_response_schema( assert "paginationInfo" in data assert "filterInfo" in data assert "sortInfo" in data + + +class TestSchemaValidation: + """Test Marshmallow schema validation for request and response paths.""" + + def test_invalid_sort_by_returns_422( + self, client: FlaskClient, enable_factory_create, db_session, user_api_key_id + ): + """Invalid sortBy value is rejected by APIFlask input validation → 422.""" + response = client.post( + "/common-grants/opportunities/search", + headers={"X-API-Key": user_api_key_id}, + json={"sorting": {"sortBy": "not_a_valid_field", "sortOrder": "asc"}}, + ) + assert response.status_code == 422 + + def test_invalid_filter_operator_returns_422( + self, client: FlaskClient, enable_factory_create, db_session, user_api_key_id + ): + """Invalid filter operator is rejected by APIFlask input validation → 422.""" + response = client.post( + "/common-grants/opportunities/search", + headers={"X-API-Key": user_api_key_id}, + json={"filters": {"status": {"operator": "invalid_op", "value": ["open"]}}}, + ) + assert response.status_code == 422 + + def test_response_schema_error_returns_500( + self, + client: FlaskClient, + enable_factory_create, + db_session, + user_api_key_id, + monkeypatch, + ): + """Marshmallow ValidationError raised during response loading → 500 with runtime message.""" + opportunity = OpportunityFactory.create(is_draft=False) + + def raise_validation_error(*args, **kwargs): + raise marshmallow.ValidationError("simulated response schema failure") + + monkeypatch.setattr( + "src.api.common_grants.common_grants_routes.OpportunityResponseSchema.load", + raise_validation_error, + ) + + response = client.get( + f"/common-grants/opportunities/{opportunity.opportunity_id}", + headers={"X-API-Key": user_api_key_id}, + ) + assert response.status_code == 500 + data = response.get_json() + assert "message" in data + assert "CommonGrants runtime exception" in data["message"] diff --git a/api/tests/src/services/common_grants/test_transformation.py b/api/tests/src/services/common_grants/test_transformation.py index a948f73396..fbfdeee498 100644 --- a/api/tests/src/services/common_grants/test_transformation.py +++ b/api/tests/src/services/common_grants/test_transformation.py @@ -6,6 +6,7 @@ from common_grants_sdk.schemas.pydantic import ( ArrayOperator, + CustomFieldType, Money, MoneyRange, MoneyRangeFilter, @@ -19,10 +20,12 @@ ) from freezegun import freeze_time +from src.api.common_grants.common_grants_schemas import OpportunityCustomFields from src.constants.lookup_constants import CommonGrantsEvent, OpportunityStatus from src.services.common_grants.transformation import ( build_filter_info, build_money_range_filter, + populate_custom_fields, transform_opportunity_to_cg, transform_search_request_from_cg, transform_search_result_to_cg, @@ -57,6 +60,26 @@ def _legacy_validate_url(value: str | None) -> str | None: return None +DEFAULT_MOCK_OPP_FIELDS = { + "legacy_opportunity_id": 67890, + "opportunity_number": "2024-010", + "category": "Mandatory", + "agency_code": "A2345", + "agency_name": "Testing Agency", + "top_level_agency_name": "Testing top level agency", + "opportunity_assistance_listings": [ + type( + "MockAssistanceListing", + (), + { + "assistance_listing_number": "10.557", + "program_title": "Special Supplemental Nutrition Program", + }, + )() + ], +} + + class TestTransformation: """Test the transformation functions.""" @@ -71,6 +94,7 @@ def __init__(self): self.opportunity_status = OpportunityStatus.POSTED self.created_at = datetime(2024, 1, 1, 12, 0, 0) # Changed from date to datetime self.updated_at = datetime(2024, 1, 2, 12, 0, 0) # Changed from date to datetime + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) self.current_opportunity_summary = type( "MockSummary", (), @@ -86,6 +110,12 @@ def __init__(self): "award_ceiling": 500000, "award_floor": 10000, "additional_info_url": "https://example.com/opportunity", + "additional_info_url_description": "Additional opportunity information", + "agency_contact_description": "Contact the grants office for questions", + "agency_email_address": "grants@test-agency.gov", + "agency_email_address_description": "Grants Office Email", + "fiscal_year": 2024, + "is_cost_sharing": False, "created_at": datetime(2024, 1, 3, 12, 0, 0), "updated_at": datetime(2024, 1, 4, 12, 0, 0), }, @@ -93,11 +123,27 @@ def __init__(self): }, )() self.summary = self.current_opportunity_summary.opportunity_summary + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "nofo.pdf", + "file_description": "Notice of Funding Opportunity", + "file_size_bytes": 204800, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 2, 12, 0, 0), + }, + )() + ] opportunity = MockOpportunity() result = transform_opportunity_to_cg(opportunity) + assert result is not None assert result.id == opportunity.opportunity_id assert result.title == "Test Opportunity" assert result.description == "Test description" @@ -126,6 +172,70 @@ def __init__(self): # Check source URL assert str(result.source) == "https://example.com/opportunity" + # Check custom fields + assert result.custom_fields is not None + + assert "legacySerialId" in result.custom_fields + assert result.custom_fields["legacySerialId"].value == opportunity.legacy_opportunity_id + + assert "federalOpportunityNumber" in result.custom_fields + assert result.custom_fields["federalOpportunityNumber"].value == "2024-010" + + assert "federalFundingSource" in result.custom_fields + assert result.custom_fields["federalFundingSource"].value == "Mandatory" + + assert "agency" in result.custom_fields + assert result.custom_fields["agency"].value["agencyCode"] == "A2345" + assert result.custom_fields["agency"].value["agencyName"] == "Testing Agency" + assert ( + result.custom_fields["agency"].value["topLevelAgencyName"] == "Testing top level agency" + ) + + assert "assistanceListings" in result.custom_fields + assert len(result.custom_fields["assistanceListings"].value) == 1 + assert ( + result.custom_fields["assistanceListings"].value[0]["assistanceListingNumber"] + == "10.557" + ) + assert ( + result.custom_fields["assistanceListings"].value[0]["programTitle"] + == "Special Supplemental Nutrition Program" + ) + + assert "contactInfo" in result.custom_fields + assert ( + result.custom_fields["contactInfo"].value["description"] + == "Contact the grants office for questions" + ) + assert result.custom_fields["contactInfo"].value["emailAddress"] == "grants@test-agency.gov" + assert ( + result.custom_fields["contactInfo"].value["emailDescription"] == "Grants Office Email" + ) + + assert "additionalInfo" in result.custom_fields + assert ( + result.custom_fields["additionalInfo"].value["url"] == "https://example.com/opportunity" + ) + assert ( + result.custom_fields["additionalInfo"].value["description"] + == "Additional opportunity information" + ) + + assert "fiscalYear" in result.custom_fields + assert result.custom_fields["fiscalYear"].value == 2024 + + assert "costSharing" in result.custom_fields + assert result.custom_fields["costSharing"].value is False + + assert "attachments" in result.custom_fields + assert len(result.custom_fields["attachments"].value) == 1 + attachment = result.custom_fields["attachments"].value[0] + assert attachment["downloadUrl"] == "https://example.com/opportunity" + assert attachment["name"] == "nofo.pdf" + assert attachment["description"] == "Notice of Funding Opportunity" + assert attachment["sizeInBytes"] == 204800 + assert attachment["mimeType"] == "application/pdf" + def test_url_validation_and_fixing(self): """Test that URLs are properly validated and fixed.""" @@ -166,6 +276,22 @@ def __init__(self, status): self.updated_at = datetime( 2024, 1, 1, 12, 0, 0 ) # Changed from date to datetime + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "nofo.pdf", + "file_description": "Notice of Funding Opportunity", + "file_size_bytes": 102400, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 1, 12, 0, 0), + }, + )() + ] self.current_opportunity_summary = type( "MockSummary", (), @@ -174,13 +300,19 @@ def __init__(self, status): "MockOppSummary", (), { - "summary_description": "Test", - "post_date": None, - "close_date": None, - "estimated_total_program_funding": None, - "award_ceiling": None, - "award_floor": None, - "additional_info_url": None, + "summary_description": "Test summary description", + "post_date": date(2024, 1, 1), + "close_date": date(2024, 12, 31), + "estimated_total_program_funding": 500000, + "award_ceiling": 100000, + "award_floor": 5000, + "additional_info_url": "https://example.com/opportunity", + "additional_info_url_description": "Grant program details", + "agency_contact_description": "Contact the program office", + "agency_email_address": "example@test.gov", + "agency_email_address_description": "Program Office Email", + "fiscal_year": 2024, + "is_cost_sharing": False, "created_at": datetime(2024, 1, 1, 12, 0, 0), "updated_at": datetime(2024, 1, 1, 12, 0, 0), }, @@ -191,6 +323,7 @@ def __init__(self, status): opportunity = MockOpportunity(db_status) result = transform_opportunity_to_cg(opportunity) + assert result is not None assert result.status.value == expected_status @freeze_time("2024-01-03 12:00:00") @@ -208,6 +341,22 @@ def __init__(self): self.updated_at = datetime( 2024, 1, 1, 12, 0, 0 ) # Provide a default datetime instead of None + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "synopsis.pdf", + "file_description": "Opportunity Synopsis", + "file_size_bytes": 51200, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 1, 12, 0, 0), + }, + )() + ] self.current_opportunity_summary = None self.summary = None @@ -215,6 +364,7 @@ def __init__(self): result = transform_opportunity_to_cg(opportunity) # The transformation should now succeed even with None summary + assert result is not None assert result.id == opportunity.opportunity_id assert result.title == "Untitled Opportunity" assert result.description == "No description available" @@ -242,6 +392,22 @@ def __init__(self): self.opportunity_status = type("MockStatus", (), {"value": "posted"})() self.created_at = datetime(2024, 1, 1, 12, 0, 0) # Changed from date to datetime self.updated_at = datetime(2024, 1, 2, 12, 0, 0) # Changed from date to datetime + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "solicitation.pdf", + "file_description": "Grant Solicitation", + "file_size_bytes": 307200, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 2, 12, 0, 0), + }, + )() + ] self.current_opportunity_summary = type( "MockSummary", (), @@ -257,6 +423,12 @@ def __init__(self): "award_ceiling": None, "award_floor": 10000, "additional_info_url": None, + "additional_info_url_description": None, + "agency_contact_description": "Contact the NSF program office", + "agency_email_address": "example@test.gov", + "agency_email_address_description": "NSF Program Office", + "fiscal_year": 2024, + "is_cost_sharing": True, "created_at": datetime(2024, 1, 3, 12, 0, 0), "updated_at": datetime(2024, 1, 4, 12, 0, 0), }, @@ -268,6 +440,7 @@ def __init__(self): opportunity = MockOpportunity() result = transform_opportunity_to_cg(opportunity) + assert result is not None assert result.description == "No description available" assert result.key_dates is not None assert result.key_dates.close_date is not None @@ -285,6 +458,22 @@ def __init__(self): self.opportunity_status = type("MockStatus", (), {"value": "unknown_status"})() self.created_at = datetime(2024, 1, 1, 12, 0, 0) # Changed from date to datetime self.updated_at = datetime(2024, 1, 1, 12, 0, 0) # Changed from date to datetime + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "announcement.pdf", + "file_description": "Grant Announcement", + "file_size_bytes": 153600, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 1, 12, 0, 0), + }, + )() + ] self.current_opportunity_summary = type( "MockSummary", (), @@ -293,13 +482,19 @@ def __init__(self): "MockOppSummary", (), { - "summary_description": "Test", - "post_date": None, - "close_date": None, - "estimated_total_program_funding": None, - "award_ceiling": None, - "award_floor": None, - "additional_info_url": None, + "summary_description": "Test summary for unknown status", + "post_date": date(2024, 3, 1), + "close_date": date(2024, 9, 30), + "estimated_total_program_funding": 750000, + "award_ceiling": 250000, + "award_floor": 25000, + "additional_info_url": "https://example.com/opportunity", + "additional_info_url_description": "Example", + "agency_contact_description": "Example Program Office", + "agency_email_address": "test@example.gov", + "agency_email_address_description": "Example Test Email", + "fiscal_year": 2024, + "is_cost_sharing": False, "created_at": datetime(2024, 1, 1, 12, 0, 0), "updated_at": datetime(2024, 1, 1, 12, 0, 0), }, @@ -311,6 +506,7 @@ def __init__(self): opportunity = MockOpportunity() result = transform_opportunity_to_cg(opportunity) + assert result is not None assert result.status.value == OppStatusOptions.FORECASTED def test_transformation_with_url_fixing(self): @@ -323,6 +519,22 @@ def __init__(self): self.opportunity_status = type("MockStatus", (), {"value": "posted"})() self.created_at = datetime(2024, 1, 1, 12, 0, 0) self.updated_at = datetime(2024, 1, 2, 12, 0, 0) + vars(self).update(DEFAULT_MOCK_OPP_FIELDS) + self.opportunity_attachments = [ + type( + "MockAttachment", + (), + { + "download_path": "https://example.com/opportunity", + "file_name": "solicitation.pdf", + "file_description": "Testing Research Solicitation", + "file_size_bytes": 409600, + "mime_type": "application/pdf", + "created_at": datetime(2024, 1, 1, 12, 0, 0), + "updated_at": datetime(2024, 1, 2, 12, 0, 0), + }, + )() + ] self.current_opportunity_summary = type( "MockSummary", (), @@ -338,6 +550,12 @@ def __init__(self): "award_ceiling": 500000, "award_floor": 10000, "additional_info_url": "sam.gov", # URL without protocol + "additional_info_url_description": "SAM.gov Listing", + "agency_contact_description": "Testing contact description", + "agency_email_address": "example@test.gov", + "agency_email_address_description": "Test Example Email", + "fiscal_year": 2024, + "is_cost_sharing": False, "created_at": datetime(2024, 1, 1, 12, 0, 0), "updated_at": datetime(2024, 1, 1, 12, 0, 0), }, @@ -350,6 +568,7 @@ def __init__(self): result = transform_opportunity_to_cg(opportunity) # Check that the URL was not fixed (current implementation returns None for invalid URLs) + assert result is not None assert result.source is None def test_transform_status_to_cg(self): @@ -636,6 +855,7 @@ def test_transform_search_request_from_cg(self): result = transform_search_request_from_cg(filters, sorting, pagination, search_query) + assert result is not None # Check pagination assert result["pagination"]["page_offset"] == 1 assert result["pagination"]["page_size"] == 20 @@ -664,6 +884,7 @@ def test_transform_search_request_from_cg_minimal(self): result = transform_search_request_from_cg(filters, sorting, pagination, None) + assert result is not None # Check pagination assert result["pagination"]["page_offset"] == 1 assert result["pagination"]["page_size"] == 10 @@ -791,3 +1012,142 @@ def test_transformation_with_invalid_url_logs_but_succeeds(self, caplog): and invalid_url in record.message for record in caplog.records ) + + +class TestPopulateCustomFields: + """Tests for the populate_custom_fields function.""" + + BASE_OPP_DATA = { + "legacy_opportunity_id": 99001, + "opportunity_number": "HHS-2024-001", + "category": "Discretionary", + "agency_code": "HHS", + "agency_name": "Dept of Health and Human Services", + "top_level_agency_name": "HHS", + "opportunity_assistance_listings": [ + {"assistance_listing_number": "93.001", "program_title": "Health Research"}, + ], + "opportunity_attachments": [ + { + "download_path": "https://example.com/nofo.pdf", + "file_name": "nofo.pdf", + "file_description": "Notice of Funding Opportunity", + "file_size_bytes": 102400, + "mime_type": "application/pdf", + "created_at": "2024-01-01T12:00:00", + "updated_at": "2024-01-02T12:00:00", + } + ], + "summary": { + "agency_contact_description": "Contact the grants office", + "agency_email_address": "grants@hhs.gov", + "agency_email_address_description": "Grants Office Email", + "additional_info_url": "https://hhs.gov/grants", + "additional_info_url_description": "More info", + "fiscal_year": 2024, + "is_cost_sharing": False, + }, + } + + def test_all_fields_populated(self): + """Test that populate_custom_fields correctly maps all fields when fully populated.""" + result = populate_custom_fields(self.BASE_OPP_DATA) + + assert result is not None + + assert result["legacySerialId"].field_type == CustomFieldType.INTEGER + assert result["legacySerialId"].value == 99001 + + assert result["federalOpportunityNumber"].field_type == CustomFieldType.STRING + assert result["federalOpportunityNumber"].value == "HHS-2024-001" + + assert result["federalFundingSource"].field_type == CustomFieldType.STRING + assert result["federalFundingSource"].value == "Discretionary" + + assert result["agency"].field_type == CustomFieldType.OBJECT + assert result["agency"].value == { + "agencyCode": "HHS", + "agencyName": "Dept of Health and Human Services", + "topLevelAgencyName": "HHS", + } + + assert result["assistanceListings"].field_type == CustomFieldType.ARRAY + assert result["assistanceListings"].value == [ + {"assistanceListingNumber": "93.001", "programTitle": "Health Research"} + ] + + assert result["contactInfo"].field_type == CustomFieldType.OBJECT + assert result["contactInfo"].value == { + "description": "Contact the grants office", + "emailAddress": "grants@hhs.gov", + "emailDescription": "Grants Office Email", + } + + assert result["additionalInfo"].field_type == CustomFieldType.OBJECT + assert result["additionalInfo"].value == { + "url": "https://hhs.gov/grants", + "description": "More info", + } + + assert result["fiscalYear"].field_type == CustomFieldType.NUMBER + assert result["fiscalYear"].value == 2024 + + assert result["costSharing"].field_type == CustomFieldType.BOOLEAN + assert result["costSharing"].value is False + + assert result["attachments"].field_type == CustomFieldType.ARRAY + assert len(result["attachments"].value) == 1 + attachment = result["attachments"].value[0] + assert attachment["downloadUrl"] == "https://example.com/nofo.pdf" + assert attachment["name"] == "nofo.pdf" + assert attachment["description"] == "Notice of Funding Opportunity" + assert attachment["sizeInBytes"] == 102400 + assert attachment["mimeType"] == "application/pdf" + + def test_missing_optional_fields(self): + """Test that None and missing values are handled gracefully and produce valid schema output.""" + opp_data = { + "legacy_opportunity_id": None, + "opportunity_number": None, + "category": None, + "agency_code": None, + "opportunity_assistance_listings": [], + "opportunity_attachments": [], + "summary": None, + } + + result = populate_custom_fields(opp_data) + + # All fields are absent/None so populate_custom_fields should return None + assert result is None + + # An empty custom fields dict should pass Marshmallow schema validation without errors + schema = OpportunityCustomFields() + errors = schema.validate({}) + assert errors == {} + + def test_malformed_data_types(self): + """Test that Marshmallow schema catches malformed data types before they reach API consumers.""" + opp_data = { + **self.BASE_OPP_DATA, + "legacy_opportunity_id": "not-an-integer", + "summary": { + **self.BASE_OPP_DATA["summary"], + "fiscal_year": "not-a-year", + }, + } + + # populate_custom_fields passes values through without type validation + result = populate_custom_fields(opp_data) + assert result is not None + + # Serialize to the format the Marshmallow schema expects and validate + serialized = {k: v.model_dump(by_alias=True) for k, v in result.items()} + schema = OpportunityCustomFields() + errors = schema.validate(serialized) + + # Marshmallow should catch the malformed integer fields + assert "legacySerialId" in errors + assert "value" in errors["legacySerialId"] + assert "fiscalYear" in errors + assert "value" in errors["fiscalYear"]