diff --git a/ckanext/datajson/blueprint.py b/ckanext/datajson/blueprint.py index a29fff99..cd55c301 100644 --- a/ckanext/datajson/blueprint.py +++ b/ckanext/datajson/blueprint.py @@ -154,6 +154,8 @@ def make_json(export_type='datajson', owner_org=None): json_export_map = get_export_map_json() if json_export_map: + packages = _get_ckan_datasets() or [] + packages = [pkg for pkg in packages if pkg] for pkg in packages: if json_export_map.get('debug'): output.append(pkg) diff --git a/ckanext/datajson/export_map/export.spatial.map.sample.json b/ckanext/datajson/export_map/export.spatial.map.sample.json index b67f553b..78415901 100644 --- a/ckanext/datajson/export_map/export.spatial.map.sample.json +++ b/ckanext/datajson/export_map/export.spatial.map.sample.json @@ -1,98 +1,72 @@ { - "validation_enabled": true, + "validation_enabled": false, "catalog_headers": { - "conformsTo": "https://project-open-data.cio.gov/v1.1/schema", - "describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json", - "@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld", "@type": "dcat:Catalog" }, "dataset_fields_map": { + "identifier": { + "field": "agency_dataset_id" + }, "title": { "field": "title" }, "description": { - "field": "notes" - }, - "modified": { - "extra": true, - "field": "Modified" + "field": "description/abstract" }, - "accessLevel": { - "extra": true, - "field": "Public Access Level" - }, - "identifier": { + "publisher": { "extra": true, - "field": "Unique Id" + "field": "source(s)" }, - "dataQuality": { + "keyword": { + "type": "array", "extra": true, - "field": "Data Quality" + "field": "classification", + "split": "," }, - "conformsTo": { + "accessLevel": { "extra": true, - "field": "Conforms To" + "field": "trust_level" }, - "describedBy": { + "spatial": { "extra": true, - "field": "Data Dictionary" + "field": "spatial_coverage" }, - "describedByType": { + "temporal": { "extra": true, - "field": "Data Dictionary Type" + "field": "years_available" }, "issued": { "extra": true, - "field": "Release Date" + "field": "program_title" }, - "landingPage": { + "accrualPeriodicity": { "extra": true, - "field": "Homepage Url" + "field": "frequency_of_data_collection" }, "license": { "extra": true, - "field": "License New" + "field": "fees" }, - "primaryITInvestmentUII": { - "extra": true, - "field": "Primary It Investment Uii" - }, - "rights": { - "extra": true, - "field": "Access Level Comment" - }, - "systemOfRecords": { - "extra": true, - "field": "System Of Records" - }, - "spatial": { - "extra": true, - "field": "Spatial" - }, - "temporal": { - "extra": true, - "field": "Temporal" - }, - "publisher": { + "landingPage": { "extra": true, - "field": "Responsible Party", - "wrapper": "catalog_publisher" + "field": "where_to_apply" }, - "accrualPeriodicity": { + "language": { + "type": "array", "extra": true, - "field": "Frequency Of Update", - "wrapper": "fix_accrual_periodicity" + "field": "language", + "split": "," }, "contactPoint": { "wrapper": "build_contact_point", "map": { "fn": { "extra": true, - "field": "Responsible Party" + "field": "program" }, "hasEmail": { "extra": true, - "field": "Contact Email" + "field": "url_for_more_info_about_restricted_file" } } }, @@ -103,63 +77,179 @@ "field": "url" }, "mediaType": { - "field": "mimetype" - }, - "format": { "field": "format" - }, - "title": { - "field": "name" - }, - "description": { - "field": "description" - }, - "conformsTo": { - "field": "conformsTo" - }, - "describedBy": { - "field": "describedBy" - }, - "describedByType": { - "field": "describedByType" } } }, - "keyword": { + "dataQuality": { + "extra": true, + "field": "method_of_data_collection" + }, + "theme": { "type": "array", - "array_key": "display_name", - "field": "tags" + "extra": true, + "field": "classification", + "split": "," + }, + "rights": { + "extra": true, + "field": "usage_restrictions" + }, + "conformsTo": { + "extra": true, + "field": "documentation_for_application" + }, + "describedBy": { + "extra": true, + "field": "supporting_documentation" }, - "bureauCode": { + "name": { + "field": "name" + }, + "owner_org": { + "field": "owner_org" + }, + "agency_id": { + "extra": true, + "field": "agency_id" + }, + "agency_datafile_id": { + "extra": true, + "field": "agency_datafile_id" + }, + "alternative_title": { + "extra": true, + "field": "alternative_title" + }, + "authorizer(s)": { "type": "array", "extra": true, - "field": "Bureau Code", - "split": ",", - "wrapper": "bureau_code" + "field": "authorizer(s)", + "split": "," }, - "programCode": { + "funder/sponsor(s)": { "type": "array", "extra": true, - "field": "Program Code", + "field": "funder/sponsor(s)", "split": "," }, - "language": { + "program": { + "extra": true, + "field": "program" + }, + "preferred_citation": { + "extra": true, + "field": "preferred_citation" + }, + "url_for_general_survey_info": { + "extra": true, + "field": "url_for_general_survey_info" + }, + "doi": { + "extra": true, + "field": "doi" + }, + "universe": { + "extra": true, + "field": "universe" + }, + "unit_of_observation": { + "extra": true, + "field": "unit_of_observation" + }, + "smallest_geographic_unit": { + "extra": true, + "field": "smallest_geographic_unit" + }, + "sample": { + "extra": true, + "field": "sample" + }, + "reference_date": { + "extra": true, + "field": "reference_date" + }, + "data_collection_notes": { + "extra": true, + "field": "data_collection_notes" + }, + "number_of_cases": { + "extra": true, + "field": "number_of_cases" + }, + "number_of_variables": { + "extra": true, + "field": "number_of_variables" + }, + "linkage_capabilities": { + "extra": true, + "field": "linkage_capabilities" + }, + "linkage_variables": { "type": "array", "extra": true, - "field": "Language", + "field": "linkage_variables", "split": "," }, - "references": { + "public-use_version": { + "extra": true, + "field": "public-use_version" + }, + "summary_of_differences": { + "extra": true, + "field": "summary_of_differences" + }, + "geography_differs_from_restricted-use_file_(ruf)": { + "extra": true, + "field": "geography_differs_from_restricted-use_file_(ruf)" + }, + "variable_detail_differs_from_ruf": { + "extra": true, + "field": "variable_detail_differs_from_ruf" + }, + "public-use_file_selectable": { + "extra": true, + "field": "public-use_file_selectable" + }, + "fti_flag": { + "extra": true, + "field": "fti_flag" + }, + "provisioned_by_state": { "type": "array", "extra": true, - "field": "Related Documents", + "field": "provisioned_by_state", "split": "," }, - "theme": { + "provisioned_by_other_geographic_unit": { + "extra": true, + "field": "provisioned_by_other_geographic_unit" + }, + "provisioned_by_month": { "type": "array", "extra": true, - "field": "Category", + "field": "provisioned_by_month", "split": "," + }, + "provisioned_by_year": { + "extra": true, + "field": "provisioned_by_year" + }, + "provisioned_by_other_time_unit": { + "extra": true, + "field": "provisioned_by_other_time_unit" + }, + "provisioning_units_other_than_time_or_geography": { + "extra": true, + "field": "provisioning_units_other_than_time_or_geography" + }, + "can_non-citizens_apply": { + "extra": true, + "field": "can_non-citizens_apply" + }, + "variable_selection_requirement": { + "extra": true, + "field": "variable_selection_requirement" } } -} \ No newline at end of file +} diff --git a/ckanext/datajson/helpers.py b/ckanext/datajson/helpers.py index 709854c2..21781f26 100644 --- a/ckanext/datajson/helpers.py +++ b/ckanext/datajson/helpers.py @@ -214,7 +214,7 @@ def store(self, package): try: self.pid = package.get('id') - current_extras = package.get('extras') + current_extras = package.get('extras') or [] new_extras = {} for extra in current_extras: if 'extras_rollup' == extra.get('key'): diff --git a/ckanext/datajson/package2pod.py b/ckanext/datajson/package2pod.py index 17588e5f..58eb3c4c 100644 --- a/ckanext/datajson/package2pod.py +++ b/ckanext/datajson/package2pod.py @@ -57,8 +57,10 @@ def convert_package(package, json_export_map, redaction_enabled=False): import sys try: + if package is None or json_export_map is None: + log.error("Invalid package or export map provided.") + return None dataset = Package2Pod.export_map_fields(package, json_export_map, redaction_enabled) - # skip validation if we export whole /data.json catalog if json_export_map.get('validation_enabled'): return Package2Pod.validate(package, dataset)