Sap branch

ckanext/datajson/blueprint.py

@@ -154,6 +154,8 @@ def make_json(export_type='datajson', owner_org=None):
         json_export_map = get_export_map_json()
 
         if json_export_map:
+            packages = _get_ckan_datasets() or []
+            packages = [pkg for pkg in packages if pkg] 
             for pkg in packages:
                 if json_export_map.get('debug'):
                     output.append(pkg)

ckanext/datajson/export_map/export.spatial.map.sample.json

@@ -1,98 +1,72 @@
 {
-  "validation_enabled": true,
+  "validation_enabled": false,
   "catalog_headers": {
-    "conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
-    "describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
-    "@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
     "@type": "dcat:Catalog"
   },
   "dataset_fields_map": {
+    "identifier": {
+      "field": "agency_dataset_id"
+    },
     "title": {
       "field": "title"
     },
     "description": {
-      "field": "notes"
-    },
-    "modified": {
-      "extra": true,
-      "field": "Modified"
+      "field": "description/abstract"
     },
-    "accessLevel": {
-      "extra": true,
-      "field": "Public Access Level"
-    },
-    "identifier": {
+    "publisher": {
       "extra": true,
-      "field": "Unique Id"
+      "field": "source(s)"
     },
-    "dataQuality": {
+    "keyword": {
+      "type": "array",
       "extra": true,
-      "field": "Data Quality"
+      "field": "classification",
+      "split": ","
     },
-    "conformsTo": {
+    "accessLevel": {
       "extra": true,
-      "field": "Conforms To"
+      "field": "trust_level"
     },
-    "describedBy": {
+    "spatial": {
       "extra": true,
-      "field": "Data Dictionary"
+      "field": "spatial_coverage"
     },
-    "describedByType": {
+    "temporal": {
       "extra": true,
-      "field": "Data Dictionary Type"
+      "field": "years_available"
     },
     "issued": {
       "extra": true,
-      "field": "Release Date"
+      "field": "program_title"
     },
-    "landingPage": {
+    "accrualPeriodicity": {
       "extra": true,
-      "field": "Homepage Url"
+      "field": "frequency_of_data_collection"
     },
     "license": {
       "extra": true,
-      "field": "License New"
+      "field": "fees"
     },
-    "primaryITInvestmentUII": {
-      "extra": true,
-      "field": "Primary It Investment Uii"
-    },
-    "rights": {
-      "extra": true,
-      "field": "Access Level Comment"
-    },
-    "systemOfRecords": {
-      "extra": true,
-      "field": "System Of Records"
-    },
-    "spatial": {
-      "extra": true,
-      "field": "Spatial"
-    },
-    "temporal": {
-      "extra": true,
-      "field": "Temporal"
-    },
-    "publisher": {
+    "landingPage": {
       "extra": true,
-      "field": "Responsible Party",
-      "wrapper": "catalog_publisher"
+      "field": "where_to_apply"
     },
-    "accrualPeriodicity": {
+    "language": {
+      "type": "array",
       "extra": true,
-      "field": "Frequency Of Update",
-      "wrapper": "fix_accrual_periodicity"
+      "field": "language",
+      "split": ","
     },
     "contactPoint": {
       "wrapper": "build_contact_point",
       "map": {
         "fn": {
           "extra": true,
-          "field": "Responsible Party"
+          "field": "program"
         },
         "hasEmail": {
           "extra": true,
-          "field": "Contact Email"
+          "field": "url_for_more_info_about_restricted_file"
         }
       }
     },
@@ -103,63 +77,179 @@
           "field": "url"
         },
         "mediaType": {
-          "field": "mimetype"
-        },
-        "format": {
           "field": "format"
-        },
-        "title": {
-          "field": "name"
-        },
-        "description": {
-          "field": "description"
-        },
-        "conformsTo": {
-          "field": "conformsTo"
-        },
-        "describedBy": {
-          "field": "describedBy"
-        },
-        "describedByType": {
-          "field": "describedByType"
         }
       }
     },
-    "keyword": {
+    "dataQuality": {
+      "extra": true,
+      "field": "method_of_data_collection"
+    },
+    "theme": {
       "type": "array",
-      "array_key": "display_name",
-      "field": "tags"
+      "extra": true,
+      "field": "classification",
+      "split": ","
+    },
+    "rights": {
+      "extra": true,
+      "field": "usage_restrictions"
+    },
+    "conformsTo": {
+      "extra": true,
+      "field": "documentation_for_application"
+    },
+    "describedBy": {
+      "extra": true,
+      "field": "supporting_documentation"
     },
-    "bureauCode": {
+    "name": {
+      "field": "name"
+    },
+    "owner_org": {
+      "field": "owner_org"
+    },
+    "agency_id": {
+      "extra": true,
+      "field": "agency_id"
+    },
+    "agency_datafile_id": {
+      "extra": true,
+      "field": "agency_datafile_id"
+    },
+    "alternative_title": {
+      "extra": true,
+      "field": "alternative_title"
+    },
+    "authorizer(s)": {
       "type": "array",
       "extra": true,
-      "field": "Bureau Code",
-      "split": ",",
-      "wrapper": "bureau_code"
+      "field": "authorizer(s)",
+      "split": ","
     },
-    "programCode": {
+    "funder/sponsor(s)": {
       "type": "array",
       "extra": true,
-      "field": "Program Code",
+      "field": "funder/sponsor(s)",
       "split": ","
     },
-    "language": {
+    "program": {
+      "extra": true,
+      "field": "program"
+    },
+    "preferred_citation": {
+      "extra": true,
+      "field": "preferred_citation"
+    },
+    "url_for_general_survey_info": {
+      "extra": true,
+      "field": "url_for_general_survey_info"
+    },
+    "doi": {
+      "extra": true,
+      "field": "doi"
+    },
+    "universe": {
+      "extra": true,
+      "field": "universe"
+    },
+    "unit_of_observation": {
+      "extra": true,
+      "field": "unit_of_observation"
+    },
+    "smallest_geographic_unit": {
+      "extra": true,
+      "field": "smallest_geographic_unit"
+    },
+    "sample": {
+    "extra": true,
+      "field": "sample"
+    },
+    "reference_date": {
+      "extra": true,
+      "field": "reference_date"
+    },
+    "data_collection_notes": {
+      "extra": true,
+      "field": "data_collection_notes"
+    },
+    "number_of_cases": {
+      "extra": true,
+      "field": "number_of_cases"
+    },
+    "number_of_variables": {
+      "extra": true,
+      "field": "number_of_variables"
+    },
+    "linkage_capabilities": {
+      "extra": true,
+      "field": "linkage_capabilities"
+    },
+    "linkage_variables": {
       "type": "array",
       "extra": true,
-      "field": "Language",
+      "field": "linkage_variables",
       "split": ","
     },
-    "references": {
+    "public-use_version": {
+      "extra": true,
+      "field": "public-use_version"
+    },
+    "summary_of_differences": {
+      "extra": true,
+      "field": "summary_of_differences"
+    },
+    "geography_differs_from_restricted-use_file_(ruf)": {
+      "extra": true,
+      "field": "geography_differs_from_restricted-use_file_(ruf)"
+    },
+    "variable_detail_differs_from_ruf": {
+      "extra": true,
+      "field": "variable_detail_differs_from_ruf"
+    },
+    "public-use_file_selectable": {
+      "extra": true,
+      "field": "public-use_file_selectable"
+    },
+    "fti_flag": {
+      "extra": true,
+      "field": "fti_flag"
+    },
+    "provisioned_by_state": {
       "type": "array",
       "extra": true,
-      "field": "Related Documents",
+      "field": "provisioned_by_state",
       "split": ","
     },
-    "theme": {
+    "provisioned_by_other_geographic_unit": {
+      "extra": true,
+      "field": "provisioned_by_other_geographic_unit"
+    },
+    "provisioned_by_month": {
       "type": "array",
       "extra": true,
-      "field": "Category",
+      "field": "provisioned_by_month",
       "split": ","
+    },
+    "provisioned_by_year": {
+      "extra": true,
+      "field": "provisioned_by_year"
+    },
+    "provisioned_by_other_time_unit": {
+      "extra": true,
+      "field": "provisioned_by_other_time_unit"
+    },
+    "provisioning_units_other_than_time_or_geography": {
+      "extra": true,
+      "field": "provisioning_units_other_than_time_or_geography"
+    },
+    "can_non-citizens_apply": {
+      "extra": true,
+      "field": "can_non-citizens_apply"
+    },
+    "variable_selection_requirement": {
+      "extra": true,
+      "field": "variable_selection_requirement"     
     }
   }
-}
+}

ckanext/datajson/helpers.py

@@ -214,7 +214,7 @@ def store(self, package):
         try:
             self.pid = package.get('id')
 
-            current_extras = package.get('extras')
+            current_extras = package.get('extras') or []
             new_extras = {}
             for extra in current_extras:
                 if 'extras_rollup' == extra.get('key'):

ckanext/datajson/package2pod.py

@@ -57,8 +57,10 @@ def convert_package(package, json_export_map, redaction_enabled=False):
         import sys
 
         try:
+            if package is None or json_export_map is None:
+                log.error("Invalid package or export map provided.")
+                return None  
             dataset = Package2Pod.export_map_fields(package, json_export_map, redaction_enabled)
-
             # skip validation if we export whole /data.json catalog
             if json_export_map.get('validation_enabled'):
                 return Package2Pod.validate(package, dataset)