Skip to content

Commit 47fcff6

Browse files
mbana641mbana641
committed
Update create_and_load_knowledge_graph_backups.ipynb
1 parent 543de58 commit 47fcff6

File tree

1 file changed

+159
-57
lines changed

1 file changed

+159
-57
lines changed

‎samples/05_content_publishers/create_and_load_knowledge_graph_backups.ipynb

Lines changed: 159 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
"Create a backup of an ArcGIS Knowledge graph you have created to store your data model and data. The backup will be in the format of json files that can read to recreate your graph or be shared for others to be able to create the same graph on a different server.\n",
5555
"\n",
5656
"_**Load a new graph easily from your backup files!**_\n",
57+
"\n",
5758
"Use the backup files with the load backup files portion of this notebook to create a graph with the same data model and data."
5859
]
5960
},
@@ -82,7 +83,7 @@
8283
"outputs": [],
8384
"source": [
8485
"# imports\n",
85-
"import os, json\n",
86+
"import os, json, requests\n",
8687
"from datetime import datetime\n",
8788
"from uuid import UUID\n",
8889
"\n",
@@ -107,14 +108,16 @@
107108
"outputs": [],
108109
"source": [
109110
"# output folder name\n",
110-
"output_folder = \"C:\\backups\\myknowledgegraph_backup\"\n",
111+
"output_folder = r\"C:\\backups\\myknowledgegraph_backup\"\n",
111112
"\n",
112113
"# output backup json file names\n",
113114
"dm_ent = \"datamodel_entities.json\"\n",
114115
"dm_rel = \"datamodel_relationships.json\"\n",
116+
"dm_prov = \"datamodel_provenance.json\"\n",
115117
"all_ent = \"all_entities.json\"\n",
116118
"all_rel = \"all_relationships.json\"\n",
117-
"prov_file = \"provenance_entities.json\" # this will only be used if you want to backup provenance records"
119+
"prov_file = \"provenance_entities.json\" # this will only be used if you want to backup provenance records\n",
120+
"servicedef = 'service_definition.json'"
118121
]
119122
},
120123
{
@@ -153,6 +156,35 @@
153156
" raise Exception(\"Knowledge graph to backup does not exist\")"
154157
]
155158
},
159+
{
160+
"cell_type": "markdown",
161+
"id": "423cbfe2",
162+
"metadata": {},
163+
"source": [
164+
"Get the service definition for the knowledge graph service and save it to the service definition file"
165+
]
166+
},
167+
{
168+
"cell_type": "code",
169+
"execution_count": null,
170+
"id": "b24f732f",
171+
"metadata": {},
172+
"outputs": [],
173+
"source": [
174+
"# create a token for request\n",
175+
"token_url = f\"https://myportal.com/portal/sharing/rest/generateToken\"\n",
176+
"creds = {\n",
177+
" \"username\": \"myUsername\",\n",
178+
" \"password\": \"myPassword\",\n",
179+
" \"referer\": \"https://myportal.com/portal\",\n",
180+
" \"f\": \"json\"\n",
181+
"}\n",
182+
"token_response = requests.post(token_url, data=creds, verify=False)\n",
183+
"sd_data = requests.get(url=kg_url, params={'f': 'json', 'token': token_response.json()['token']}, verify=False)\n",
184+
"with open(os.path.join(output_folder, servicedef), 'w') as f:\n",
185+
" json.dump(sd_data.text, f)"
186+
]
187+
},
156188
{
157189
"cell_type": "markdown",
158190
"id": "ced3a37e",
@@ -362,6 +394,19 @@
362394
"If you have provenance records that you want to maintain in the backups, this will get all provenance records and save them to a json backup file"
363395
]
364396
},
397+
{
398+
"cell_type": "code",
399+
"execution_count": null,
400+
"id": "d4db5b47",
401+
"metadata": {},
402+
"outputs": [],
403+
"source": [
404+
"# write provenance information to the provenance data model file\n",
405+
"prov_structure = knowledgegraph_backup.datamodel['meta_entity_types']['Provenance']\n",
406+
"with open(os.path.join(output_folder, dm_prov), 'w') as f:\n",
407+
" json.dump(prov_structure, f)"
408+
]
409+
},
365410
{
366411
"cell_type": "code",
367412
"execution_count": null,
@@ -420,61 +465,18 @@
420465
"## Load backup files"
421466
]
422467
},
423-
{
424-
"cell_type": "markdown",
425-
"id": "db4a59d8",
426-
"metadata": {},
427-
"source": [
428-
"### Connect to portal and create knowledge graph\n",
429-
"Connect to the portal and create a new knowledge graph service to load data model and data into"
430-
]
431-
},
432468
{
433469
"cell_type": "code",
434470
"execution_count": null,
435-
"id": "dd042d14",
436-
"metadata": {},
437-
"outputs": [],
438-
"source": [
439-
"# connect to portal via GIS\n",
440-
"gis_load = GIS(\n",
441-
" \"https://myportal.com/portal\", \"username\", \"password\"\n",
442-
")\n",
443-
"# create a knowledge graph without provenance enabled\n",
444-
"result = gis_load.content.create_service(\n",
445-
" name=\"myknowledgegraph\",\n",
446-
" capabilities=\"Query,Editing,Create,Update,Delete\",\n",
447-
" service_type=\"KnowledgeGraph\",\n",
448-
")\n",
449-
"# OPTIONAL: switch line above with line below to create knowledge graph WITH provenance enabled\n",
450-
"# result = gis_load.content.create_service(name=\"\", service_type=\"KnowledgeGraph\",create_params={\"name\": \"myknowledgegraph\", \"capabilities\": \"Query\", \"jsonProperties\": {\"supportsProvenance\": True}})\n",
451-
"knowledgegraph_load = KnowledgeGraph(result.url, gis=gis_load)"
452-
]
453-
},
454-
{
455-
"cell_type": "markdown",
456-
"id": "70b11b8a",
457-
"metadata": {},
458-
"source": [
459-
"### Populate data model from saved json files\n",
460-
"Populate entity and relationship types from saved json files. This is using the variables defined in the 'Setup' section above."
461-
]
462-
},
463-
{
464-
"cell_type": "code",
465-
"execution_count": null,
466-
"id": "ad6f6d63",
471+
"id": "0b62742e",
467472
"metadata": {},
468473
"outputs": [],
469474
"source": [
470475
"# load data model json files into graph data model\n",
471476
"with open(os.path.join(output_folder, dm_ent), \"r\") as file:\n",
472477
" dm_ents = json.load(file)\n",
473478
"with open(os.path.join(output_folder, dm_rel), \"r\") as file:\n",
474-
" dm_rels = json.load(file)\n",
475-
"knowledgegraph_load.named_object_type_adds(\n",
476-
" entity_types=dm_ents, relationship_types=dm_rels\n",
477-
")"
479+
" dm_rels = json.load(file)"
478480
]
479481
},
480482
{
@@ -519,6 +521,77 @@
519521
" doc_rel_type_name = relationship_type[\"name\"]"
520522
]
521523
},
524+
{
525+
"cell_type": "markdown",
526+
"id": "db4a59d8",
527+
"metadata": {},
528+
"source": [
529+
"### Connect to portal and create knowledge graph\n",
530+
"Connect to the portal and create a new knowledge graph service to load data model and data into"
531+
]
532+
},
533+
{
534+
"cell_type": "code",
535+
"execution_count": null,
536+
"id": "dd042d14",
537+
"metadata": {},
538+
"outputs": [],
539+
"source": [
540+
"# connect to portal via GIS\n",
541+
"gis_load = GIS(\n",
542+
" \"https://myportal.com/portal\", \"username\", \"password\"\n",
543+
")\n",
544+
"\n",
545+
"with open(os.path.join(output_folder, servicedef), 'r') as file:\n",
546+
" service_def = json.load(file)\n",
547+
"service_def = json.loads(service_def)\n",
548+
"updated_service_def = {\n",
549+
" 'name': \"myNewKnowledgeGraph\", #replace with the name for your new knowledge graph\n",
550+
" 'capabilities': service_def['capabilities'],\n",
551+
" 'jsonProperties': {\n",
552+
" 'allowGeometryUpdates': service_def['allowGeometryUpdates'],\n",
553+
" 'searchMaxRecordCount': service_def['searchMaxRecordCount'],\n",
554+
" 'spatialReference': service_def['spatialReference'],\n",
555+
" 'maxRecordCount': service_def['maxRecordCount'],\n",
556+
" 'description': service_def['description'],\n",
557+
" 'copyrightText': service_def['copyrightText'],\n",
558+
" 'documentEntityTypeInfo': {\n",
559+
" \"documentEntityTypeName\": doc_type_name,\n",
560+
" \"hasDocumentsRelationshipTypeName\": doc_rel_type_name\n",
561+
" },\n",
562+
" 'supportsDocuments': service_def['supportsDocuments'],\n",
563+
" 'supportsSearch': service_def['supportsSearch'],\n",
564+
" 'supportsProvenance': service_def['supportsProvenance']\n",
565+
" }\n",
566+
"}\n",
567+
"\n",
568+
"result = gis_copyto.content.create_service(name=\"\",service_type=\"KnowledgeGraph\",create_params=updated_service_def)\n",
569+
"\n",
570+
"knowledgegraph_load = KnowledgeGraph(result.url, gis=gis_load)"
571+
]
572+
},
573+
{
574+
"cell_type": "markdown",
575+
"id": "70b11b8a",
576+
"metadata": {},
577+
"source": [
578+
"### Populate data model from saved json files\n",
579+
"Populate entity and relationship types from saved json files. This is using the variables defined in the 'Setup' section above."
580+
]
581+
},
582+
{
583+
"cell_type": "code",
584+
"execution_count": null,
585+
"id": "ad6f6d63",
586+
"metadata": {},
587+
"outputs": [],
588+
"source": [
589+
"# populate entity and relationship types based on data model (errors about Document/HasDocument in the output are expected)\n",
590+
"knowledgegraph_load.named_object_type_adds(\n",
591+
" entity_types=dm_ents, relationship_types=dm_rels\n",
592+
")"
593+
]
594+
},
522595
{
523596
"cell_type": "markdown",
524597
"id": "fd1eecc7",
@@ -535,7 +608,7 @@
535608
"metadata": {},
536609
"outputs": [],
537610
"source": [
538-
"# load any additional document entity type properties\n",
611+
"# load any additional document entity type properties (errors about properties that already exist are expected)\n",
539612
"origin_document_properties = None\n",
540613
"for entity_type in dm_ents:\n",
541614
" if entity_type[\"name\"] == doc_type_name:\n",
@@ -544,7 +617,7 @@
544617
"for prop in origin_document_properties:\n",
545618
" prop_list.append(origin_document_properties[prop])\n",
546619
"knowledgegraph_load.graph_property_adds(\n",
547-
" type_name=\"Document\", graph_properties=prop_list\n",
620+
" type_name=doc_type_name, graph_properties=prop_list\n",
548621
")"
549622
]
550623
},
@@ -555,15 +628,15 @@
555628
"metadata": {},
556629
"outputs": [],
557630
"source": [
558-
"# load any additional document relationship type properties\n",
631+
"# load any additional document relationship type properties (errors about properties that already exist are expected)\n",
559632
"for relationship_type in dm_rels:\n",
560633
" if relationship_type[\"name\"] == doc_rel_type_name:\n",
561634
" origin_document_rel_properties = relationship_type[\"properties\"]\n",
562635
"prop_list = []\n",
563636
"for prop in origin_document_rel_properties:\n",
564637
" prop_list.append(origin_document_rel_properties[prop])\n",
565638
"knowledgegraph_load.graph_property_adds(\n",
566-
" type_name=\"HasDocument\", graph_properties=prop_list\n",
639+
" type_name=doc_rel_type_name, graph_properties=prop_list\n",
567640
")"
568641
]
569642
},
@@ -766,16 +839,16 @@
766839
"outputs": [],
767840
"source": [
768841
"# add search indexes for all relationship text properties\n",
769-
"for entity_type in load_dm[\"relationship_types\"]:\n",
842+
"for relationship_type in load_dm[\"relationship_types\"]:\n",
770843
" prop_list = []\n",
771-
" for prop in load_dm[\"relationship_types\"][entity_type][\"properties\"]:\n",
844+
" for prop in load_dm[\"relationship_types\"][relationship_type][\"properties\"]:\n",
772845
" if (\n",
773-
" load_dm[\"relationship_types\"][entity_type][\"properties\"][prop][\"fieldType\"]\n",
846+
" load_dm[\"relationship_types\"][relationship_type][\"properties\"][prop][\"fieldType\"]\n",
774847
" == \"esriFieldTypeString\"\n",
775848
" ):\n",
776849
" prop_list.append(prop)\n",
777850
" knowledgegraph_load.update_search_index(\n",
778-
" adds={entity_type: {\"property_names\": prop_list}}\n",
851+
" adds={relationship_type: {\"property_names\": prop_list}}\n",
779852
" )"
780853
]
781854
},
@@ -788,6 +861,35 @@
788861
"This will only apply if you created a backup of provenance records and have enabled provenance on your knowledge graph service."
789862
]
790863
},
864+
{
865+
"cell_type": "markdown",
866+
"id": "9bce8d5b",
867+
"metadata": {},
868+
"source": [
869+
"### Add additional provenance entity type properties\n",
870+
"In the case additional properties exist in the original graph on the provenance type, we need to add those properties to the data model using graph_property_adds."
871+
]
872+
},
873+
{
874+
"cell_type": "code",
875+
"execution_count": null,
876+
"id": "e3563b59",
877+
"metadata": {},
878+
"outputs": [],
879+
"source": [
880+
"with open(os.path.join(file_loc, dm_prov), 'r') as file:\n",
881+
" prov_dm = json.load(file)\n",
882+
"\n",
883+
"prop_list = []\n",
884+
"for prop in prov_dm['properties']:\n",
885+
" prop_list.append(prov_dm['properties'][prop])\n",
886+
"knowledgegraph_load.graph_property_adds(\n",
887+
" type_name=\"Provenance\",\n",
888+
" graph_properties=prop_list\n",
889+
")\n",
890+
"# errors about already existing properties are expected"
891+
]
892+
},
791893
{
792894
"cell_type": "code",
793895
"execution_count": null,
@@ -838,7 +940,7 @@
838940
"name": "python",
839941
"nbconvert_exporter": "python",
840942
"pygments_lexer": "ipython3",
841-
"version": "3.9.17"
943+
"version": "3.9.16"
842944
}
843945
},
844946
"nbformat": 4,

0 commit comments

Comments
 (0)