restore publications

Author: OriHoch

notebooks/restore_publications.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-08-11T06:43:00.200528Z",
+     "start_time": "2025-08-11T06:41:23.376700Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import os\n",
+    "import dataflows as DF\n",
+    "from datapackage_pipelines_migdar.flows.dump_to_es import my_dump_to_es, BoostingMappingGenerator\n",
+    "\n",
+    "# kubectl port-forward to the migdar elasticsearch\n",
+    "os.environ[\"DATAFLOWS_ELASTICSEARCH\"] = \"localhost:9200\"\n",
+    "\n",
+    "# download from the pipelines pod data/ folder\n",
+    "data_source = '../data/publications_in_es/datapackage.json'\n",
+    "\n",
+    "revision = 202538\n",
+    "resource_name = 'publications'\n",
+    "\n",
+    "DF.Flow(\n",
+    "    DF.load(data_source),\n",
+    "    my_dump_to_es(\n",
+    "        indexes={\n",
+    "            'migdar__' + resource_name: [\n",
+    "                {\n",
+    "                    'resource-name': resource_name,\n",
+    "                    'revision': revision\n",
+    "                }\n",
+    "            ]\n",
+    "        },\n",
+    "        mapper_cls=BoostingMappingGenerator,\n",
+    "        index_settings={'index.mapping.coerce': True},\n",
+    "        elasticsearch_options=dict(timeout=60)\n",
+    "    ),\n",
+    ").process()"
+   ],
+   "id": "7f6e9369e2a5b6b7",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(<datapackage.package.Package at 0x77a0152f3f90>,\n",
+       " {'count_of_rows': 4363,\n",
+       "  'bytes': None,\n",
+       "  'hash': '5f5818e16db4ba82b5038b9c42c4ea61',\n",
+       "  'dataset_name': None})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 8
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pipeline-spec.yaml

@@ -24,11 +24,12 @@ zotero_fetch:
   - flow: datapackage_pipelines_migdar.flows.zotero
 
 
-publications:
-  dependencies:
-    - pipeline: ./zotero_fetch
-  pipeline:
-  - flow: datapackage_pipelines_migdar.flows.publications
+# google drive was deleted, we rely on existing data in elasticsearch restored by notebooks/restore_publications.ipynb
+#publications:
+#  dependencies:
+#    - pipeline: ./zotero_fetch
+#  pipeline:
+#  - flow: datapackage_pipelines_migdar.flows.publications
 
 
 sitemap: