add upsert step

Author: junefish

learn/experimental/merge-namespaces/merge-namespaces.ipynb

@@ -78,6 +78,56 @@
     "vectors_ns1 = fetch_all_vectors(\"ns1\")\n",
     "vectors_ns2 = fetch_all_vectors(\"ns2\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Convert fetched vectors to the required upsert format\n",
+    "def format_vectors_for_upsert(fetched_vectors):\n",
+    "    return [{\"id\": match['id'], \"values\": match['values']} for match in fetched_vectors]\n",
+    "\n",
+    "formatted_vectors_ns1 = format_vectors_for_upsert(vectors_ns1)\n",
+    "formatted_vectors_ns2 = format_vectors_for_upsert(vectors_ns2)\n",
+    "\n",
+    "print(f\"Preparing to upsert {len(formatted_vectors_ns1)} vectors from ns1 and \\\n",
+    "{len(formatted_vectors_ns2)} vectors from ns2\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that any vectors with overlapping IDs between `ns1` and `ns2` will be overwritten by the `ns2` upsert."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from itertools import islice\n",
+    "\n",
+    "# Upsert vectors in batches of 100\n",
+    "def chunks(data, size=100):\n",
+    "    it = iter(data)\n",
+    "    for chunk in iter(lambda: tuple(islice(it, size)), ()):\n",
+    "        yield chunk\n",
+    "\n",
+    "# Upsert vectors into the merged namespace\n",
+    "target_namespace = 'merged'\n",
+    "for batch in chunks(formatted_vectors_ns1):\n",
+    "    index.upsert(vectors=batch, namespace=target_namespace)\n",
+    "\n",
+    "for batch in chunks(formatted_vectors_ns2):\n",
+    "    index.upsert(vectors=batch, namespace=target_namespace)\n",
+    "\n",
+    "print(f\"Upserted {len(formatted_vectors_ns1)} vectors from ns1 and \\\n",
+    "{len(formatted_vectors_ns2)} vectors from ns2 into {target_namespace}\")"
+   ]
   }
  ],
  "metadata": {