docs: Update tutorials for DataJoint 2.0 API and add how-to guide

- Update dj-top.ipynb and json.ipynb for 2.0 API changes:
  - Replace fetch("KEY") with keys()
  - Replace fetch(as_dict=True) with to_dicts()
  - Replace fetch(column) with to_arrays(column)
- Add schema cleanup at start of tutorials for re-runnability
- Add datajoint.json and .secrets/ to .gitignore for local config
- Add deferred schema activation how-to guide
- Update mkdocs.yaml with How-To Guides section

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: dimitri-yatsenko

.gitignore

@@ -183,6 +183,8 @@ cython_debug/
 
 # Datajoint related files
 dj_local_conf.json
+datajoint.json
+.secrets/
 *.env
 !.vscode/launch.json
 # pixi environments

docs/mkdocs.yaml

@@ -7,6 +7,8 @@ repo_name: datajoint/datajoint-python
 nav:
   - Home: index.md
   - Contributing: develop.md
+  - How-To Guides:
+    - Deferred Schema Activation: how-to/deferred-schema-activation.md
   - Architecture:
     - architecture/index.md
     - SQL Transpilation: architecture/transpilation.md

docs/src/archive/tutorials/dj-top.ipynb

@@ -18,24 +18,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[2024-12-20 11:10:20,120][INFO]: Connecting [email protected]:3306\n",
-      "[2024-12-20 11:10:20,259][INFO]: Connected [email protected]:3306\n"
-     ]
-    }
-   ],
-   "source": [
-    "import datajoint as dj\n",
-    "\n",
-    "dj.config[\"database.host\"] = \"127.0.0.1\"\n",
-    "schema = dj.Schema(\"university\")"
-   ]
+   "outputs": [],
+   "source": "import datajoint as dj\n\n# Clean up any existing schema from previous runs\nschema = dj.Schema(\"tutorial_university\", create_tables=False)\nschema.drop()\n\n# Create fresh schema\nschema = dj.Schema(\"tutorial_university\")"
   },
   {
    "cell_type": "markdown",
@@ -236,144 +222,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "Student.insert(dict(k, student_id=i) for i, k in zip(range(100, 300), yield_students()))\n",
-    "\n",
-    "Department.insert(\n",
-    "    dict(\n",
-    "        dept=dept,\n",
-    "        dept_name=name,\n",
-    "        dept_address=fake.address(),\n",
-    "        dept_phone=fake.phone_number()[:20],\n",
-    "    )\n",
-    "    for dept, name in [\n",
-    "        [\"CS\", \"Computer Science\"],\n",
-    "        [\"BIOL\", \"Life Sciences\"],\n",
-    "        [\"PHYS\", \"Physics\"],\n",
-    "        [\"MATH\", \"Mathematics\"],\n",
-    "    ]\n",
-    ")\n",
-    "\n",
-    "StudentMajor.insert(\n",
-    "    {**s, **d, \"declare_date\": fake.date_between(start_date=datetime.date(1999, 1, 1))}\n",
-    "    for s, d in zip(Student.fetch(\"KEY\"), random.choices(Department.fetch(\"KEY\"), k=len(Student())))\n",
-    "    if random.random() < 0.75\n",
-    ")\n",
-    "\n",
-    "# from https://www.utah.edu/\n",
-    "Course.insert(\n",
-    "    [\n",
-    "        [\"BIOL\", 1006, \"World of Dinosaurs\", 3],\n",
-    "        [\"BIOL\", 1010, \"Biology in the 21st Century\", 3],\n",
-    "        [\"BIOL\", 1030, \"Human Biology\", 3],\n",
-    "        [\"BIOL\", 1210, \"Principles of Biology\", 4],\n",
-    "        [\"BIOL\", 2010, \"Evolution & Diversity of Life\", 3],\n",
-    "        [\"BIOL\", 2020, \"Principles of Cell Biology\", 3],\n",
-    "        [\"BIOL\", 2021, \"Principles of Cell Science\", 4],\n",
-    "        [\"BIOL\", 2030, \"Principles of Genetics\", 3],\n",
-    "        [\"BIOL\", 2210, \"Human Genetics\", 3],\n",
-    "        [\"BIOL\", 2325, \"Human Anatomy\", 4],\n",
-    "        [\"BIOL\", 2330, \"Plants & Society\", 3],\n",
-    "        [\"BIOL\", 2355, \"Field Botany\", 2],\n",
-    "        [\"BIOL\", 2420, \"Human Physiology\", 4],\n",
-    "        [\"PHYS\", 2040, \"Classcal Theoretical Physics II\", 4],\n",
-    "        [\"PHYS\", 2060, \"Quantum Mechanics\", 3],\n",
-    "        [\"PHYS\", 2100, \"General Relativity and Cosmology\", 3],\n",
-    "        [\"PHYS\", 2140, \"Statistical Mechanics\", 4],\n",
-    "        [\"PHYS\", 2210, \"Physics for Scientists and Engineers I\", 4],\n",
-    "        [\"PHYS\", 2220, \"Physics for Scientists and Engineers II\", 4],\n",
-    "        [\"PHYS\", 3210, \"Physics for Scientists I (Honors)\", 4],\n",
-    "        [\"PHYS\", 3220, \"Physics for Scientists II (Honors)\", 4],\n",
-    "        [\"MATH\", 1250, \"Calculus for AP Students I\", 4],\n",
-    "        [\"MATH\", 1260, \"Calculus for AP Students II\", 4],\n",
-    "        [\"MATH\", 1210, \"Calculus I\", 4],\n",
-    "        [\"MATH\", 1220, \"Calculus II\", 4],\n",
-    "        [\"MATH\", 2210, \"Calculus III\", 3],\n",
-    "        [\"MATH\", 2270, \"Linear Algebra\", 4],\n",
-    "        [\"MATH\", 2280, \"Introduction to Differential Equations\", 4],\n",
-    "        [\"MATH\", 3210, \"Foundations of Analysis I\", 4],\n",
-    "        [\"MATH\", 3220, \"Foundations of Analysis II\", 4],\n",
-    "        [\"CS\", 1030, \"Foundations of Computer Science\", 3],\n",
-    "        [\"CS\", 1410, \"Introduction to Object-Oriented Programming\", 4],\n",
-    "        [\"CS\", 2420, \"Introduction to Algorithms & Data Structures\", 4],\n",
-    "        [\"CS\", 2100, \"Discrete Structures\", 3],\n",
-    "        [\"CS\", 3500, \"Software Practice\", 4],\n",
-    "        [\"CS\", 3505, \"Software Practice II\", 3],\n",
-    "        [\"CS\", 3810, \"Computer Organization\", 4],\n",
-    "        [\"CS\", 4400, \"Computer Systems\", 4],\n",
-    "        [\"CS\", 4150, \"Algorithms\", 3],\n",
-    "        [\"CS\", 3100, \"Models of Computation\", 3],\n",
-    "        [\"CS\", 3200, \"Introduction to Scientific Computing\", 3],\n",
-    "        [\"CS\", 4000, \"Senior Capstone Project - Design Phase\", 3],\n",
-    "        [\"CS\", 4500, \"Senior Capstone Project\", 3],\n",
-    "        [\"CS\", 4940, \"Undergraduate Research\", 3],\n",
-    "        [\"CS\", 4970, \"Computer Science Bachelors Thesis\", 3],\n",
-    "    ]\n",
-    ")\n",
-    "\n",
-    "Term.insert(dict(term_year=year, term=term) for year in range(1999, 2019) for term in [\"Spring\", \"Summer\", \"Fall\"])\n",
-    "\n",
-    "Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]\n",
-    "\n",
-    "CurrentTerm().insert1({**Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]})\n",
-    "\n",
-    "\n",
-    "def make_section(prob):\n",
-    "    for c in (Course * Term).proj():\n",
-    "        for sec in \"abcd\":\n",
-    "            if random.random() < prob:\n",
-    "                break\n",
-    "            yield {\n",
-    "                **c,\n",
-    "                \"section\": sec,\n",
-    "                \"auditorium\": random.choice(\"ABCDEF\") + str(random.randint(1, 100)),\n",
-    "            }\n",
-    "\n",
-    "\n",
-    "Section.insert(make_section(0.5))"
-   ]
+   "source": "Student.insert(dict(k, student_id=i) for i, k in zip(range(100, 300), yield_students()))\n\nDepartment.insert(\n    dict(\n        dept=dept,\n        dept_name=name,\n        dept_address=fake.address(),\n        dept_phone=fake.phone_number()[:20],\n    )\n    for dept, name in [\n        [\"CS\", \"Computer Science\"],\n        [\"BIOL\", \"Life Sciences\"],\n        [\"PHYS\", \"Physics\"],\n        [\"MATH\", \"Mathematics\"],\n    ]\n)\n\nStudentMajor.insert(\n    {**s, **d, \"declare_date\": fake.date_between(start_date=datetime.date(1999, 1, 1))}\n    for s, d in zip(Student().keys(), random.choices(Department().keys(), k=len(Student())))\n    if random.random() < 0.75\n)\n\n# from https://www.utah.edu/\nCourse.insert(\n    [\n        [\"BIOL\", 1006, \"World of Dinosaurs\", 3],\n        [\"BIOL\", 1010, \"Biology in the 21st Century\", 3],\n        [\"BIOL\", 1030, \"Human Biology\", 3],\n        [\"BIOL\", 1210, \"Principles of Biology\", 4],\n        [\"BIOL\", 2010, \"Evolution & Diversity of Life\", 3],\n        [\"BIOL\", 2020, \"Principles of Cell Biology\", 3],\n        [\"BIOL\", 2021, \"Principles of Cell Science\", 4],\n        [\"BIOL\", 2030, \"Principles of Genetics\", 3],\n        [\"BIOL\", 2210, \"Human Genetics\", 3],\n        [\"BIOL\", 2325, \"Human Anatomy\", 4],\n        [\"BIOL\", 2330, \"Plants & Society\", 3],\n        [\"BIOL\", 2355, \"Field Botany\", 2],\n        [\"BIOL\", 2420, \"Human Physiology\", 4],\n        [\"PHYS\", 2040, \"Classcal Theoretical Physics II\", 4],\n        [\"PHYS\", 2060, \"Quantum Mechanics\", 3],\n        [\"PHYS\", 2100, \"General Relativity and Cosmology\", 3],\n        [\"PHYS\", 2140, \"Statistical Mechanics\", 4],\n        [\"PHYS\", 2210, \"Physics for Scientists and Engineers I\", 4],\n        [\"PHYS\", 2220, \"Physics for Scientists and Engineers II\", 4],\n        [\"PHYS\", 3210, \"Physics for Scientists I (Honors)\", 4],\n        [\"PHYS\", 3220, \"Physics for Scientists II (Honors)\", 4],\n        [\"MATH\", 1250, \"Calculus for AP Students I\", 4],\n        [\"MATH\", 1260, \"Calculus for AP Students II\", 4],\n        [\"MATH\", 1210, \"Calculus I\", 4],\n        [\"MATH\", 1220, \"Calculus II\", 4],\n        [\"MATH\", 2210, \"Calculus III\", 3],\n        [\"MATH\", 2270, \"Linear Algebra\", 4],\n        [\"MATH\", 2280, \"Introduction to Differential Equations\", 4],\n        [\"MATH\", 3210, \"Foundations of Analysis I\", 4],\n        [\"MATH\", 3220, \"Foundations of Analysis II\", 4],\n        [\"CS\", 1030, \"Foundations of Computer Science\", 3],\n        [\"CS\", 1410, \"Introduction to Object-Oriented Programming\", 4],\n        [\"CS\", 2420, \"Introduction to Algorithms & Data Structures\", 4],\n        [\"CS\", 2100, \"Discrete Structures\", 3],\n        [\"CS\", 3500, \"Software Practice\", 4],\n        [\"CS\", 3505, \"Software Practice II\", 3],\n        [\"CS\", 3810, \"Computer Organization\", 4],\n        [\"CS\", 4400, \"Computer Systems\", 4],\n        [\"CS\", 4150, \"Algorithms\", 3],\n        [\"CS\", 3100, \"Models of Computation\", 3],\n        [\"CS\", 3200, \"Introduction to Scientific Computing\", 3],\n        [\"CS\", 4000, \"Senior Capstone Project - Design Phase\", 3],\n        [\"CS\", 4500, \"Senior Capstone Project\", 3],\n        [\"CS\", 4940, \"Undergraduate Research\", 3],\n        [\"CS\", 4970, \"Computer Science Bachelors Thesis\", 3],\n    ]\n)\n\nTerm.insert(dict(term_year=year, term=term) for year in range(1999, 2019) for term in [\"Spring\", \"Summer\", \"Fall\"])\n\nTerm().to_dicts(order_by=(\"term_year DESC\", \"term DESC\"), limit=1)[0]\n\nCurrentTerm().insert1({**Term().to_dicts(order_by=(\"term_year DESC\", \"term DESC\"), limit=1)[0]})\n\n\ndef make_section(prob):\n    for c in (Course * Term).proj():\n        for sec in \"abcd\":\n            if random.random() < prob:\n                break\n            yield {\n                **c,\n                \"section\": sec,\n                \"auditorium\": random.choice(\"ABCDEF\") + str(random.randint(1, 100)),\n            }\n\n\nSection.insert(make_section(0.5))"
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 200/200 [00:27<00:00,  7.17it/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Enrollment\n",
-    "terms = Term().fetch(\"KEY\")\n",
-    "quit_prob = 0.1\n",
-    "for student in tqdm(Student.fetch(\"KEY\")):\n",
-    "    start_term = random.randrange(len(terms))\n",
-    "    for term in terms[start_term:]:\n",
-    "        if random.random() < quit_prob:\n",
-    "            break\n",
-    "        else:\n",
-    "            sections = ((Section & term) - (Course & (Enroll & student))).fetch(\"KEY\")\n",
-    "            if sections:\n",
-    "                Enroll.insert(\n",
-    "                    {**student, **section} for section in random.sample(sections, random.randrange(min(5, len(sections))))\n",
-    "                )\n",
-    "\n",
-    "# assign random grades\n",
-    "grades = LetterGrade.fetch(\"grade\")\n",
-    "\n",
-    "grade_keys = Enroll.fetch(\"KEY\")\n",
-    "random.shuffle(grade_keys)\n",
-    "grade_keys = grade_keys[: len(grade_keys) * 9 // 10]\n",
-    "\n",
-    "Grade.insert({**key, \"grade\": grade} for key, grade in zip(grade_keys, random.choices(grades, k=len(grade_keys))))"
-   ]
+   "outputs": [],
+   "source": "# Enrollment\nterms = Term().keys()\nquit_prob = 0.1\nfor student in tqdm(Student().keys()):\n    start_term = random.randrange(len(terms))\n    for term in terms[start_term:]:\n        if random.random() < quit_prob:\n            break\n        else:\n            sections = ((Section & term) - (Course & (Enroll & student))).keys()\n            if sections:\n                Enroll.insert(\n                    {**student, **section} for section in random.sample(sections, random.randrange(min(5, len(sections))))\n                )\n\n# assign random grades\ngrades = LetterGrade().to_arrays(\"grade\")[0]\n\ngrade_keys = Enroll().keys()\nrandom.shuffle(grade_keys)\ngrade_keys = grade_keys[: len(grade_keys) * 9 // 10]\n\nGrade.insert({**key, \"grade\": grade} for key, grade in zip(grade_keys, random.choices(grades, k=len(grade_keys))))"
   },
   {
    "cell_type": "markdown",
@@ -988,7 +847,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": "# Cleanup\nschema.drop()"
   }
  ],
  "metadata": {
@@ -1012,4 +871,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

docs/src/archive/tutorials/json.ipynb

@@ -66,22 +66,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "dc318298-b819-4f06-abbd-7bb7544dd431",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[2023-02-12 00:14:33,027][INFO]: Connecting [email protected]:3306\n",
-      "[2023-02-12 00:14:33,039][INFO]: Connected [email protected]:3306\n"
-     ]
-    }
-   ],
-   "source": [
-    "schema = dj.Schema(f\"{dj.config['database.user']}_json\")"
-   ]
+   "outputs": [],
+   "source": "# Clean up any existing schema from previous runs\nschema_name = f\"{dj.config['database.user']}_json\"\nschema = dj.Schema(schema_name, create_tables=False)\nschema.drop()\n\n# Create fresh schema\nschema = dj.Schema(schema_name)"
   },
   {
    "cell_type": "code",
@@ -828,26 +817,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "bb5f0448",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[{'name': 'business', 'car_name': 'Chaching', 'car_length': '100'},\n",
-       " {'name': 'engineering', 'car_name': 'Rever', 'car_length': '20.5'},\n",
-       " {'name': 'marketing', 'car_name': None, 'car_length': None}]"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "q_untyped.fetch(as_dict=True)"
-   ]
+   "outputs": [],
+   "source": "q_untyped.to_dicts()"
   },
   {
    "cell_type": "code",
@@ -958,26 +932,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "8a93dbf9",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[{'name': 'business', 'car_name': 'Chaching', 'car_length': 100.0},\n",
-       " {'name': 'engineering', 'car_name': 'Rever', 'car_length': 20.5},\n",
-       " {'name': 'marketing', 'car_name': None, 'car_length': None}]"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "q_typed.fetch(as_dict=True)"
-   ]
+   "outputs": [],
+   "source": "q_typed.to_dicts()"
   },
   {
    "cell_type": "markdown",
@@ -1077,4 +1036,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}