Merge pull request #44 from NOAA-GSL/staging

Staging

Author: Hackshaven

.devcontainer/entrypoint.sh

@@ -1,112 +1,12 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# ====== WIKI SYNC SECTION ======
-FORCE_UPDATE=0
-REFRESH_SECONDS=${DOCS_REFRESH_SECONDS:-3600} # default 1 hour
-# Allow skipping wiki sync entirely (useful in restricted-network envs)
-SKIP_WIKI=${DATAVIZHUB_SKIP_WIKI_SYNC:-0}
-# Hard timeout for git clone so startup never blocks indefinitely
-GIT_CLONE_TIMEOUT_SECONDS=${GIT_TIMEOUT_SECONDS:-20}
-
-while [[ ${1:-} ]]; do
-  case "$1" in
-    --force) FORCE_UPDATE=1; shift ;;
-    --refresh-seconds) REFRESH_SECONDS="$2"; shift 2 ;;
-    *) echo "[entrypoint] Unknown option: "$1"" >&2; exit 2 ;;
-  esac
-done
-
 # Copy default .env if missing
 if [[ ! -f .env && -f .devcontainer/.env ]]; then
   echo "[entrypoint] Seeding .env from .devcontainer/.env"
   cp .devcontainer/.env .env
 fi
 
-WIKI_URL="https://github.com/NOAA-GSL/datavizhub.wiki.git"
-DOCS_DIR="wiki"
-META_FILE="$DOCS_DIR/.mirror_meta"
-
-# Fix stale, non-writable wiki dirs
-if [[ -d "$DOCS_DIR" && ! -w "$DOCS_DIR" ]]; then
-  echo "[entrypoint] Existing $DOCS_DIR not writable — removing"
-  rm -rf "$DOCS_DIR" || true
-fi
-
-now_epoch() { date +%s; }
-should_update=0
-
-if [[ ! -d "$DOCS_DIR" ]]; then
-  should_update=1
-elif [[ $FORCE_UPDATE -eq 1 ]]; then
-  should_update=1
-else
-  last_sync=0
-  if [[ -f "$META_FILE" ]]; then
-    # Avoid sourcing untrusted content; parse the expected key instead
-    last_sync=$(grep -E '^last_sync_epoch=' "$META_FILE" | tail -n1 | cut -d'=' -f2 | tr -d '"' || true)
-    last_sync=${last_sync:-0}
-  fi
-  now=$(now_epoch)
-  age=$(( now - last_sync ))
-  if (( age >= REFRESH_SECONDS )); then
-    should_update=1
-  fi
-fi
-
-if [[ "$SKIP_WIKI" == "1" ]]; then
-  echo "[entrypoint] Skipping wiki sync (DATAVIZHUB_SKIP_WIKI_SYNC=1)"
-else
-  if [[ $should_update -eq 1 ]]; then
-    echo "[entrypoint] Cloning wiki (timeout ${GIT_CLONE_TIMEOUT_SECONDS}s)..."
-    tmp_dir="${DOCS_DIR}.new"
-    rm -rf "$tmp_dir"
-    CLONE_CMD=(git clone --depth=1 "$WIKI_URL" "$tmp_dir")
-    if command -v timeout >/dev/null 2>&1; then
-      if timeout "${GIT_CLONE_TIMEOUT_SECONDS}s" "${CLONE_CMD[@]}"; then
-        clone_ok=1
-      else
-        clone_ok=0
-      fi
-    else
-      # Fallback: run clone in background and wait up to N seconds
-      set +e
-      "${CLONE_CMD[@]}" &
-      pid=$!
-      waited=0
-      while kill -0 "$pid" >/dev/null 2>&1; do
-        if [[ $waited -ge $GIT_CLONE_TIMEOUT_SECONDS ]]; then
-          echo "[entrypoint] WARN: git clone exceeded ${GIT_CLONE_TIMEOUT_SECONDS}s; continuing without wiki"
-          kill "$pid" >/dev/null 2>&1 || true
-          clone_ok=0
-          break
-        fi
-        sleep 1
-        waited=$(( waited + 1 ))
-      done
-      if [[ ${clone_ok:-1} -ne 0 && ! -d "$tmp_dir" ]]; then
-        # If the process finished but dir not present, treat as failure
-        clone_ok=0
-      fi
-      set -e
-    fi
-    if [[ ${clone_ok:-0} -eq 1 ]]; then
-      rm -rf "$tmp_dir/.git"
-      echo "source_url=\"$WIKI_URL\"" > "$tmp_dir/.mirror_meta"
-      echo "last_sync_epoch=$(now_epoch)" >> "$tmp_dir/.mirror_meta"
-      rm -rf "$DOCS_DIR" || true
-      mv "$tmp_dir" "$DOCS_DIR" || cp -R "$tmp_dir"/. "$DOCS_DIR"
-      rm -rf "$tmp_dir"
-      echo "[entrypoint] Wiki synced"
-    else
-      echo "[entrypoint] WARN: Wiki clone skipped/failed; proceeding without update"
-      rm -rf "$tmp_dir" || true
-    fi
-  else
-    echo "[entrypoint] Wiki is fresh; skipping sync"
-  fi
-fi
-
 # ====== SERVICE START SECTION ======
 AUTOSTART_API=${DATAVIZHUB_AUTOSTART_API:-1}
 AUTOSTART_RQ=${DATAVIZHUB_AUTOSTART_RQ:-${DATAVIZHUB_USE_REDIS:-0}}

.github/workflows/sync-wiki.yml

@@ -0,0 +1,50 @@
+name: Sync Wiki to Docs
+
+on:
+  push:
+    branches:
+      - "**"          # Run on all branches
+  pull_request:        # Ensure PR builds include wiki
+  schedule:
+    - cron: "0 6 * * *"  # Daily sync at 6 AM UTC
+  workflow_dispatch:    # Manual trigger
+
+permissions:
+  contents: write  # Allow pushing synced wiki changes
+
+jobs:
+  sync-wiki:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main repo
+        uses: actions/checkout@v4
+        with:
+          path: main
+
+      - name: Checkout wiki repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}.wiki
+          path: wiki
+
+      - name: Sync wiki into docs/source/wiki
+        run: |
+          rsync -av --delete wiki/ main/docs/source/wiki/
+
+      - name: Commit wiki changes (main only)
+        run: |
+          cd main
+          if [ "${{ github.ref_name }}" = "main" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add docs/source/wiki/
+            if ! git diff --cached --quiet; then
+              git commit -m "chore(docs): sync wiki into docs/source/wiki"
+              git push
+            else
+              echo "No changes to commit"
+            fi
+          else
+            echo "Skipping commit: branch is ${{ github.ref_name }}"
+          fi

.gitignore

@@ -82,9 +82,6 @@ cover/
 docs/_build/
 docs/build/
 
-# Wiki documentation in dev container (used as context)
-wiki/
-app/wiki/
 
 # Samples: generated artifacts (keep repo small and clean)
 samples/samples/quickstart_outputs/

AGENTS.md

@@ -52,4 +52,4 @@ Dev container:
 
 ## Documentation Sources
 - Wiki: https://github.com/NOAA-GSL/datavizhub/wiki (authoritative documentation for humans and AIs).
-- Dev container mirror: `/wiki` contains an auto-cloned snapshot of the wiki for offline/context use. It auto-refreshes at most once per hour on container start. Force refresh with `bash .devcontainer/postStart.sh --force`. This folder lives outside the repo and is not part of the main repository—do not commit its contents.
+- CI-synced wiki: A GitHub Action mirrors the GitHub Wiki into `docs/source/wiki/` so it is built with Sphinx. Do not edit `docs/source/wiki/` directly; changes are committed only on `main`.

CONTRIBUTING.md

@@ -0,0 +1,92 @@
+# Contributing to DataVizHub
+
+Thanks for your interest in contributing!  
+This project thrives on community contributions, and we welcome improvements of all kinds.
+
+---
+
+## License and Contributor Terms
+
+- DataVizHub is licensed under the MIT License. See `LICENSE` at the repository root.
+- By submitting a pull request, issue suggestion, or any code/documentation/artwork (“Contribution”),
+  you agree to license your Contribution under the MIT License, and you represent that you have the
+  right to do so.
+- Do not contribute code or assets you don’t have rights to. If you include third‑party code or data,
+  ensure it is compatible with MIT and include proper attribution as required by the original license.
+- No CLA is required at this time; contributions are accepted under the project’s MIT terms.
+
+If you have questions about licensing or attribution, please open an issue before submitting your PR.
+
+---
+
+## Branching Workflow
+
+To keep development organized, we use a two-branch model:
+
+- **`main`** → Stable, production-ready branch.  
+  - Always passes tests and CI/CD.  
+  - Used for tagged releases.  
+  - Do **not** commit directly to `main`.
+
+- **`staging`** → Integration branch.  
+  - Collects feature branches and fixes before merging into `main`.  
+  - Used for testing, docs, and CI validation.  
+
+### Rules
+1. **Feature Development**
+   - Branch off `staging`:  
+     ```bash
+     git checkout staging
+     git pull origin staging
+     git checkout -b feature/my-feature
+     ```
+   - Open a Pull Request (PR) into `staging`.
+
+2. **Testing & Integration**
+   - PRs are merged into `staging`.  
+   - CI/CD runs against `staging`.  
+   - Once stable, `staging` is merged into `main`.  
+
+3. **Syncing Main & Staging**
+   - Occasionally merge `main → staging` to keep hotfixes and metadata aligned.  
+   - Always merge `staging → main` via PR when ready to release.
+
+---
+
+## Code Style
+
+- Python 3.10+ required.  
+- Follow [PEP8](https://peps.python.org/pep-0008/).  
+- Run `ruff` and `pytest` locally before opening a PR.
+
+---
+
+## Testing
+
+1. Install dev dependencies:
+   ```bash
+   poetry install
+   ```
+2. Run tests:
+   ```bash
+   pytest
+   ```
+
+---
+
+## Pull Requests
+
+- Make sure your branch is up-to-date with `staging`.  
+- Include descriptive commit messages.  
+- Request a review from at least one maintainer.  
+
+---
+
+## Releases
+
+- Releases are tagged from `main`.  
+- `staging` must be fully merged into `main` before tagging.
+
+---
+
+Thanks again for contributing!

README.md

@@ -689,9 +689,7 @@ with open("/data/out/movie.mp4", "rb") as f:
   - API Routers and Endpoints: https://github.com/NOAA-GSL/datavizhub/wiki/DataVizHub-API-Routers-and-Endpoints
   - Security Quickstart: https://github.com/NOAA-GSL/datavizhub/wiki/DataVizHub-API-Security-Quickstart
 - API docs (GitHub Pages): https://noaa-gsl.github.io/datavizhub/
-- Dev container: A read-only mirror of the wiki is auto-cloned into `/wiki` when the dev container starts. It auto-refreshes at most once per hour. This folder is ignored by Git and is not part of the repository on GitHub.
-- Force refresh: `bash .devcontainer/postStart.sh --force` (or set `DOCS_REFRESH_SECONDS` to adjust the hourly cadence).
-- Note: There is no `docs/` directory in the main repo. If you are not using the dev container, read the wiki directly.
+- CI-synced wiki: A GitHub Action mirrors the wiki into `docs/source/wiki/` so Sphinx can build it with the docs. Sync commits occur only on `main`; PRs/branches use the synced copy for builds without committing changes.
 
 ## Notes
 - Paths: examples use absolute paths (e.g., `/data/...`) for clarity, but the library does not assume a specific root; configure paths via your own settings or env vars if preferred.

.devcontainer/postStart.sh docs/source/_static/.gitkeep.devcontainer/postStart.sh renamed to docs/source/_static/.gitkeep

@@ -19,14 +19,19 @@
 # -- General configuration ---------------------------------------------------
 
 extensions = [
+    "myst_parser",  # Enable Markdown support
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
     "sphinx.ext.napoleon",
     "sphinx.ext.viewcode",
 ]
 
 templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+exclude_patterns = [
+    "_build",
+    "Thumbs.db",
+    ".DS_Store",
+]
 
 # Napoleon settings for NumPy-style docstrings
 napoleon_google_docstring = False
@@ -43,22 +48,42 @@
 
 # -- Options for HTML output -------------------------------------------------
 
+# Default to alabaster; override if RTD theme is available
+html_theme = "alabaster"
 if _ils.find_spec("sphinx_rtd_theme") is not None:  # pragma: no cover - docs build env
     html_theme = "sphinx_rtd_theme"
-else:
-    html_theme = "alabaster"
 html_static_path = ["_static"]
 
 # Autosummary/napoleon tweaks for cleaner API pages
 autosummary_generate = True
 autosummary_imported_members = False
-html_theme_options = {
-    "navigation_depth": 3,
-    "collapse_navigation": False,
-    "sticky_navigation": True,
+if html_theme == "sphinx_rtd_theme":
+    html_theme_options = {
+        "navigation_depth": 3,
+        "collapse_navigation": False,
+        "sticky_navigation": True,
+    }
+else:
+    html_theme_options = {}
+
+# Support both .rst and .md sources
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
 }
 
 # Mock optional heavy dependencies to allow building API docs without extras
+# Note for maintainers:
+# - Importing some API modules executes module-level side effects that are
+#   incompatible with a docs-build environment (CI or local), for example:
+#   - Creating or touching filesystem paths (e.g., '/data/uploads') during import
+#   - Initializing FastAPI app and router wiring (which pulls in optional deps)
+#   - Triggering background worker wiring (RQ/Redis) or reading env config
+#   - Potential network or service assumptions when modules import clients
+# - To keep the docs build hermetic and fast, we mock these modules so that
+#   autodoc can still render signatures without executing their import-time code.
+# - If you reduce import-time side effects in these modules, feel free to
+#   remove them from this list.
 autodoc_mock_imports = [
     "boto3",
     "botocore",
@@ -77,4 +102,13 @@
     "ffmpeg",
     "ffmpeg_python",
     "ffmpeg-python",
+    # Mock API submodules that cause side effects on import during docs build
+    "datavizhub.api",
+    "datavizhub.api.server",
+    "datavizhub.api.routers",
+    "datavizhub.api.routers.cli",
+    "datavizhub.api.routers.files",
+    "datavizhub.api.workers",
+    "datavizhub.api.workers.executor",
+    "datavizhub.api.workers.jobs",
 ]

docs/source/conf.py

@@ -0,0 +1,15 @@
+# Wiki Sync Workflow
+
+## Why
+- Keeps wiki knowledge versioned with code for branches and PRs.
+- Ensures contributors can build docs with up-to-date wiki content.
+- Simplifies builds by removing Docker-based wiki mirroring.
+
+## How
+- A GitHub Action syncs the GitHub Wiki into `docs/source/wiki/` on pushes, PRs, and a daily schedule.
+- Sphinx (with `myst_parser`) builds Markdown wiki pages alongside the docs.
+- Commits of wiki changes occur only on `main` to keep history clean.
+
+## Editing Rules
+- Edit wiki pages via the GitHub Wiki UI.
+- Do not edit `docs/source/wiki/` directly; it is overwritten on sync.