Generalize datastore analysis and cleanup tools

.github/workflows/build.yml

@@ -0,0 +1,49 @@
+name: build
+
+on:
+  push:
+    branches: [ main ]
+    tags: [ "v*" ]
+  pull_request:
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          python -m pip install -U pip
+          python -m pip install .
+          python -m pip install pytest
+      - name: Test
+        run: |
+          pytest -q
+
+  publish:
+    needs: ci
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Build
+        run: |
+          python -m pip install -U pip build
+          python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/pr.yml

@@ -0,0 +1,35 @@
+name: pr
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install
+        run: |
+          python -m pip install -U pip
+          python -m pip install .
+          python -m pip install pytest ruff black build pip-audit
+      - name: Lint
+        run: |
+          ruff check .
+          black --check .
+      - name: Test
+        run: pytest -q
+      - name: Build and verify
+        run: |
+          python -m build
+          twine check dist/* || true
+      - name: Security audit
+        run: |
+          pip-audit -r requirements.txt || true

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: release
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Install
+        run: |
+          python -m pip install -U pip
+          python -m pip install .
+          python -m pip install pytest ruff black build python-semantic-release pip-audit
+      - name: Lint
+        run: |
+          ruff check .
+          black --check .
+      - name: Test
+        run: pytest -q
+      - name: Build and verify
+        run: |
+          python -m build
+          twine check dist/* || true
+      - name: Security audit
+        run: |
+          pip-audit -r requirements.txt || true
+      - name: Semantic Release (version, tag, GitHub release, PyPI)
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: semantic-release publish

.gitignore

@@ -205,3 +205,10 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+
+# Local configuration
+config.yaml
+
+# Editor/OS
+.DS_Store
+Thumbs.db

CONTRIBUTING.md

@@ -0,0 +1,21 @@
+# Contributing
+
+Thanks for your interest in contributing!
+
+- Open an issue to discuss substantial changes.
+- Fork and create feature branches from `main`.
+- Run formatting and tests before submitting a PR.
+
+## Dev setup
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install -U pip
+pip install -e .
+```
+
+## Testing
+
+```bash
+python -m pytest -q
+```

README.md

@@ -1,2 +1,107 @@
 # local-storage-utils
-Set of scripts and tools for managing GCP Datastore data in local
+
+Utilities for analyzing and managing local Datastore/Firestore (Datastore mode) data. Works with both the Datastore Emulator and GCP using Application Default Credentials.
+
+## Install (PyPI)
+
+```bash
+pip install local-storage-utils
+```
+
+This installs the `lsu` CLI.
+
+## Install (from source)
+
+git clone <this-repo-url>
+cd local-storage-utils
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install -U pip
+pip install -e .
+
+### Troubleshooting local installs
+- If you see "Command 'python' not found", use `python3 -m venv .venv` (above). Inside the venv, `python` will point to Python 3.
+- If you see "externally-managed-environment", you are attempting a system-wide install. Always install into a virtual environment:
+  - Create a venv: `python3 -m venv .venv && source .venv/bin/activate`
+  - Then use the venv pip: `python -m pip install -U pip && pip install -e .`
+  ```bash
+  sudo apt-get update && sudo apt-get install -y python3-venv
+  ```
+
+## Configuration
+
+- Create a local `config.yaml` in your working directory. It is gitignored and not included in the repo.
+- Any CLI flag overrides values from `config.yaml`.
+- If neither config nor flags provide a value, the tool falls back to environment variables (for emulator detection) or sensible defaults.
+
+Example `config.yaml`:
+
+```yaml
+project_id: "my-project"          # If omitted, ADC/env will be used
+emulator_host: "localhost:8010"   # If set, uses Datastore Emulator
+
+# Explicit filters (empty means all)
+namespaces: [""]                   # Empty -> iterate all namespaces (including default "")
+kinds: []                          # Empty -> iterate all kinds per namespace
+
+# Optional defaults
+kind: "SourceCollectionStateEntity"  # Default for analyze-fields
+
+# Cleanup
+ttl_field: "expireAt"
+delete_missing_ttl: true
+batch_size: 500
+
+# Analysis
+group_by_field: null
+
+# Logging
+log_level: "INFO"
+```
+
+## CLI usage
+
+```bash
+# Kind-level counts and size estimates
+lsu analyze-kinds --project my-project
+
+# Use all namespaces/kinds by default, or restrict explicitly
+lsu analyze-kinds --namespace "" --namespace tenant-a --kind SourceCollectionStateEntity
+
+# Field contribution analysis (falls back to config.kind/config.namespace if not provided)
+lsu analyze-fields --kind SourceCollectionStateEntity --namespace "" --group-by batchId
+
+# TTL cleanup across namespaces/kinds (dry-run)
+lsu cleanup --ttl-field expireAt --dry-run
+
+# TTL cleanup restricted to specific namespaces/kinds
+lsu cleanup --namespace "" --namespace tenant-a --kind pipeline-job
+```
+
+Use `--help` on any command for full options. Config can be provided via `config.yaml` or flags.
+
+## Development
+
+- Create a virtual environment and install in editable mode as shown above
+- Run tests:
+
+```bash
+python -m pip install pytest
+pytest -q
+```
+
+- Lint/format (optional if you use pre-commit/CI):
+```bash
+python -m pip install ruff black
+ruff check .
+black .
+```
+
+## Publishing
+
+- Automated: pushing to `main` triggers versioning, tagging, GitHub release, and PyPI publish via semantic-release.
+- Prerequisites:
+  - Add a PyPI token to repo secrets as `PYPI_API_TOKEN`.
+  - Use conventional commits for proper versioning.
+
+Main branch should be protected (require PRs, disallow direct pushes) in repository settings.