feat: fully featured bigtable and hbase clients with tests

Author: akhileshh

.bumpversion.cfg

@@ -0,0 +1,9 @@
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+tag_name = v{new_version}
+
+[bumpversion:file:pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"

.github/workflows/test.yml

@@ -0,0 +1,76 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    env:
+      BIGTABLE_EMULATOR_HOST: localhost:8086
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start BigTable emulator
+        run: |
+          docker run -d -p 8086:8086 --name bigtable-emulator \
+            gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators \
+            gcloud beta emulators bigtable start --host-port=0.0.0.0:8086
+          timeout 30 bash -c 'until nc -z localhost 8086; do sleep 1; done'
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-cov
+          pip install -e .
+
+      - name: Run tests with coverage
+        run: pytest --cov=kvdbclient --cov-report=xml tests/
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

Dockerfile

@@ -1,6 +1,18 @@
-FROM python:3.7
+FROM python:3.12
+
+WORKDIR /app
+
+# Install Google Cloud SDK + BigTable emulator
+RUN apt-get update && \
+    apt-get install -y apt-transport-https ca-certificates gnupg curl && \
+    curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+      gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
+      > /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    apt-get update && \
+    apt-get install -y google-cloud-cli google-cloud-cli-bigtable-emulator && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY . /app
-RUN pip install pip==20.0.1 \
-    && pip install --no-cache-dir --upgrade -r requirements.txt \
-    && pip install --no-cache-dir --upgrade -r requirements-dev.txt
+RUN pip install --no-cache-dir --upgrade -r requirements-dev.txt
+RUN pip install -e .

MANIFEST.in

@@ -1 +1,2 @@
-include requirements.txt
+include LICENSE
+include README.md

README.md

@@ -1,2 +1,56 @@
 # KVDbClient
-generic key-value database client
+
+[![codecov](https://codecov.io/gh/seung-lab/KVDbClient/graph/badge.svg)](https://app.codecov.io/gh/seung-lab/KVDbClient)
+
+A Python client library providing a unified interface for key-value database backends. Currently supports Google Cloud BigTable and Apache HBase.
+
+Built for:
+
+- Node read/write operations with automatic serialization
+- Concurrency control via row-level locking
+- Atomic unique ID generation
+- Operation logging and auditing
+- Configurable column families with per-attribute serializers (NumPy arrays, JSON, Pickle, strings)
+
+## Installation
+
+```bash
+pip install kvdbclient
+```
+
+For development:
+
+```bash
+git clone https://github.com/seung-lab/KVDbClient.git
+cd KVDbClient
+pip install -e .
+```
+
+## Usage
+
+```python
+from kvdbclient import get_client_class, BigTableConfig
+
+config = BigTableConfig(PROJECT="my-project", INSTANCE="my-instance", ADMIN=True, READ_ONLY=False)
+client = get_client_class("bigtable")("my_table", config)
+```
+
+The backend is selected by passing `"bigtable"` or `"hbase"` to `get_client_class()`. Alternatively, `get_default_client_info()` reads configuration from environment variables automatically.
+
+## Backends
+
+**Google BigTable** — Uses the `google-cloud-bigtable` SDK. Configure with `BigTableConfig` or set `BIGTABLE_PROJECT` and `BIGTABLE_INSTANCE` environment variables.
+
+**Apache HBase** — Communicates via the HBase REST API using HTTP. Configure with `HBaseConfig` or set the `HBASE_REST_URL` environment variable.
+
+Set `PCG_BACKEND_TYPE` to `bigtable` or `hbase` to control which backend `get_default_client_info()` uses.
+
+## Testing
+
+```bash
+pytest
+```
+
+## License
+
+MIT

codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+    patch:
+      default:
+        target: 25%

kvdbclient/__init__.py

@@ -1,28 +1,67 @@
+"""
+Sub packages/modules for backend storage clients.
+Supports Google BigTable and Apache HBase.
+
+A simple client needs to be able to create the table,
+store table meta and to write and read node information.
+Also needs locking support to prevent race conditions
+when modifying root/parent nodes.
+
+In addition, clients with more features like generating unique IDs
+and logging facilities can be implemented by inherting respective base classes.
+
+These methods are in separate classes because they are logically related.
+This also makes it possible to have different backend storage solutions,
+making it possible to use any unique features these solutions may provide.
+
+Please see `base.py` for more details.
+"""
+
 from collections import namedtuple
+from os import environ
+from typing import Union
 
+from .base import ColumnFamilyConfig, DEFAULT_COLUMN_FAMILIES
+from .bigtable import BigTableConfig
+from .bigtable import get_client_info as get_bigtable_client_info
 from .bigtable.client import Client as BigTableClient
-from .bigtable.attributes import Attribute
-from .serializers import Serializer
-from .serializers import UInt64String
+from .hbase import HBaseConfig
+from .hbase import get_client_info as get_hbase_client_info
+from .hbase.client import Client as HBaseClient
+
+ClientType = Union[BigTableClient, HBaseClient]
 
 
 _backend_clientinfo_fields = ("TYPE", "CONFIG")
-_backend_clientinfo_defaults = (None, None)
+_backend_clientinfo_defaults = ("bigtable", None)
 BackendClientInfo = namedtuple(
     "BackendClientInfo",
     _backend_clientinfo_fields,
     defaults=_backend_clientinfo_defaults,
 )
 
 
+def get_client_class(backend_type: str = "bigtable"):
+    """Return the client class for the given backend type."""
+    backend_type = (backend_type or "bigtable").lower()
+    if backend_type == "bigtable":
+        return BigTableClient
+    elif backend_type == "hbase":
+        return HBaseClient
+    else:
+        raise ValueError(f"Unknown backend type: {backend_type}")
+
+
 def get_default_client_info():
     """
     Load client from env variables.
     """
-
-    # TODO make dynamic after multiple platform support is added
-    from .bigtable import get_client_config as get_bigtable_client_config
+    backend_type = environ.get("PCG_BACKEND_TYPE", "bigtable").lower()
+    if backend_type == "hbase":
+        return BackendClientInfo(
+            TYPE="hbase", CONFIG=get_hbase_client_info()
+        )
 
     return BackendClientInfo(
-        CONFIG=get_bigtable_client_config(admin=True, read_only=False)
+        TYPE="bigtable", CONFIG=get_bigtable_client_info(admin=True, read_only=False)
     )