feat: Add GitHub CI/CD versioning support with env profile and Bearer tokens (#393)

Add native support for NiFi's GitHub Flow Registry Client to enable CI/CD workflows for versioned NiFi flows.

Git-specific versioning helpers:
- list_git_registry_buckets, get_git_registry_bucket
- list_git_registry_flows, get_git_registry_flow
- list_git_registry_flow_versions, deploy_git_registry_flow
- update_git_flow_ver with revert_flow_ver wait param
- ensure_registry_client, update_registry_client

Profile system enhancements:
- Add 'env' profile for pure environment variable configuration
- Add bearer token authentication (nifi_bearer_token, NIFI_BEARER_TOKEN)
- Ideal for GitHub Actions, containers, and CI/CD pipelines

Canvas improvements:
- Add name param and auto_stop to update_processor
- Add schedule_all_controllers for bulk enable/disable operations
- revert_flow_ver now refreshes revision internally

Infrastructure:
- Add SANs and cert config for github-cicd profile
- Update pylintrc for pylint 3.x compatibility
- Add GH_REGISTRY_TOKEN to coverage job

Bug fixes:
- Fix test_create_controller leaving orphaned controller services

Documentation:
- Add 1.1.0 release notes and env/bearer-token profile guides

Related: nipyapi-actions and nipyapi-workflow companion repositories

Author: Chaffelson

.github/workflows/ci.yml

@@ -39,6 +39,8 @@ jobs:
       run: make wait-ready NIPYAPI_PROFILE=single-user
 
     - name: Run integration tests with coverage
+      env:
+        GH_REGISTRY_TOKEN: ${{ secrets.GH_REGISTRY_TOKEN }}
       run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest --cov=nipyapi --cov-report=xml --cov-report=term-missing
 
     - name: Upload coverage to Codecov

.gitignore

@@ -190,3 +190,4 @@ htmlcov/
 
 # setuptools-scm generated version file
 nipyapi/_version.py
+.cursor/

Makefile

@@ -4,6 +4,10 @@
 # Default NiFi/Registry version for docker compose profiles
 NIFI_VERSION ?= 2.6.0
 
+# Load .env file if it exists (for secrets like GH_REGISTRY_TOKEN)
+-include .env
+export
+
 # Python command for cross-platform compatibility
 # Defaults to 'python' for conda/venv users, override with PYTHON=python3 for system installs
 PYTHON ?= python
@@ -227,17 +231,17 @@ extract-jks: ## extract PEM certificates from JKS: make extract-jks JKS_FILE=tru
 	fi
 	@echo "✅ Certificates extracted to resources/certs/extracted/"
 
-up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc (uses NIFI_VERSION=$(NIFI_VERSION))
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd (uses NIFI_VERSION=$(NIFI_VERSION))
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	$(DC) --profile $(NIPYAPI_PROFILE) up -d
 
 down: ## bring down all docker services
 	@echo "Bringing down Docker services (NIFI_VERSION=$(NIFI_VERSION))"
-	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc down -v --remove-orphans || true
+	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc --profile github-cicd down -v --remove-orphans || true
 	@echo "Verifying expected containers are stopped/removed:"
 	@COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) NIFI_VERSION=$(NIFI_VERSION) docker compose -f $(COMPOSE_FILE) ps --format "table {{.Name}}\t{{.State}}" | tail -n +2 | awk '{print " - " $$1 ": " $$2}' || true
 
-wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
+wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
 	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required"; exit 1; fi
 	@echo "Waiting for $(NIPYAPI_PROFILE) infrastructure to be ready..."
 	@NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) $(PYTHON) resources/scripts/wait_ready.py
@@ -269,7 +273,7 @@ gen-clients: ## generate NiFi and Registry clients from specs (use wv_spec_varia
 # Individual testing
 
 test: ## run pytest with provided NIPYAPI_PROFILE; config resolved by tests/conftest.py
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q
 
 test-su: ## shortcut: NIPYAPI_PROFILE=single-user pytest
@@ -285,7 +289,7 @@ test-oidc: check-certs ## shortcut: NIPYAPI_PROFILE=secure-oidc pytest (requires
 	NIPYAPI_PROFILE=secure-oidc $(MAKE) test
 
 test-specific: ## run specific pytest with provided NIPYAPI_PROFILE and TEST_ARGS
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	if [ -z "$(TEST_ARGS)" ]; then echo "TEST_ARGS is required (e.g., tests/test_utils.py::test_dump -v)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q $(TEST_ARGS)
 
@@ -333,8 +337,8 @@ test-all: ensure-certs ## run full e2e tests across automated profiles (requires
 	done
 	@echo "All profiles tested successfully"
 
-sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	@echo "🏗️ Setting up NiPyAPI sandbox with profile: $(NIPYAPI_PROFILE)"
 	@echo "=== 1/4: Starting infrastructure ==="
 	$(MAKE) up NIPYAPI_PROFILE=$(NIPYAPI_PROFILE)

docs/history.rst

@@ -2,6 +2,54 @@
 History
 =======
 
+1.1.0 (2025-12-04)
+-------------------
+
+| GitHub CI/CD Integration - Native support for NiFi's GitHub Flow Registry Client
+
+**GitHub Flow Registry Support**
+
+- **Git-specific versioning helpers**: New functions to work with NiFi's native GitHub Flow Registry Client
+    - ``list_git_registry_buckets``: List buckets (folders) in a Git-backed registry
+    - ``get_git_registry_bucket``: Get a specific bucket by name
+    - ``list_git_registry_flows``: List flows in a bucket
+    - ``get_git_registry_flow``: Get a specific flow by name
+    - ``list_git_registry_flow_versions``: List all versions (commits) of a flow
+    - ``deploy_git_registry_flow``: Deploy a versioned flow from GitHub to the NiFi canvas
+    - ``update_git_flow_ver``: Change version of an already-deployed Git-registry flow
+- **Registry client management**: ``ensure_registry_client`` and ``update_registry_client`` for idempotent registry configuration
+- **Enhanced revert**: ``revert_flow_ver`` now accepts ``wait=True`` parameter for synchronous operation
+
+**Profile System Enhancements**
+
+- **"env" profile for CI/CD**: New special profile name that configures nipyapi entirely from environment variables
+    - No profiles file required - ideal for GitHub Actions, containers, and CI/CD pipelines
+    - Uses ``nipyapi.profiles.switch('env')`` to activate
+    - All standard environment variable mappings (``NIFI_API_ENDPOINT``, ``NIFI_USERNAME``, etc.) apply
+
+**Controller Service Management**
+
+- **Bulk controller service operations**: ``schedule_all_controllers(pg_id, scheduled)`` to enable/disable all controller services in a process group
+    - Uses NiFi's native bulk activation API
+    - Handles all descendant controller services automatically
+    - Simplifies flow start/stop operations in CI/CD workflows
+
+**Bug Fixes**
+
+- Fixed ``test_create_controller`` leaving orphaned ADLS controller services after test runs
+- Improved test cleanup fixtures for better isolation
+- Fixed version sorting in ``deploy_git_registry_flow`` to correctly identify latest version
+
+**Infrastructure**
+
+- Updated ``pylintrc`` for pylint 3.x compatibility
+- Increased module line limit for versioning.py to accommodate new functions
+- Added comprehensive test fixtures for Git registry integration testing
+
+**Related Projects**
+
+- **nipyapi-actions**: Companion GitHub Action for NiFi CI/CD workflows (https://github.com/Chaffelson/nipyapi-actions)
+
 1.0.1 (2025-11-10)
 -------------------
 
@@ -195,41 +243,6 @@ History
 * If Client is not instantiated, optimistically instantiate for version checking
 * add socks proxy support
 
-0.16.3 (2021-10-11)
--------------------
-
-| Removed force reset of configuration.password and configuration.username to empty string. This was not increasing security, and was causing unexpected errors for users connecting to multiple services in a single script.
-| Add greedy control to versioning.get_registry_bucket and versioning.get_flow_in_bucket to avoid undesirable partial string match.
-
-* Update readme to reflect switch from 'master' branch naming to 'main'.
-* Update tox to pin testing to Python 3.8, as Python 3.9 is producing unexpected and unrelated SSL failures
-* Minor lint formatting improvements
-
-0.16.2 (2021-02-10)
--------------------
-
-| NOTE: If you are using secured Registry, this release will enforce access controls for the swagger interface which is used to determine which version of Registry is connected in order to correctly provide features - you may have to update your authorizations
-
-* Update requirements.txt to unpin future and lxml
-* Update lxml to 4.6.2 or newer to resolve vulnerability
-* Pin watchdog to <1.0.0 per their docs to maintain Python2.7 compatibility
-* Revert 0.14.3 changes to Authentication handling which introduced basicAuth support but resulted in some NiFi connections appearing incorrectly as Anonymous
-* Added simpler basicAuth control to force it via a config switch without changing tokenAuth and other Authorization header behavior during normal usage
-* nipyapi.config.global_force_basic_auth is now available for use for this purpose
-* Secured Registry users will now require the authorization policy to retrieve the swagger so we may use it to validate which version of
-* Registry is in use for feature enablement
-* Moved all Security controls in config.py to a common area at the foot of the file
-* Removed auth_type from security.service_login as it is now redundant
-* Added controls to handle certificate checking behavior which has become more strict in recently versions of Python3, ssl_verify and check_hostname are now handled
-* security.set_service_auth_token now has an explicit flag for ssl host checking as well
-* Fix oversight where improved model serialisation logic was not correctly applied to Registry
-* Removed unusused parameter refresh from parameters.update_parameter_context
-* Reduced unecessary complexity in utils.dump with no change in functionality
-* Updated client gen mustache templates to reflect refactored security and api client code
-* Minor linting and docstring and codestyle improvements
-* Set pyUp to ignore Watchdog as it must stay between versions to statisfy py2 and py3 compatibility
-* If Client is not instantiated, optimistically instantiate for version checking
-* add socks proxy support
 
 0.15.0 (2020-11-06)
 -------------------

docs/profiles.rst

@@ -37,6 +37,8 @@ Quick Start
 - ``secure-ldap`` - LDAP authentication over TLS
 - ``secure-mtls`` - Mutual TLS certificate authentication
 - ``secure-oidc`` - OpenID Connect (OAuth2) authentication
+- ``bearer-token`` - Pre-obtained JWT bearer token
+- ``env`` - Pure environment variable configuration (no profiles file required)
 
 Why Use Profiles?
 =================
@@ -87,6 +89,7 @@ Default profiles are defined in ``examples/profiles.yml`` (JSON is also supporte
       # Authentication credentials
       nifi_user: "einstein"
       nifi_pass: "password1234"
+      nifi_bearer_token: null
       registry_user: "einstein"
       registry_pass: "password1234"
 
@@ -132,6 +135,7 @@ Core connection settings:
 
 Authentication credentials:
   - ``nifi_user`` / ``nifi_pass`` - NiFi Basic authentication credentials
+  - ``nifi_bearer_token`` - Pre-obtained JWT bearer token for NiFi
   - ``registry_user`` / ``registry_pass`` - Registry Basic authentication credentials
 
 Shared SSL/TLS certificates (simple PKI - convenience options where both NiFi and Registry share configuration):
@@ -154,7 +158,7 @@ Advanced settings:
   - ``nifi_proxy_identity`` - Identity for NiFi → Registry proxied requests
 
 Authentication method control:
-  - ``nifi_auth_method`` - Explicit authentication method for NiFi (overrides auto-detection). Valid values: ``oidc``, ``mtls``, ``basic``
+  - ``nifi_auth_method`` - Explicit authentication method for NiFi (overrides auto-detection). Valid values: ``bearer``, ``oidc``, ``mtls``, ``basic``
   - ``registry_auth_method`` - Explicit authentication method for Registry (overrides auto-detection). Valid values: ``mtls``, ``basic``, ``unauthenticated``
 
 OIDC authentication:
@@ -169,7 +173,7 @@ Profile Switching Behavior
 1. **Explicit method specification** (highest priority): If ``nifi_auth_method`` or ``registry_auth_method`` are set, that method is used regardless of other available credentials.
 2. **Auto-detection** (fallback): When no explicit method is specified, the system auto-detects based on available credentials. Detection order varies by service:
 
-   - **NiFi**: **1) OIDC** (``oidc_token_endpoint``), **2) mTLS** (``client_cert`` + ``client_key``), **3) Basic Auth** (``nifi_user`` + ``nifi_pass``)
+   - **NiFi**: **1) Bearer Token** (``nifi_bearer_token``), **2) OIDC** (``oidc_token_endpoint``), **3) mTLS** (``client_cert`` + ``client_key``), **4) Basic Auth** (``nifi_user`` + ``nifi_pass``)
    - **Registry**: **1) mTLS** (``client_cert`` + ``client_key``), **2) Basic Auth** (``registry_user`` + ``registry_pass``), **3) Unauthenticated** (no credentials required)
 
 For predictable behavior, either use explicit method specification or design profiles with only one authentication method per environment.
@@ -317,6 +321,37 @@ OpenID Connect (OAuth2) authentication:
 
 **Use case**: Modern OAuth2 integration, external identity providers
 
+bearer-token (Simplest Configuration)
+--------------------------------------
+
+Pre-obtained JWT bearer token authentication - the simplest possible configuration:
+
+.. code-block:: python
+
+    nipyapi.profiles.switch('bearer-token')
+
+**Authentication method**: Bearer Token (detected by presence of ``nifi_bearer_token``)
+
+**Required properties**:
+  - ``nifi_bearer_token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..."`` (your JWT token)
+
+**Additional properties used**:
+  - ``nifi_url: https://nifi.example.com/nifi-api``
+  - ``nifi_verify_ssl: true`` (or false for self-signed certificates)
+
+**Use case**: CI/CD pipelines, GitHub Actions, Kubernetes jobs, or any scenario where you have a pre-obtained JWT token from your identity provider. This is the simplest authentication method - just a URL and a token.
+
+**Example using environment variables only**:
+
+.. code-block:: shell
+
+    # Set environment variables
+    export NIFI_API_ENDPOINT=https://nifi.production.example.com/nifi-api
+    export NIFI_BEARER_TOKEN=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9...
+
+    # Use the 'env' profile to load from environment
+    python -c "import nipyapi; nipyapi.profiles.switch('env')"
+
 cli-properties
 --------------
 
@@ -342,6 +377,34 @@ NiFi CLI properties file integration with OIDC Client Credentials flow:
 
 **Use case**: Integration with existing NiFi CLI configurations, client credentials OIDC
 
+env (Environment-Only Configuration)
+------------------------------------
+
+Pure environment variable configuration without requiring a profiles file:
+
+.. code-block:: python
+
+    nipyapi.profiles.switch('env')
+
+**Authentication method**: Auto-detected from environment variables
+
+**Required environment variables** (minimum for NiFi connection):
+  - ``NIFI_API_ENDPOINT`` - NiFi API URL
+  - ``NIFI_USERNAME`` - NiFi username (for basic auth)
+  - ``NIFI_PASSWORD`` - NiFi password (for basic auth)
+
+**Optional environment variables**:
+  - ``REGISTRY_API_ENDPOINT`` - Registry API URL
+  - ``REGISTRY_USERNAME`` / ``REGISTRY_PASSWORD`` - Registry credentials
+  - ``TLS_CA_CERT_PATH`` - CA certificate path
+  - ``MTLS_CLIENT_CERT`` / ``MTLS_CLIENT_KEY`` - mTLS certificates
+  - ``OIDC_TOKEN_ENDPOINT`` / ``OIDC_CLIENT_ID`` / ``OIDC_CLIENT_SECRET`` - OIDC configuration
+  - All other environment variables listed in the Environment Variable Overrides section
+
+**Use case**: CI/CD pipelines, containerized deployments, GitHub Actions, any environment where configuration is injected via environment variables rather than files.
+
+**Key benefit**: No profiles file needed. The ``env`` profile starts with all-null defaults and populates values entirely from environment variables using the standard ``ENV_VAR_MAPPINGS``. This keeps secrets out of files and allows dynamic configuration in deployment environments.
+
 Environment Variable Overrides
 ===============================
 
@@ -381,6 +444,7 @@ URLs and credentials:
   - ``REGISTRY_API_ENDPOINT`` → ``registry_url``
   - ``NIFI_USERNAME`` → ``nifi_user``
   - ``NIFI_PASSWORD`` → ``nifi_pass``
+  - ``NIFI_BEARER_TOKEN`` → ``nifi_bearer_token``
   - ``REGISTRY_USERNAME`` → ``registry_user``
   - ``REGISTRY_PASSWORD`` → ``registry_pass``
 
@@ -504,6 +568,12 @@ You don't have to specify values that are otherwise null unless required for tha
 
 You don't have to use profiles. You can also directly configure the NiPyAPI client using ``nipyapi.config`` and ``nipyapi.utils.set_endpoint()`` as shown in earlier examples.
 
+GitHub Flow Registry Client
+===========================
+
+NiFi 2.x supports versioning flows directly to GitHub repositories using the GitHub Flow Registry Client.
+For CI/CD workflows with GitHub Actions, see the `nipyapi-actions <https://github.com/Chaffelson/nipyapi-actions>`_ repository which provides reusable actions for deploying NiFi flows from Git repositories.
+
 Integration with Examples
 =========================
 

examples/profiles.yml

@@ -85,6 +85,26 @@ secure-oidc:
   oidc_client_id: "nipyapi-client"
   oidc_client_secret: "nipyapi-secret"
 
+# Bearer token profile - simplest possible configuration
+# Use when you have a pre-obtained JWT token (e.g., from CI/CD, identity provider)
+# This is ideal for GitHub Actions, Kubernetes jobs, or any environment with injected tokens
+bearer-token:
+  nifi_url: https://nifi.example.com/nifi-api
+  nifi_bearer_token: null  # Set via NIFI_BEARER_TOKEN environment variable
+  nifi_verify_ssl: true
+
+# GitHub CI/CD profile - NiFi only, for testing Git-based registry workflows
+# Uses GitHub Registry Client instead of NiFi Registry
+# GitHub registry configuration is handled by nipyapi-actions, not nipyapi profiles
+github-cicd:
+  nifi_url: https://localhost:9447/nifi-api
+  nifi_user: einstein
+  nifi_pass: password1234
+  nifi_verify_ssl: false
+  suppress_ssl_warnings: true
+  # No NiFi Registry - using GitHub as flow registry
+  registry_url: null
+
 # Example profile using NiFi CLI properties file integration
 cli-properties:
   # Load configuration from existing NiFi CLI properties file