feat: Add GitHub CI/CD versioning support with env profile

.github/workflows/ci.yml

@@ -39,6 +39,8 @@ jobs:
       run: make wait-ready NIPYAPI_PROFILE=single-user
 
     - name: Run integration tests with coverage
+      env:
+        GH_REGISTRY_TOKEN: ${{ secrets.GH_REGISTRY_TOKEN }}
       run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest --cov=nipyapi --cov-report=xml --cov-report=term-missing
 
     - name: Upload coverage to Codecov

.gitignore

@@ -190,3 +190,4 @@ htmlcov/
 
 # setuptools-scm generated version file
 nipyapi/_version.py
+.cursor/

Makefile

@@ -4,6 +4,10 @@
 # Default NiFi/Registry version for docker compose profiles
 NIFI_VERSION ?= 2.6.0
 
+# Load .env file if it exists (for secrets like GH_REGISTRY_TOKEN)
+-include .env
+export
+
 # Python command for cross-platform compatibility
 # Defaults to 'python' for conda/venv users, override with PYTHON=python3 for system installs
 PYTHON ?= python
@@ -227,17 +231,17 @@ extract-jks: ## extract PEM certificates from JKS: make extract-jks JKS_FILE=tru
 	fi
 	@echo "✅ Certificates extracted to resources/certs/extracted/"
 
-up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc (uses NIFI_VERSION=$(NIFI_VERSION))
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd (uses NIFI_VERSION=$(NIFI_VERSION))
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	$(DC) --profile $(NIPYAPI_PROFILE) up -d
 
 down: ## bring down all docker services
 	@echo "Bringing down Docker services (NIFI_VERSION=$(NIFI_VERSION))"
-	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc down -v --remove-orphans || true
+	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc --profile github-cicd down -v --remove-orphans || true
 	@echo "Verifying expected containers are stopped/removed:"
 	@COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) NIFI_VERSION=$(NIFI_VERSION) docker compose -f $(COMPOSE_FILE) ps --format "table {{.Name}}\t{{.State}}" | tail -n +2 | awk '{print " - " $$1 ": " $$2}' || true
 
-wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
+wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
 	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required"; exit 1; fi
 	@echo "Waiting for $(NIPYAPI_PROFILE) infrastructure to be ready..."
 	@NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) $(PYTHON) resources/scripts/wait_ready.py
@@ -269,7 +273,7 @@ gen-clients: ## generate NiFi and Registry clients from specs (use wv_spec_varia
 # Individual testing
 
 test: ## run pytest with provided NIPYAPI_PROFILE; config resolved by tests/conftest.py
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q
 
 test-su: ## shortcut: NIPYAPI_PROFILE=single-user pytest
@@ -285,7 +289,7 @@ test-oidc: check-certs ## shortcut: NIPYAPI_PROFILE=secure-oidc pytest (requires
 	NIPYAPI_PROFILE=secure-oidc $(MAKE) test
 
 test-specific: ## run specific pytest with provided NIPYAPI_PROFILE and TEST_ARGS
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	if [ -z "$(TEST_ARGS)" ]; then echo "TEST_ARGS is required (e.g., tests/test_utils.py::test_dump -v)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q $(TEST_ARGS)
 
@@ -333,8 +337,8 @@ test-all: ensure-certs ## run full e2e tests across automated profiles (requires
 	done
 	@echo "All profiles tested successfully"
 
-sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	@echo "🏗️ Setting up NiPyAPI sandbox with profile: $(NIPYAPI_PROFILE)"
 	@echo "=== 1/4: Starting infrastructure ==="
 	$(MAKE) up NIPYAPI_PROFILE=$(NIPYAPI_PROFILE)

docs/history.rst

@@ -2,6 +2,54 @@
 History
 =======
 
+1.1.0 (2025-12-04)
+-------------------
+
+| GitHub CI/CD Integration - Native support for NiFi's GitHub Flow Registry Client
+
+**GitHub Flow Registry Support**
+
+- **Git-specific versioning helpers**: New functions to work with NiFi's native GitHub Flow Registry Client
+    - ``list_git_registry_buckets``: List buckets (folders) in a Git-backed registry
+    - ``get_git_registry_bucket``: Get a specific bucket by name
+    - ``list_git_registry_flows``: List flows in a bucket
+    - ``get_git_registry_flow``: Get a specific flow by name
+    - ``list_git_registry_flow_versions``: List all versions (commits) of a flow
+    - ``deploy_git_registry_flow``: Deploy a versioned flow from GitHub to the NiFi canvas
+    - ``update_git_flow_ver``: Change version of an already-deployed Git-registry flow
+- **Registry client management**: ``ensure_registry_client`` and ``update_registry_client`` for idempotent registry configuration
+- **Enhanced revert**: ``revert_flow_ver`` now accepts ``wait=True`` parameter for synchronous operation
+
+**Profile System Enhancements**
+
+- **"env" profile for CI/CD**: New special profile name that configures nipyapi entirely from environment variables
+    - No profiles file required - ideal for GitHub Actions, containers, and CI/CD pipelines
+    - Uses ``nipyapi.profiles.switch('env')`` to activate
+    - All standard environment variable mappings (``NIFI_API_ENDPOINT``, ``NIFI_USERNAME``, etc.) apply
+
+**Controller Service Management**
+
+- **Bulk controller service operations**: ``schedule_all_controllers(pg_id, scheduled)`` to enable/disable all controller services in a process group
+    - Uses NiFi's native bulk activation API
+    - Handles all descendant controller services automatically
+    - Simplifies flow start/stop operations in CI/CD workflows
+
+**Bug Fixes**
+
+- Fixed ``test_create_controller`` leaving orphaned ADLS controller services after test runs
+- Improved test cleanup fixtures for better isolation
+- Fixed version sorting in ``deploy_git_registry_flow`` to correctly identify latest version
+
+**Infrastructure**
+
+- Updated ``pylintrc`` for pylint 3.x compatibility
+- Increased module line limit for versioning.py to accommodate new functions
+- Added comprehensive test fixtures for Git registry integration testing
+
+**Related Projects**
+
+- **nipyapi-actions**: Companion GitHub Action for NiFi CI/CD workflows (https://github.com/Chaffelson/nipyapi-actions)
+
 1.0.1 (2025-11-10)
 -------------------
 
@@ -195,41 +243,6 @@ History
 * If Client is not instantiated, optimistically instantiate for version checking
 * add socks proxy support
 
-0.16.3 (2021-10-11)
--------------------
-
-| Removed force reset of configuration.password and configuration.username to empty string. This was not increasing security, and was causing unexpected errors for users connecting to multiple services in a single script.
-| Add greedy control to versioning.get_registry_bucket and versioning.get_flow_in_bucket to avoid undesirable partial string match.
-
-* Update readme to reflect switch from 'master' branch naming to 'main'.
-* Update tox to pin testing to Python 3.8, as Python 3.9 is producing unexpected and unrelated SSL failures
-* Minor lint formatting improvements
-
-0.16.2 (2021-02-10)
--------------------
-
-| NOTE: If you are using secured Registry, this release will enforce access controls for the swagger interface which is used to determine which version of Registry is connected in order to correctly provide features - you may have to update your authorizations
-
-* Update requirements.txt to unpin future and lxml
-* Update lxml to 4.6.2 or newer to resolve vulnerability
-* Pin watchdog to <1.0.0 per their docs to maintain Python2.7 compatibility
-* Revert 0.14.3 changes to Authentication handling which introduced basicAuth support but resulted in some NiFi connections appearing incorrectly as Anonymous
-* Added simpler basicAuth control to force it via a config switch without changing tokenAuth and other Authorization header behavior during normal usage
-* nipyapi.config.global_force_basic_auth is now available for use for this purpose
-* Secured Registry users will now require the authorization policy to retrieve the swagger so we may use it to validate which version of
-* Registry is in use for feature enablement
-* Moved all Security controls in config.py to a common area at the foot of the file
-* Removed auth_type from security.service_login as it is now redundant
-* Added controls to handle certificate checking behavior which has become more strict in recently versions of Python3, ssl_verify and check_hostname are now handled
-* security.set_service_auth_token now has an explicit flag for ssl host checking as well
-* Fix oversight where improved model serialisation logic was not correctly applied to Registry
-* Removed unusused parameter refresh from parameters.update_parameter_context
-* Reduced unecessary complexity in utils.dump with no change in functionality
-* Updated client gen mustache templates to reflect refactored security and api client code
-* Minor linting and docstring and codestyle improvements
-* Set pyUp to ignore Watchdog as it must stay between versions to statisfy py2 and py3 compatibility
-* If Client is not instantiated, optimistically instantiate for version checking
-* add socks proxy support
 
 0.15.0 (2020-11-06)
 -------------------

docs/profiles.rst

@@ -37,6 +37,7 @@ Quick Start
 - ``secure-ldap`` - LDAP authentication over TLS
 - ``secure-mtls`` - Mutual TLS certificate authentication
 - ``secure-oidc`` - OpenID Connect (OAuth2) authentication
+- ``env`` - Pure environment variable configuration (no profiles file required)
 
 Why Use Profiles?
 =================
@@ -342,6 +343,34 @@ NiFi CLI properties file integration with OIDC Client Credentials flow:
 
 **Use case**: Integration with existing NiFi CLI configurations, client credentials OIDC
 
+env (Environment-Only Configuration)
+------------------------------------
+
+Pure environment variable configuration without requiring a profiles file:
+
+.. code-block:: python
+
+    nipyapi.profiles.switch('env')
+
+**Authentication method**: Auto-detected from environment variables
+
+**Required environment variables** (minimum for NiFi connection):
+  - ``NIFI_API_ENDPOINT`` - NiFi API URL
+  - ``NIFI_USERNAME`` - NiFi username (for basic auth)
+  - ``NIFI_PASSWORD`` - NiFi password (for basic auth)
+
+**Optional environment variables**:
+  - ``REGISTRY_API_ENDPOINT`` - Registry API URL
+  - ``REGISTRY_USERNAME`` / ``REGISTRY_PASSWORD`` - Registry credentials
+  - ``TLS_CA_CERT_PATH`` - CA certificate path
+  - ``MTLS_CLIENT_CERT`` / ``MTLS_CLIENT_KEY`` - mTLS certificates
+  - ``OIDC_TOKEN_ENDPOINT`` / ``OIDC_CLIENT_ID`` / ``OIDC_CLIENT_SECRET`` - OIDC configuration
+  - All other environment variables listed in the Environment Variable Overrides section
+
+**Use case**: CI/CD pipelines, containerized deployments, GitHub Actions, any environment where configuration is injected via environment variables rather than files.
+
+**Key benefit**: No profiles file needed. The ``env`` profile starts with all-null defaults and populates values entirely from environment variables using the standard ``ENV_VAR_MAPPINGS``. This keeps secrets out of files and allows dynamic configuration in deployment environments.
+
 Environment Variable Overrides
 ===============================
 
@@ -504,6 +533,12 @@ You don't have to specify values that are otherwise null unless required for tha
 
 You don't have to use profiles. You can also directly configure the NiPyAPI client using ``nipyapi.config`` and ``nipyapi.utils.set_endpoint()`` as shown in earlier examples.
 
+GitHub Flow Registry Client
+===========================
+
+NiFi 2.x supports versioning flows directly to GitHub repositories using the GitHub Flow Registry Client.
+For CI/CD workflows with GitHub Actions, see the `nipyapi-actions <https://github.com/Chaffelson/nipyapi-actions>`_ repository which provides reusable actions for deploying NiFi flows from Git repositories.
+
 Integration with Examples
 =========================
 

examples/profiles.yml

@@ -85,6 +85,18 @@ secure-oidc:
   oidc_client_id: "nipyapi-client"
   oidc_client_secret: "nipyapi-secret"
 
+# GitHub CI/CD profile - NiFi only, for testing Git-based registry workflows
+# Uses GitHub Registry Client instead of NiFi Registry
+# GitHub registry configuration is handled by nipyapi-actions, not nipyapi profiles
+github-cicd:
+  nifi_url: https://localhost:9447/nifi-api
+  nifi_user: einstein
+  nifi_pass: password1234
+  nifi_verify_ssl: false
+  suppress_ssl_warnings: true
+  # No NiFi Registry - using GitHub as flow registry
+  registry_url: null
+
 # Example profile using NiFi CLI properties file integration
 cli-properties:
   # Load configuration from existing NiFi CLI properties file

nipyapi/canvas.py

@@ -46,6 +46,7 @@
     "delete_controller",
     "update_controller",
     "schedule_controller",
+    "schedule_all_controllers",
     "get_controller",
     "list_all_controller_types",
     "list_all_by_kind",
@@ -734,36 +735,80 @@ def update_process_group(pg, update, refresh=True):
         )
 
 
-def update_processor(processor, update, refresh=True):
+def update_processor(processor, update=None, name=None, refresh=True, auto_stop=False):
     """
-    Updates configuration parameters for a given Processor.
+    Updates a Processor's configuration and/or name.
 
-    An example update would be:
-    nifi.ProcessorConfigDTO(scheduling_period='3s')
+    For configuration changes, pass a ProcessorConfigDTO:
+        nifi.ProcessorConfigDTO(scheduling_period='3s')
+
+    For renaming, pass the new name. Both can be provided together.
+
+    Processors must be stopped for certain updates (including renaming).
+    If auto_stop is True (default), the processor will be stopped before
+    updating and restarted afterward if it was originally running.
 
     Args:
         processor (ProcessorEntity): The Processor to target for update
-        update (ProcessorConfigDTO): The new configuration parameters
+        update (ProcessorConfigDTO, optional): Configuration parameters to update
+        name (str, optional): New name for the processor
         refresh (bool): Whether to refresh the Processor object state
-          before applying the update
+            before applying the update. Default True.
+        auto_stop (bool): If True, automatically stop the processor before
+            updating and restart afterward if it was running. Default False.
 
     Returns:
         :class:`~nipyapi.nifi.models.ProcessorEntity`: The updated ProcessorEntity
 
+    Raises:
+        ValueError: If neither update nor name is provided, or if update is not
+            a ProcessorConfigDTO, or if processor is running and auto_stop=False.
     """
-    if not isinstance(update, nipyapi.nifi.ProcessorConfigDTO):
+    if update is None and name is None:
+        raise ValueError("Must provide 'update' (ProcessorConfigDTO) and/or 'name'")
+    if update is not None and not isinstance(update, nipyapi.nifi.ProcessorConfigDTO):
         raise ValueError("update param is not an instance of nifi.ProcessorConfigDTO")
+
     with nipyapi.utils.rest_exceptions():
         if refresh:
             processor = get_processor(processor.id, "id")
-        return nipyapi.nifi.ProcessorsApi().update_processor(
+
+        was_running = processor.component.state == "RUNNING"
+
+        if was_running and not auto_stop:
+            raise ValueError(
+                f"Processor '{processor.component.name}' is running. "
+                "Stop it first or set auto_stop=True."
+            )
+
+        # Stop if running
+        if was_running:
+            schedule_processor(processor, scheduled=False, refresh=True)
+            processor = get_processor(processor.id, "id")
+
+        # Build the update DTO with whatever fields are provided
+        dto_kwargs = {"id": processor.component.id}
+        if name is not None:
+            dto_kwargs["name"] = name
+        if update is not None:
+            dto_kwargs["config"] = update
+
+        result = nipyapi.nifi.ProcessorsApi().update_processor(
             id=processor.id,
             body=nipyapi.nifi.ProcessorEntity(
-                component=nipyapi.nifi.ProcessorDTO(config=update, id=processor.id),
+                id=processor.id,
                 revision=processor.revision,
+                component=nipyapi.nifi.ProcessorDTO(**dto_kwargs),
             ),
         )
 
+        # Restart if it was running
+        if was_running:
+            schedule_processor(result, scheduled=True, refresh=True)
+            result = get_processor(result.id, "id")
+
+        return result
+
 
 def get_variable_registry(process_group, ancestors=True):
     """
@@ -1216,6 +1261,33 @@ def _schedule_controller_state(cont_id, tgt_state):
     raise ValueError("Scheduling request timed out")
 
 
+def schedule_all_controllers(pg_id, scheduled):
+    """
+    Enable or Disable all Controller Services in a Process Group.
+
+    Uses NiFi's native bulk controller service activation API which handles
+    all descendant controller services automatically.
+
+    Args:
+        pg_id (str): The UUID of the Process Group
+        scheduled (bool): True to enable, False to disable
+
+    Returns:
+        ActivateControllerServicesEntity: The result of the operation
+
+    """
+    assert isinstance(pg_id, str)
+    assert isinstance(scheduled, bool)
+
+    target_state = "ENABLED" if scheduled else "DISABLED"
+
+    with nipyapi.utils.rest_exceptions():
+        return nipyapi.nifi.FlowApi().activate_controller_services(
+            id=pg_id,
+            body=nipyapi.nifi.ActivateControllerServicesEntity(id=pg_id, state=target_state),
+        )
+
+
 def get_controller(
     identifier, identifier_type="name", bool_response=False, include_reporting_tasks=True
 ):

nipyapi/config.py

@@ -69,6 +69,13 @@
     "ControllerServiceEntity": {"id": ["id"], "name": ["component", "name"]},
     "ParameterContextEntity": {"id": ["id"], "name": ["component", "name"]},
     "ReportingTaskEntity": {"id": ["id"], "name": ["component", "name"]},
+    # Git-based Flow Registry types (GitHub, GitLab, Bitbucket, Azure DevOps)
+    "FlowRegistryBucketEntity": {"id": ["id"], "name": ["bucket", "name"]},
+    "VersionedFlowEntity": {
+        "flow_id": ["versioned_flow", "flow_id"],
+        "flow_name": ["versioned_flow", "flow_name"],
+    },
+    "VersionedFlowDTO": {"flow_id": ["flow_id"], "flow_name": ["flow_name"]},
 }