feat(plumber): retention policy worker

plumber/AGENTS.md

@@ -0,0 +1,20 @@
+# Plumber Repository Guidelines
+
+## Layout
+- `ppl/` hosts the pipeline orchestrator (Elixir/OTP application) together with its GRPC endpoints and database migrations.
+- `block/` implements block/task execution logic and is started as a sibling OTP app inside the same release.
+- `proto/`, `spec/`, and other top-level folders expose shared protobuf contracts, YAML schemas, and helper libraries.
+
+## Build & Test
+- Run `mix deps.get` inside both `ppl/` and `block/` after cloning; each app keeps its own `mix.lock`.
+- Use the provided `Makefile` targets (e.g., `make unit-test`) to spin up Postgres/RabbitMQ containers and execute the umbrella test suites.
+- Database migrations live under `ppl/priv/ecto_repo/migrations` and `block/priv/ecto_repo/migrations`; run them with `mix ecto.migrate -r <Repo>`.
+- All new code must pass `mix credo --strict`, `mix format`, and the relevant `mix test` suites before submitting.
+
+## Development Notes
+- Services rely on RabbitMQ (`RABBITMQ_URL`) for event streams; keep it running locally when exercising background workers (e.g., the retention policy consumer).
+- Watchman (StatsD) is the default metrics sink; configure `METRICS_HOST`/`METRICS_PORT` for local debugging if needed.
+
+## Documentation
+- Service-specific guidance lives under `ppl/AGENTS.md` and `block/AGENTS.md`.
+- Architectural notes, including retention policies, are documented in `docs/` (see `docs/pipeline_retention.md` for the event-driven marking flow).

plumber/block/AGENTS.md

@@ -0,0 +1,19 @@
+# Block Service Guidelines
+
+## Project Layout
+- `lib/block/` contains block request processing, task orchestration, and API clients.
+- `priv/ecto_repo/migrations/` stores migrations for `Block.EctoRepo`.
+- `config/` defines repo settings, looper timings, and runtime overrides.
+- Tests live under `test/`; shared helpers reside in `test/support`.
+
+## Setup & Commands
+- Install deps: `mix deps.get`.
+- Migrate DB: `mix ecto.migrate -r Block.EctoRepo`.
+- Run unit tests: `mix test` (requires Postgres running; use the root Makefile to spin up containers).
+- Format/lint: `mix format`, `mix credo --strict`.
+
+## Integration Notes
+- Block is started as part of the Plumber release; ensure both repos (`Ppl.EctoRepo` and `Block.EctoRepo`) are migrated.
+- RabbitMQ (`RABBITMQ_URL`) is required for task lifecycle events; keep it reachable during development.
+- When pipelines are deleted (e.g., via the retention worker) `Block.delete_blocks_from_ppl/1` is invoked to purge block-state tables—be mindful of this coupling when making schema changes.
+

plumber/ppl/AGENTS.md

@@ -0,0 +1,22 @@
+# Ppl Service Guidelines
+
+## Project Layout
+- `lib/ppl/` contains the OTP application code (workers, GRPC servers, caches, etc.).
+- `priv/ecto_repo/migrations/` holds database migrations for `Ppl.EctoRepo`.
+- `config/` hosts the environment-specific config (runtime DB credentials, Watchman, RabbitMQ consumers).
+- `test/` mirrors `lib/` with ExUnit suites; helpers live in `test/support`.
+
+## Common Tasks
+- Install deps: `mix deps.get`.
+- Run migrations: `mix ecto.migrate -r Ppl.EctoRepo`.
+- Launch tests: `mix test` (set `MIX_ENV=test`, and start Postgres/RabbitMQ via the repo `Makefile` if not already running).
+- Format & lint: `mix format`, `mix credo --strict`.
+
+## Background Workers
+- Looper STMs and Beholders are supervised via `Ppl.Application`.
+- `Ppl.Retention.PolicyConsumer` (Tackle) listens for `usage.OrganizationPolicyApply` events and marks pipelines by setting `expires_at`.
+
+## Configuration Tips
+- StatsD via Watchman: set `METRICS_HOST`, `METRICS_PORT`, `METRICS_NAMESPACE`.
+- DB settings: `POSTGRES_DB_*` env vars control `Ppl.EctoRepo`; separate vars govern the `block` repo.
+- Retention consumer: `USAGE_POLICY_EXCHANGE` / `USAGE_POLICY_ROUTING_KEY` configure which RabbitMQ route the policy consumer listens to.

plumber/ppl/config/config.exs

@@ -43,6 +43,11 @@ config :gofer_client, gofer_grpc_timeout: 4_567
 # Time in hours before delete requests are processed
 config :ppl, deletion_offset_h: 24
 
+# Retention policy event consumer
+config :ppl, Ppl.Retention.PolicyConsumer,
+  exchange: System.get_env("USAGE_POLICY_EXCHANGE"),
+  routing_key: System.get_env("USAGE_POLICY_ROUTING_KEY")
+
 # How many times should wormhole retry to publish pipeline events to RabbitMQ
 config :ppl, publish_retry_count: 3
 # Timeout for publishing pipeline events to RabbitMQ

plumber/ppl/config/test.exs

@@ -47,6 +47,10 @@ config :ppl, dr_pending_sp: 100
 config :ppl, dr_deleting_sp: 100
 config :ppl, dr_queue_deleting_sp: 100
 
+config :ppl, Ppl.Retention.PolicyConsumer,
+  exchange: "usage_internal_api_test",
+  routing_key: "usage.apply_organization_policy.test"
+
 config :watchman,
   host: "localhost",
   port: 8125,

plumber/ppl/lib/ppl/application.ex

@@ -52,6 +52,7 @@ defmodule Ppl.Application do
     [
       Ppl.Sup.STM,
       worker(Ppl.OrgEventsConsumer, []),
+      worker(Ppl.Retention.PolicyConsumer, [])
     ]
   end
 

plumber/ppl/lib/ppl/ppl_requests/model/ppl_requests.ex

@@ -23,6 +23,7 @@ defmodule Ppl.PplRequests.Model.PplRequests do
     field :wf_id, :string
     field :source_args, :map
     field :pre_flight_checks, :map
+    field :expires_at, :naive_datetime_usec
 
     timestamps(type: :naive_datetime_usec)
   end

plumber/ppl/lib/ppl/retention/policy_applier.ex

@@ -0,0 +1,29 @@
+defmodule Ppl.Retention.PolicyApplier do
+  @moduledoc """
+  Marks pipelines for expiration when an organization policy is applied.
+  """
+
+  import Ecto.Query
+
+  alias Ppl.EctoRepo
+  alias Ppl.PplRequests.Model.PplRequests
+
+  @doc """
+  Sets `expires_at` on all pipelines for `org_id` inserted before `cutoff`.
+  Pipelines already marked with an earlier or equal expiration are left untouched.
+  Returns the number of rows affected.
+  """
+  @spec mark_expiring(String.t(), NaiveDateTime.t()) :: non_neg_integer()
+  def mark_expiring(org_id, cutoff) do
+    query =
+      from(pr in PplRequests,
+        where: fragment("?->>?", pr.request_args, "organization_id") == ^org_id,
+        where: pr.inserted_at < ^cutoff,
+        where: is_nil(pr.expires_at) or pr.expires_at > ^cutoff
+      )
+
+    {count, _} = EctoRepo.update_all(query, set: [expires_at: ^cutoff])
+    count
+  end
+end
+

plumber/ppl/lib/ppl/retention/policy_consumer.ex

@@ -0,0 +1,55 @@
+defmodule Ppl.Retention.PolicyConsumer do
+  @moduledoc """
+  Subscribes to usage.ApplyOrganizationPolicyEvent and marks pipelines for expiration.
+  """
+
+  @consumer_opts Application.compile_env(:ppl, __MODULE__, [])
+
+  use Tackle.Consumer,
+    url: System.get_env("RABBITMQ_URL"),
+    exchange: Keyword.get(@consumer_opts, :exchange, "usage_internal_api"),
+    routing_key: Keyword.get(@consumer_opts, :routing_key, "usage.apply_organization_policy"),
+    service: "plumber-retention"
+
+  require Logger
+
+  alias Google.Protobuf.Timestamp
+  alias InternalApi.Usage.OrganizationPolicyApply
+  alias Ppl.Retention.PolicyApplier
+
+  def handle_message(message) do
+    with {:ok, event} <- decode(message),
+         {:ok, cutoff} <- cutoff_to_naive(event.cutoff_date),
+         {:ok, org_id} <- non_empty(event.org_id) do
+      count = PolicyApplier.mark_expiring(org_id, cutoff)
+      Watchman.increment({"retention.marked", [org_id]})
+      Logger.info("[Retention] Marked #{count} pipelines for org #{org_id} until #{cutoff}")
+    else
+      {:error, reason} ->
+        Logger.error("[Retention] Failed to apply policy event: #{inspect(reason)}")
+    end
+  end
+
+  defp decode(message) do
+    OrganizationPolicyApply.decode(message)
+    |> case do
+      %OrganizationPolicyApply{} = event -> {:ok, event}
+      other -> {:error, {:unexpected_payload, other}}
+    end
+  rescue
+    e -> {:error, e}
+  end
+
+  defp cutoff_to_naive(%Timestamp{seconds: 0, nanos: 0}), do: {:error, :missing_cutoff}
+
+  defp cutoff_to_naive(%Timestamp{seconds: seconds, nanos: nanos}) do
+    {:ok, datetime} = DateTime.from_unix(seconds, :second)
+    micros = div(nanos, 1_000)
+    {:ok, datetime |> DateTime.add(micros, :microsecond) |> DateTime.to_naive()}
+  end
+
+  defp cutoff_to_naive(nil), do: {:error, :missing_cutoff}
+
+  defp non_empty(value) when value in [nil, ""], do: {:error, :missing_org_id}
+  defp non_empty(value), do: {:ok, value}
+end

plumber/ppl/priv/ecto_repo/migrations/20250224121500_add_expires_at_to_pipeline_requests.exs

@@ -0,0 +1,11 @@
+defmodule Ppl.EctoRepo.Migrations.AddExpiresAtToPipelineRequests do
+  use Ecto.Migration
+
+  def change do
+    alter table(:pipeline_requests) do
+      add :expires_at, :naive_datetime_usec
+    end
+
+    create index(:pipeline_requests, [:created_at, :expires_at], name: :idx_pipeline_requests_created_at_expires_at_not_null, concurrently: true, where: "expires_at IS NOT NULL")
+  end
+end

plumber/proto/Makefile

@@ -48,6 +48,7 @@ endif
 	docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/artifacthub.proto
 	docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/health.proto
 	docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/pre_flight_checks_hub.proto
+	docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/usage.proto
 	rm -rf $(TMP_INTERNAL_REPO_DIR)
 
 console:

plumber/proto/lib/internal_api/artifacthub.pb.ex

@@ -37,17 +37,23 @@ defmodule InternalApi.Artifacthub.RetentionPolicy do
     :last_cleaned_at
   ]
 
-  field(:project_level_retention_policies, 1,
+  field(
+    :project_level_retention_policies,
+    1,
     repeated: true,
     type: InternalApi.Artifacthub.RetentionPolicy.RetentionPolicyRule
   )
 
-  field(:workflow_level_retention_policies, 2,
+  field(
+    :workflow_level_retention_policies,
+    2,
     repeated: true,
     type: InternalApi.Artifacthub.RetentionPolicy.RetentionPolicyRule
   )
 
-  field(:job_level_retention_policies, 3,
+  field(
+    :job_level_retention_policies,
+    3,
     repeated: true,
     type: InternalApi.Artifacthub.RetentionPolicy.RetentionPolicyRule
   )
@@ -281,7 +287,9 @@ defmodule InternalApi.Artifacthub.ListBucketsResponse do
         }
   defstruct [:bucket_names_for_ids]
 
-  field(:bucket_names_for_ids, 1,
+  field(
+    :bucket_names_for_ids,
+    1,
     repeated: true,
     type: InternalApi.Artifacthub.ListBucketsResponse.BucketNamesForIdsEntry,
     map: true