diff --git a/AGENTS.md b/AGENTS.md index 690c9c9..fdaa84b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ lfx-v1-sync-helper/ │ ├── meltano.yml # Main Meltano configuration │ └── load/target-nats-kv/ # Custom NATS KV target plugin ├── cmd/lfx-v1-sync-helper/ # Go microservice source -├── charts/lfx-v1-sync-helper/ # Helm deployment charts +├── charts/lfx-v1-sync-helper/ # Helm deployment charts (Chart.yaml version is dynamic on release) ├── docker/ # Docker build configurations │ ├── Dockerfile.v1-sync-helper # Go service container │ └── Dockerfile.meltano # Python ETL container @@ -142,7 +142,7 @@ lfx-v1-sync-helper/ #### Data Format Support - **JSON** (default): Standard JSON encoding for record storage -- **MessagePack**: Compact binary serialization with `msgpack: true` configuration +- **MessagePack**: Compact binary serialization with `msgpack: true` configuration (Meltano) or `USE_MSGPACK=true` (WAL handler) - **Automatic Detection**: Both Go service and Python plugin automatically detect format when reading existing data ## CI/CD Integration @@ -189,8 +189,11 @@ lfx-v1-sync-helper/ ### Data Serialization - **target-nats-kv** supports both JSON and MessagePack encoding - Set `msgpack: true` in Meltano configuration to enable MessagePack +- Set `USE_MSGPACK=true` environment variable for WAL handler to use MessagePack +- Boolean environment variables accept truthy values: "true", "yes", "t", "y", "1" (case-insensitive) - Automatic format detection when reading existing data for compatibility - Go service handles both formats transparently +- WAL handler respects the same encoding configuration as Meltano for consistency ### Container Standards - Multi-stage builds for size optimization diff --git a/README.md b/README.md index fd36187..9016b1a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Data sync components for LFX One -This repository contains tools and services for synchronizing data between LFX v1 and LFX One (v2) platforms. This solution uses Meltano for data replication into the v2 ecosystem, after which a sync helper service handles data mapping and ingestion. +This repository contains tools and services for synchronizing data between LFX v1 and LFX One (v2) platforms. This solution uses Meltano for data extraction and loading, a WAL listener for real-time PostgreSQL change streaming, and a sync helper service that handles data mapping and ingestion into the v2 ecosystem. ## Overview @@ -69,12 +69,12 @@ Data extraction and loading pipeline that extracts data from LFX v1 sources (Dyn Go service that monitors NATS KV stores for replicated v1 data and synchronizes it with the LFX v2 platform APIs, handling data transformation and conflict resolution. ### [Helm charts](./charts/lfx-v1-sync-helper/README.md) -Kubernetes deployment manifests for the v1-sync-helper service, providing scalable deployment options for production environments. +Kubernetes deployment manifests for the custom app service and WAL listener component, providing scalable deployment options for production environments. ## Architecture Diagrams Regarding the following diagrams: -- The planned realtime sync for PostgreSQL is included in the diagrams. + - The DynamoDB source (incremental or realtime) is not currently included in the diagrams. - The planned bidirectional sync (LFX One changes back to v1) is included in the diagrams. - "Projects API" is representative of most data entities. However, v1 Meetings push straight to OpenSearch and OpenFGA (via platform services)—this is not shown. @@ -134,7 +134,7 @@ sequenceDiagram participant opensearch as OpenSearch v1_kv-)+v1-sync-helper: notification on KV bucket subject - v1-sync-helper->>v1-sync-helper: check if delete or upsert + v1-sync-helper->>v1-sync-helper: check if delete (hard or soft) or upsert v1-sync-helper->>v1-sync-helper: check if upsert was by v1-sync-helper's M2M client ID v1-sync-helper->>+mapping-db: check for v1->v2 ID mapping mapping-db--)-v1-sync-helper: v2 ID, deletion tombstone, or empty @@ -155,8 +155,8 @@ sequenceDiagram projects-api -) opensearch: index deletion transection (via indexer) Note right of v1-sync-helper: if the DELETE fails, notify team and abort projects-api --)- v1-sync-helper: 204 (no body) - v1-sync-helper -) mapping-db: delete v1->v2 mapping - v1-sync-helper -) mapping-db: delete v2->v1 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v1->v2 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v2->v1 mapping else item upsert & NOT last-modified-by v1-sync-helper & mapping empty Note right of v1-sync-helper: This is a "create" from v1 v1-sync-helper->>v1-sync-helper: impersonate v1 principal w/ Heimdall key @@ -291,8 +291,8 @@ sequenceDiagram projects-api -) opensearch: index deletion transection (via indexer) Note right of v1-sync-helper: if the DELETE fails, notify team and abort projects-api --)- v1-sync-helper: 204 (no body) - v1-sync-helper -) mapping-db: delete v1->v2 mapping - v1-sync-helper -) mapping-db: delete v2->v1 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v1->v2 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v2->v1 mapping else item upsert & NOT last-modified-by v1-sync-helper & mapping empty Note right of v1-sync-helper: This is a "create" from v1 v1-sync-helper->>v1-sync-helper: impersonate v1 principal w/ Heimdall key @@ -339,8 +339,8 @@ sequenceDiagram mapping-db--)-v1-sync-helper: v1 ID v1-sync-helper->>+lfx_v1: delete in v1 lfx_v1->>-v1-sync-helper: 204 (no content) - v1-sync-helper -) mapping-db: delete v1->v2 mapping - v1-sync-helper -) mapping-db: delete v2->v1 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v2->v1 mapping + v1-sync-helper -) mapping-db: tombstone 🪦 v1->v2 mapping end deactivate v1-sync-helper ``` diff --git a/charts/lfx-v1-sync-helper/README.md b/charts/lfx-v1-sync-helper/README.md index f8b884f..9097c6c 100644 --- a/charts/lfx-v1-sync-helper/README.md +++ b/charts/lfx-v1-sync-helper/README.md @@ -1,6 +1,6 @@ # LFX v1 Sync Helper Helm Chart -This Helm chart deploys the LFX v1 Sync Helper service, which monitors NATS KV stores for v1 data and synchronizes it with the LFX v2 platform APIs, handling data transformation and conflict resolution. +This Helm chart deploys the LFX v1 Sync Helper service, which monitors NATS KV stores for v1 data and synchronizes it with the LFX v2 platform APIs, handling data transformation and conflict resolution. The chart also includes an optional PostgreSQL WAL listener component for real-time database change streaming. ## Prerequisites @@ -32,11 +32,11 @@ kubectl create secret generic v1-sync-helper-auth0-credentials \ # Install the chart with required image tag and AUTH0_TENANT helm install -n lfx lfx-v1-sync-helper \ ./charts/lfx-v1-sync-helper \ - --set image.tag=latest \ + --set app.image.tag=latest \ --set app.environment.AUTH0_TENANT.value=my_tenant ``` -**Note**: When using the local chart, you must specify `--set image.tag=latest` because the committed chart does not have an appVersion, so a version must always be specified when not using the published chart. The AUTH0_TENANT environment variable and Auth0 secret are also required. +**Note**: When using the local chart, you must specify `--set app.image.tag=latest` because the committed chart does not have an appVersion, so a version must always be specified when not using the published chart. The AUTH0_TENANT environment variable and Auth0 secret are also required. ### Installing from OCI registry @@ -52,6 +52,13 @@ kubectl create secret generic v1-sync-helper-auth0-credentials \ --from-literal=client_private_key="$(cat auth0-private-key.pem)" \ -n lfx +# Create PostgreSQL credentials secret (for wal-listener component) +kubectl create secret generic v1-platform-db-credentials \ + --from-literal=host=your-postgres-host \ + --from-literal=username=your-postgres-user \ + --from-literal=password=your-postgres-password \ + -n lfx + # Create values.yaml with required AUTH0_TENANT cat > values.yaml << EOF app: @@ -96,9 +103,19 @@ The chart requires the following secrets to be created before installation (if t -n lfx ``` -### Environment Variables +3. **PostgreSQL credentials** (default name: `v1-platform-db-credentials`): + Required for the WAL listener component to connect to the PostgreSQL database. + ```bash + kubectl create secret generic v1-platform-db-credentials \ + --from-literal=host=your-postgres-host \ + --from-literal=username=your-postgres-user \ + --from-literal=password=your-postgres-password \ + -n lfx + ``` + +### App Component -The following environment variables have defaults configured in the chart's `app.environment` section: +The following environment variables for the custom app component have defaults configured in the chart's `app.environment` section: | Variable | Default | Description | |-------------------------|----------------------------------------------------------------------------|---------------------------| @@ -113,7 +130,49 @@ The following environment variables have defaults configured in the chart's `app For a complete list of all supported environment variables, including required ones like `AUTH0_TENANT`, see the [v1-sync-helper README](../../cmd/lfx-v1-sync-helper/README.md#environment-variables). +### WAL Listener Component + +The chart includes an optional PostgreSQL WAL (Write-Ahead Log) listener component that provides real-time streaming of database changes to NATS. This component is enabled by default and can be configured or disabled as needed. + +#### WAL Listener Configuration + +| Parameter | Default | Description | +|-------------------------------------------|------------------------------------------------|----------------------------------------| +| `walListener.enabled` | `true` | Enable/disable WAL listener deployment | +| `walListener.replicas` | `1` | Number of WAL listener replicas | +| `walListener.image.repository` | `ihippik/wal-listener` | WAL listener container image | +| `walListener.image.tag` | `latest` | WAL listener image tag | +| `walListener.config.listener.slotName` | `lfx_v2` | PostgreSQL replication slot name | +| `walListener.config.database.secret.name` | `v1-platform-db-credentials` | Secret containing database credentials | +| `walListener.config.publisher.address` | `lfx-platform-nats.lfx.svc.cluster.local:4222` | NATS server address | +| `walListener.config.publisher.topic` | `wal_listener` | NATS topic for publishing changes | + +The WAL listener monitors the following PostgreSQL tables by default (matching the meltano.yml tap-postgres configuration): +- `collaboration__c` (platform schema) +- `community__c` (platform schema) +- `project__c` (salesforce schema) +- `alternate_email__c` (salesforce schema) +- `merged_user` (salesforce schema) + +To disable the WAL listener: +```yaml +walListener: + enabled: false +``` + +To customize monitored tables: +```yaml +walListener: + config: + listener: + filter: + tables: + your_table: + - insert + - update + - delete +``` + ### Additional Configuration For all available configuration options and their default values, please see the [values.yaml](values.yaml) file in this chart directory. You can override these values in your own `values.yaml` file or by using the `--set` flag when installing the chart. - diff --git a/charts/lfx-v1-sync-helper/templates/deployment.yaml b/charts/lfx-v1-sync-helper/templates/app-deployment.yaml similarity index 72% rename from charts/lfx-v1-sync-helper/templates/deployment.yaml rename to charts/lfx-v1-sync-helper/templates/app-deployment.yaml index f072eb1..0603fcf 100644 --- a/charts/lfx-v1-sync-helper/templates/deployment.yaml +++ b/charts/lfx-v1-sync-helper/templates/app-deployment.yaml @@ -4,29 +4,31 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Chart.Name }} + name: {{ .Chart.Name }}-app namespace: {{ .Release.Namespace }} {{- with .Values.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.replicas | default 1 }} + replicas: {{ .Values.app.replicas | default 1 }} selector: matchLabels: app: {{ .Chart.Name }} + component: app template: metadata: labels: app: {{ .Chart.Name }} + component: app spec: serviceAccountName: {{ .Values.serviceAccount.name | default .Chart.Name }} containers: - name: app - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.app.image.repository }}:{{ .Values.app.image.tag | default .Chart.AppVersion }}" securityContext: allowPrivilegeEscalation: false - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ .Values.app.image.pullPolicy }} env: {{- range $name, $config := .Values.app.environment }} - name: {{ $name }} @@ -39,23 +41,23 @@ spec: {{- end }} # JWT configuration from Heimdall - name: HEIMDALL_CLIENT_ID - value: {{ .Values.heimdall.clientId | quote }} + value: {{ .Values.app.heimdall.clientId | quote }} - name: HEIMDALL_PRIVATE_KEY valueFrom: secretKeyRef: - name: {{ .Values.heimdall.secret.name }} - key: {{ .Values.heimdall.secret.privateKeyKey }} + name: {{ .Values.app.heimdall.secret.name }} + key: {{ .Values.app.heimdall.secret.privateKeyKey }} # Auth0 configuration for v1 API authentication - name: AUTH0_CLIENT_ID valueFrom: secretKeyRef: - name: {{ .Values.auth0.secret.name }} - key: {{ .Values.auth0.secret.clientIdKey }} + name: {{ .Values.app.auth0.secret.name }} + key: {{ .Values.app.auth0.secret.clientIdKey }} - name: AUTH0_PRIVATE_KEY valueFrom: secretKeyRef: - name: {{ .Values.auth0.secret.name }} - key: {{ .Values.auth0.secret.privateKeyKey }} + name: {{ .Values.app.auth0.secret.name }} + key: {{ .Values.app.auth0.secret.privateKeyKey }} ports: - containerPort: 8080 name: web diff --git a/charts/lfx-v1-sync-helper/templates/meltano-postgres-cronjob.yaml b/charts/lfx-v1-sync-helper/templates/meltano-postgres-cronjob.yaml new file mode 100644 index 0000000..2078384 --- /dev/null +++ b/charts/lfx-v1-sync-helper/templates/meltano-postgres-cronjob.yaml @@ -0,0 +1,123 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT +--- +# Meltano PostgreSQL ETL CronJob for extracting data from PostgreSQL and loading to NATS KV. +{{- if .Values.meltano.postgresEtl.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Chart.Name }}-meltano-postgres + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: meltano + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + schedule: {{ .Values.meltano.postgresEtl.schedule | quote }} + concurrencyPolicy: {{ .Values.meltano.postgresEtl.concurrencyPolicy }} + successfulJobsHistoryLimit: {{ .Values.meltano.postgresEtl.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.meltano.postgresEtl.failedJobsHistoryLimit }} + jobTemplate: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: meltano + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + ttlSecondsAfterFinished: {{ .Values.meltano.postgresEtl.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.meltano.postgresEtl.backoffLimit }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: meltano + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + restartPolicy: Never + serviceAccountName: {{ .Values.serviceAccount.name | default .Chart.Name }} + containers: + - name: meltano + image: "{{ .Values.meltano.postgresEtl.image.repository }}:{{ .Values.meltano.postgresEtl.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.meltano.postgresEtl.image.pullPolicy }} + workingDir: /app/meltano + # Meltano args must be in specific order: el [--catalog path] [--state-id id] + args: + - "el" + {{- if and .Values.meltano.postgresEtl.catalog.enabled .Values.meltano.postgresEtl.catalog.configMap.name }} + - "--catalog" + - "{{ .Values.meltano.postgresEtl.catalog.configMap.mountPath }}/{{ .Values.meltano.postgresEtl.catalog.configMap.key }}" + {{- end }} + - "--state-id" + - {{ .Values.meltano.postgresEtl.stateId | quote }} + - "tap-postgres" + - "target-nats-kv" + env: + # Meltano environment configuration + - name: MELTANO_ENVIRONMENT + value: {{ .Values.meltano.postgresEtl.environment | quote }} + - name: MELTANO_STATE_BACKEND_URI + value: "s3://{{ .Values.meltano.postgresEtl.aws.stateBucketPrefix }}-{{ .Values.meltano.postgresEtl.environment }}" + # AWS configuration + - name: AWS_DEFAULT_REGION + value: {{ .Values.meltano.postgresEtl.aws.region | quote }} + - name: AWS_REGION + value: {{ .Values.meltano.postgresEtl.aws.region | quote }} + # PostgreSQL configuration + - name: TAP_POSTGRES_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.meltano.postgresEtl.postgres.host.secret.name }} + key: {{ .Values.meltano.postgresEtl.postgres.host.secret.key }} + optional: true + - name: TAP_POSTGRES_PORT + value: {{ .Values.meltano.postgresEtl.postgres.port | quote }} + - name: TAP_POSTGRES_DATABASE + value: {{ .Values.meltano.postgresEtl.postgres.database | quote }} + - name: TAP_POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ .Values.meltano.postgresEtl.postgres.username.secret.name }} + key: {{ .Values.meltano.postgresEtl.postgres.username.secret.key }} + optional: true + - name: TAP_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.meltano.postgresEtl.postgres.password.secret.name }} + key: {{ .Values.meltano.postgresEtl.postgres.password.secret.key }} + optional: true + # Target NATS KV configuration + - name: TARGET_NATS_KV_URL + value: {{ .Values.meltano.postgresEtl.nats.url | quote }} + - name: TARGET_NATS_KV_BUCKET + value: {{ .Values.meltano.postgresEtl.nats.bucket | quote }} + - name: TARGET_NATS_KV_REFRESH_MODE + value: {{ .Values.meltano.postgresEtl.nats.refreshMode | quote }} + - name: TARGET_NATS_KV_VALIDATE_RECORDS + value: {{ .Values.meltano.postgresEtl.nats.validateRecords | quote }} + - name: TARGET_NATS_KV_MSGPACK + value: {{ .Values.meltano.postgresEtl.nats.useMessagePack | quote }} + resources: + {{- toYaml .Values.meltano.postgresEtl.resources | nindent 16 }} + livenessProbe: + exec: + command: ["pgrep", "-f", "meltano"] + initialDelaySeconds: 30 + periodSeconds: 60 + failureThreshold: 3 + {{- if and .Values.meltano.postgresEtl.catalog.enabled .Values.meltano.postgresEtl.catalog.configMap.name }} + volumeMounts: + - name: tap-postgres-catalog + mountPath: {{ .Values.meltano.postgresEtl.catalog.configMap.mountPath }} + readOnly: true + {{- end }} + {{- if and .Values.meltano.postgresEtl.catalog.enabled .Values.meltano.postgresEtl.catalog.configMap.name }} + volumes: + - name: tap-postgres-catalog + configMap: + name: {{ .Values.meltano.postgresEtl.catalog.configMap.name }} + {{- end }} +{{- end }} diff --git a/charts/lfx-v1-sync-helper/templates/nats-kv-buckets.yaml b/charts/lfx-v1-sync-helper/templates/nats-kv-buckets.yaml deleted file mode 100644 index 71e2b26..0000000 --- a/charts/lfx-v1-sync-helper/templates/nats-kv-buckets.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright The Linux Foundation and each contributor to LFX. -# SPDX-License-Identifier: MIT ---- -{{- if .Values.nats.kv_bucket_v1_objects.creation }} -apiVersion: jetstream.nats.io/v1beta2 -kind: KeyValue -metadata: - name: {{ .Values.nats.kv_bucket_v1_objects.name }} - namespace: lfx - {{- if .Values.nats.kv_bucket_v1_objects.keep }} - annotations: - "helm.sh/resource-policy": keep - {{- end }} -spec: - bucket: {{ .Values.nats.kv_bucket_v1_objects.name }} - history: {{ .Values.nats.kv_bucket_v1_objects.history }} - storage: {{ .Values.nats.kv_bucket_v1_objects.storage }} - maxValueSize: {{ .Values.nats.kv_bucket_v1_objects.maxValueSize }} - maxBytes: {{ .Values.nats.kv_bucket_v1_objects.maxBytes }} - compression: {{ .Values.nats.kv_bucket_v1_objects.compression }} -{{- end }} ---- -{{- if .Values.nats.kv_bucket_v1_mappings.creation }} -apiVersion: jetstream.nats.io/v1beta2 -kind: KeyValue -metadata: - name: {{ .Values.nats.kv_bucket_v1_mappings.name }} - namespace: lfx - {{- if .Values.nats.kv_bucket_v1_mappings.keep }} - annotations: - "helm.sh/resource-policy": keep - {{- end }} -spec: - bucket: {{ .Values.nats.kv_bucket_v1_mappings.name }} - history: {{ .Values.nats.kv_bucket_v1_mappings.history }} - storage: {{ .Values.nats.kv_bucket_v1_mappings.storage }} - maxValueSize: {{ .Values.nats.kv_bucket_v1_mappings.maxValueSize }} - maxBytes: {{ .Values.nats.kv_bucket_v1_mappings.maxBytes }} - compression: {{ .Values.nats.kv_bucket_v1_mappings.compression }} -{{- end }} diff --git a/charts/lfx-v1-sync-helper/templates/nats-resources.yaml b/charts/lfx-v1-sync-helper/templates/nats-resources.yaml new file mode 100644 index 0000000..4f508ec --- /dev/null +++ b/charts/lfx-v1-sync-helper/templates/nats-resources.yaml @@ -0,0 +1,65 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT +--- +{{- if .Values.natsResources.kv_bucket_v1_objects.creation }} +apiVersion: jetstream.nats.io/v1beta2 +kind: KeyValue +metadata: + name: {{ .Values.natsResources.kv_bucket_v1_objects.name }} + namespace: lfx + {{- if .Values.natsResources.kv_bucket_v1_objects.keep }} + annotations: + "helm.sh/resource-policy": keep + {{- end }} +spec: + bucket: {{ .Values.natsResources.kv_bucket_v1_objects.name }} + history: {{ .Values.natsResources.kv_bucket_v1_objects.history }} + storage: {{ .Values.natsResources.kv_bucket_v1_objects.storage }} + maxValueSize: {{ .Values.natsResources.kv_bucket_v1_objects.maxValueSize }} + maxBytes: {{ .Values.natsResources.kv_bucket_v1_objects.maxBytes }} + compression: {{ .Values.natsResources.kv_bucket_v1_objects.compression }} +{{- end }} +--- +{{- if .Values.natsResources.kv_bucket_v1_mappings.creation }} +apiVersion: jetstream.nats.io/v1beta2 +kind: KeyValue +metadata: + name: {{ .Values.natsResources.kv_bucket_v1_mappings.name }} + namespace: lfx + {{- if .Values.natsResources.kv_bucket_v1_mappings.keep }} + annotations: + "helm.sh/resource-policy": keep + {{- end }} +spec: + bucket: {{ .Values.natsResources.kv_bucket_v1_mappings.name }} + history: {{ .Values.natsResources.kv_bucket_v1_mappings.history }} + storage: {{ .Values.natsResources.kv_bucket_v1_mappings.storage }} + maxValueSize: {{ .Values.natsResources.kv_bucket_v1_mappings.maxValueSize }} + maxBytes: {{ .Values.natsResources.kv_bucket_v1_mappings.maxBytes }} + compression: {{ .Values.natsResources.kv_bucket_v1_mappings.compression }} +{{- end }} +--- +{{- if .Values.natsResources.stream_wal_listener.creation }} +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: {{ .Values.natsResources.stream_wal_listener.name | replace "_" "-" }} + namespace: lfx + {{- if .Values.natsResources.stream_wal_listener.keep }} + annotations: + "helm.sh/resource-policy": keep + {{- end }} +spec: + name: {{ .Values.natsResources.stream_wal_listener.name }} + subjects: + {{- range .Values.natsResources.stream_wal_listener.subjects }} + - {{ . }} + {{- end }} + storage: {{ .Values.natsResources.stream_wal_listener.storage }} + retention: {{ .Values.natsResources.stream_wal_listener.retention }} + maxAge: {{ .Values.natsResources.stream_wal_listener.maxAge }} + maxBytes: {{ .Values.natsResources.stream_wal_listener.maxBytes }} + maxMsgs: {{ .Values.natsResources.stream_wal_listener.maxMsgs }} + replicas: {{ .Values.natsResources.stream_wal_listener.replicas }} + compression: {{ .Values.natsResources.stream_wal_listener.compression }} +{{- end }} diff --git a/charts/lfx-v1-sync-helper/templates/wal-listener-configmap.yaml b/charts/lfx-v1-sync-helper/templates/wal-listener-configmap.yaml new file mode 100644 index 0000000..b924156 --- /dev/null +++ b/charts/lfx-v1-sync-helper/templates/wal-listener-configmap.yaml @@ -0,0 +1,45 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT +{{- if .Values.walListener.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-wal-listener-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + component: wal-listener +data: + config.yml: | + listener: + serverPort: {{ .Values.walListener.config.listener.serverPort }} + slotName: {{ .Values.walListener.config.listener.slotName | quote }} + refreshConnection: {{ .Values.walListener.config.listener.refreshConnection }} + heartbeatInterval: {{ .Values.walListener.config.listener.heartbeatInterval }} + filter: + tables: + {{- range $table, $operations := .Values.walListener.config.listener.filter.tables }} + {{ $table }}: + {{- range $operation := $operations }} + - {{ $operation }} + {{- end }} + {{- end }} + logger: + level: {{ .Values.walListener.config.logger.level }} + fmt: {{ .Values.walListener.config.logger.fmt }} + database: + host: localhost + port: {{ .Values.walListener.config.database.port }} + name: {{ .Values.walListener.config.database.name | quote }} + user: placeholder + password: placeholder + enableTLS: {{ .Values.walListener.config.database.enableTLS }} + {{- if .Values.walListener.config.database.sslMode }} + sslMode: {{ .Values.walListener.config.database.sslMode | quote }} + {{- end }} + publisher: + type: {{ .Values.walListener.config.publisher.type }} + address: {{ .Values.walListener.config.publisher.address | quote }} + topic: {{ .Values.walListener.config.publisher.topic | quote }} +{{- end }} diff --git a/charts/lfx-v1-sync-helper/templates/wal-listener-deployment.yaml b/charts/lfx-v1-sync-helper/templates/wal-listener-deployment.yaml new file mode 100644 index 0000000..3b3ee7c --- /dev/null +++ b/charts/lfx-v1-sync-helper/templates/wal-listener-deployment.yaml @@ -0,0 +1,82 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT +{{- if .Values.walListener.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }}-wal-listener + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + component: wal-listener +spec: + replicas: {{ .Values.walListener.replicas }} + selector: + matchLabels: + app: {{ .Chart.Name }} + component: wal-listener + template: + metadata: + labels: + app: {{ .Chart.Name }} + component: wal-listener + spec: + serviceAccountName: {{ .Values.serviceAccount.name | default .Chart.Name }} + containers: + - name: wal-listener + image: "{{ .Values.walListener.image.repository }}:{{ .Values.walListener.image.tag }}" + imagePullPolicy: {{ .Values.walListener.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + ports: + - name: http + containerPort: {{ .Values.walListener.config.listener.serverPort }} + protocol: TCP + env: + - name: WAL_DATABASE_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.walListener.config.database.secret.name }} + key: {{ .Values.walListener.config.database.secret.hostKey }} + optional: true + - name: WAL_DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ .Values.walListener.config.database.secret.name }} + key: {{ .Values.walListener.config.database.secret.usernameKey }} + optional: true + - name: WAL_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.walListener.config.database.secret.name }} + key: {{ .Values.walListener.config.database.secret.passwordKey }} + optional: true + volumeMounts: + - name: config + mountPath: /app/config.yml + subPath: config.yml + readOnly: true + resources: + {{- toYaml .Values.walListener.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + volumes: + - name: config + configMap: + name: {{ .Chart.Name }}-wal-listener-config +{{- end }} diff --git a/charts/lfx-v1-sync-helper/templates/wal-listener-service.yaml b/charts/lfx-v1-sync-helper/templates/wal-listener-service.yaml new file mode 100644 index 0000000..7557c11 --- /dev/null +++ b/charts/lfx-v1-sync-helper/templates/wal-listener-service.yaml @@ -0,0 +1,23 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT +{{- if .Values.walListener.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-wal-listener + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + component: wal-listener +spec: + type: ClusterIP + ports: + - port: {{ .Values.walListener.config.listener.serverPort }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Chart.Name }} + component: wal-listener +{{- end }} diff --git a/charts/lfx-v1-sync-helper/values.yaml b/charts/lfx-v1-sync-helper/values.yaml index 74b8026..c5cf184 100644 --- a/charts/lfx-v1-sync-helper/values.yaml +++ b/charts/lfx-v1-sync-helper/values.yaml @@ -1,50 +1,7 @@ # Copyright The Linux Foundation and each contributor to LFX. # SPDX-License-Identifier: MIT --- -# replicas is the number of service instances to run for horizontal scaling -replicas: 1 - -# image is the configuration for the container images -image: - # repository is the container image repository - repository: ghcr.io/linuxfoundation/lfx-v1-sync-helper/lfx-v1-sync-helper - # tag is the container image tag (overrides appVersion from Chart.yaml) - tag: "" - # pullPolicy is the image pull policy - pullPolicy: IfNotPresent - -# app is the configuration for the application -app: - environment: - # NATS_URL is required - NATS_URL: - value: nats://lfx-platform-nats.lfx.svc.cluster.local:4222 - # PORT is optional - PORT: - value: "8080" - # BIND is optional - BIND: - value: "*" - # PROJECT_SERVICE_URL is required for making API calls to project service - PROJECT_SERVICE_URL: - value: http://lfx-v2-project-service.lfx.svc.cluster.local:8080 - # COMMITTEE_SERVICE_URL is required for making API calls to committee service - COMMITTEE_SERVICE_URL: - value: http://lfx-v2-committee-service.lfx.svc.cluster.local:8080 - # AUTH0_TENANT is required for Auth0 authentication - AUTH0_TENANT: - value: "" - # HEIMDALL_KEY_ID is optional - JWT key ID (if not provided, fetches from JWKS) - HEIMDALL_KEY_ID: - value: "" - # HEIMDALL_JWKS_URL is optional - JWKS endpoint URL - HEIMDALL_JWKS_URL: - value: http://lfx-platform-heimdall.lfx.svc.cluster.local:4457/.well-known/jwks - # LFX_API_GW is optional - LFX API Gateway URL - LFX_API_GW: - value: https://api-gw.dev.platform.linuxfoundation.org/ - -# serviceAccount is the configuration for the Kubernetes service account +# serviceAccount is the configuration for the Kubernetes service account (used by multiple components) serviceAccount: # create specifies whether a service account should be created create: true @@ -56,8 +13,8 @@ serviceAccount: # automountServiceAccountToken is a boolean to determine if the service account token should be automatically mounted automountServiceAccountToken: true -# nats is the configuration for the NATS server -nats: +# natsResources is the configuration for the NATS resources managed by this chart (used by multiple components) +natsResources: # kv_bucket_v1_objects is the configuration for the KV bucket for storing v1 objects from Meltano kv_bucket_v1_objects: # creation is a boolean to determine if the KV bucket should be created via the helm chart. @@ -73,9 +30,9 @@ nats: # storage is the storage type for the KV bucket storage: file # maxValueSize is the maximum size of a value in the KV bucket - maxValueSize: 10485760 # 10MB + maxValueSize: 10485760 # 10MB # maxBytes is the maximum number of bytes in the KV bucket - maxBytes: 10737418240 # 10GB + maxBytes: 10737418240 # 10GB # compression is a boolean to determine if the KV bucket should be compressed compression: true @@ -94,33 +51,324 @@ nats: # storage is the storage type for the KV bucket storage: file # maxValueSize is the maximum size of a value in the KV bucket - maxValueSize: 10485760 # 10MB + maxValueSize: 10485760 # 10MB # maxBytes is the maximum number of bytes in the KV bucket - maxBytes: 2147483648 # 2GB + maxBytes: 2147483648 # 2GB # compression is a boolean to determine if the KV bucket should be compressed compression: true -# heimdall is the configuration for JWT impersonation of Heimdall-authorized -# principals for v1 data ingest. -heimdall: + # stream_wal_listener is the configuration for the JetStream stream for wal-listener events + stream_wal_listener: + # creation is a boolean to determine if the JetStream stream should be created via the helm chart. + # set it to false if you want to use an existing stream. + creation: true + # keep is a boolean to determine if the stream should be preserved during helm uninstall + # set it to false if you want the stream to be deleted when the chart is uninstalled + keep: true + # name is the name of the JetStream stream for wal-listener events + name: wal_listener + # subjects is the list of subjects this stream will subscribe to + subjects: + - wal_listener.* + # storage is the storage type for the stream + storage: file + # retention is the retention policy type: "limits", "interest", or "workqueue" + retention: limits + # maxAge is the maximum age of messages in the stream (uses time.ParseDuration() format) + maxAge: 336h # 2 weeks + # maxBytes is the maximum number of bytes in the stream (-1 for unlimited) + maxBytes: -1 + # maxMsgs is the maximum number of messages in the stream (-1 for unlimited) + maxMsgs: -1 + # replicas is the number of replicas for the stream (1 for single instance) + replicas: 1 + # compression can be "s2" or "none" (s2 is default) + compression: s2 + +# app is the configuration for the application +app: + # replicas is the number of service instances to run for horizontal scaling + replicas: 1 + # image is the configuration for the container images + image: + # repository is the container image repository + repository: ghcr.io/linuxfoundation/lfx-v1-sync-helper/lfx-v1-sync-helper + # tag is the container image tag (overrides appVersion from Chart.yaml) + tag: "" + # pullPolicy is the image pull policy + pullPolicy: IfNotPresent + environment: + # NATS_URL is required + NATS_URL: + value: nats://lfx-platform-nats.lfx.svc.cluster.local:4222 + # PORT is optional + PORT: + value: "8080" + # BIND is optional + BIND: + value: "*" + # PROJECT_SERVICE_URL is required for making API calls to project service + PROJECT_SERVICE_URL: + value: http://lfx-v2-project-service.lfx.svc.cluster.local:8080 + # COMMITTEE_SERVICE_URL is required for making API calls to committee service + COMMITTEE_SERVICE_URL: + value: http://lfx-v2-committee-service.lfx.svc.cluster.local:8080 + # AUTH0_TENANT is required for Auth0 authentication + AUTH0_TENANT: + value: "" + # HEIMDALL_KEY_ID is optional - JWT key ID (if not provided, fetches from JWKS) + HEIMDALL_KEY_ID: + value: "" + # HEIMDALL_JWKS_URL is optional - JWKS endpoint URL + HEIMDALL_JWKS_URL: + value: http://lfx-platform-heimdall.lfx.svc.cluster.local:4457/.well-known/jwks + # LFX_API_GW is optional - LFX API Gateway URL + LFX_API_GW: + value: https://api-gw.dev.platform.linuxfoundation.org/ + # USE_MSGPACK is optional - Use MessagePack encoding for KV bucket values (default: false for JSON) + # Accepts truthy values: "true", "yes", "t", "y", "1" (case-insensitive) + USE_MSGPACK: + value: "false" + + # heimdall is the configuration for JWT impersonation of Heimdall-authorized + # principals for v1 data ingest. + heimdall: + enabled: true + url: http://lfx-platform-heimdall.lfx.svc.cluster.local:4456 + # fallback principal for JWT impersonation, without the "@clients" suffix. + clientId: v1_sync_helper + # secret contains the configuration for JWT signing + secret: + # name of the secret containing JWT private key and configuration + name: heimdall-signer-cert + # key in the secret which contains the signing certificate + privateKeyKey: signer.pem + + # auth0 is the configuration for Auth0 authentication for v1 API calls + auth0: + # secret contains the configuration for Auth0 authentication + secret: + # name of the secret containing Auth0 client ID and private key + name: v1-sync-helper-auth0-credentials + # key in the secret which contains the Auth0 client ID + clientIdKey: client_id + # key in the secret which contains the Auth0 private key + privateKeyKey: client_private_key + +# wal-listener is the configuration for the PostgreSQL WAL listener component +walListener: + # enabled specifies whether the wal-listener should be deployed enabled: true - url: http://lfx-platform-heimdall.lfx.svc.cluster.local:4456 - # fallback principal for JWT impersonation, without the "@clients" suffix. - clientId: v1_sync_helper - # secret contains the configuration for JWT signing - secret: - # name of the secret containing JWT private key and configuration - name: heimdall-signer-cert - # key in the secret which contains the signing certificate - privateKeyKey: signer.pem + # replicas is the number of wal-listener instances to run + replicas: 1 + # image is the configuration for the wal-listener container image + image: + # repository is the container image repository + repository: ihippik/wal-listener + # tag is the container image tag + tag: latest + # pullPolicy is the image pull policy + pullPolicy: IfNotPresent + # config is the configuration for the wal-listener + config: + listener: + # serverPort is the port for health checks and monitoring + serverPort: 8080 + # slotName is the PostgreSQL replication slot name + slotName: lfx_v2 + # refreshConnection is the connection refresh interval + refreshConnection: 30s + # heartbeatInterval is the heartbeat interval + heartbeatInterval: 10s + # filter contains table filtering configuration + filter: + # tables is a map of table names to operations to listen for + # Default tables match those from meltano.yml tap-postgres configuration + tables: + collaboration__c: + - insert + - update + - delete + community__c: + - insert + - update + - delete + project__c: + - insert + - update + - delete + alternate_email__c: + - insert + - update + - delete + merged_user: + - insert + - update + - delete + logger: + # level is the logging level + level: info + # fmt is the log format + fmt: json + database: + # PostgreSQL database configuration + # Uses the same secret as meltano-el-postgres by default + secret: + # name is the name of the secret containing database credentials + name: v1-platform-db-credentials + # hostKey is the key in the secret containing the database host + hostKey: host + # usernameKey is the key in the secret containing the database username + usernameKey: username + # passwordKey is the key in the secret containing the database password + passwordKey: password + # port is the database port + port: 5432 + # name is the database name + name: sfdc + # TLS configuration for PostgreSQL connection + # enableTLS determines whether to use TLS/SSL connection to PostgreSQL + enableTLS: false + # sslMode specifies the TLS verification mode when enableTLS is true: require, verify-ca, verify-full + sslMode: "require" + + publisher: + # type is the publisher type (nats or kafka) + type: nats + # address is the NATS server address (matches meltano.yml default) + address: lfx-platform-nats.lfx.svc.cluster.local:4222 + # topic is the NATS topic to publish to + topic: wal_listener + # resources is the resource configuration for the wal-listener container + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" -# auth0 is the configuration for Auth0 authentication for v1 API calls -auth0: - # secret contains the configuration for Auth0 authentication - secret: - # name of the secret containing Auth0 client ID and private key - name: v1-sync-helper-auth0-credentials - # key in the secret which contains the Auth0 client ID - clientIdKey: client_id - # key in the secret which contains the Auth0 private key - privateKeyKey: client_private_key +# meltano is the configuration for Meltano ETL CronJobs +meltano: + # postgresEtl is the configuration for the PostgreSQL ETL CronJob + postgresEtl: + # enabled specifies whether the PostgreSQL ETL CronJob should be deployed + enabled: true + # schedule is the cron schedule for the job (every 5 minutes by default) + schedule: "*/5 * * * *" + # concurrencyPolicy controls concurrent execution (Forbid, Allow, Replace) + concurrencyPolicy: Forbid + # successfulJobsHistoryLimit is the number of successful jobs to keep + successfulJobsHistoryLimit: 3 + # failedJobsHistoryLimit is the number of failed jobs to keep + failedJobsHistoryLimit: 3 + # ttlSecondsAfterFinished is the TTL for completed jobs (1 hour) + ttlSecondsAfterFinished: 3600 + # backoffLimit is the number of retries before marking job as failed + backoffLimit: 1 + # image is the configuration for the Meltano container image + image: + # repository is the container image repository + repository: ghcr.io/linuxfoundation/lfx-v1-sync-helper/meltano + # tag is the container image tag (overrides appVersion from Chart.yaml) + tag: "" + # pullPolicy is the image pull policy + pullPolicy: IfNotPresent + # stateId is the state ID for incremental extractions (used with --state-id) + stateId: platform-db-incremental + # environment specifies the Meltano environment (dev/staging/prod) + # This value is used to: + # - Set MELTANO_ENVIRONMENT variable + # - Construct S3 state bucket name as: {aws.stateBucketPrefix}-{environment} + environment: dev + # aws configuration for state backend and data access + # Environment variables built: AWS_DEFAULT_REGION, AWS_REGION, MELTANO_STATE_BACKEND_URI + aws: + # region for AWS services (S3, DynamoDB) + region: us-west-2 + # stateBucketPrefix for S3 state backend naming (becomes: s3://{prefix}-{environment}) + stateBucketPrefix: lfx-v2-meltano-state + # postgres configuration for database connection + # Environment variables built: TAP_POSTGRES_HOST, TAP_POSTGRES_PORT, TAP_POSTGRES_DATABASE, + # TAP_POSTGRES_USER, TAP_POSTGRES_PASSWORD (from secrets) + postgres: + # host configuration + host: + # secret containing PostgreSQL host + secret: + name: v1-platform-db-credentials + key: host + # port for PostgreSQL connection + port: 5432 + # database name + database: sfdc + # username configuration + username: + secret: + name: v1-platform-db-credentials + key: username + # password configuration + password: + secret: + name: v1-platform-db-credentials + key: password + # nats configuration for target NATS KV + # Environment variables built: TARGET_NATS_KV_URL, TARGET_NATS_KV_BUCKET, + # TARGET_NATS_KV_REFRESH_MODE, TARGET_NATS_KV_VALIDATE_RECORDS, TARGET_NATS_KV_MSGPACK + nats: + # url for NATS connection + url: nats://lfx-platform-nats.lfx.svc.cluster.local:4222 + # bucket name for storing v1 objects + bucket: v1-objects + # refresh mode for KV operations + refreshMode: newer + # validate records before storing (true for PostgreSQL, false for DynamoDB) + validateRecords: true + # use MessagePack encoding + useMessagePack: false + # resources is the resource configuration for the Meltano container + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + # catalog configuration for PostgreSQL tap + # Note: Meltano arguments are built dynamically in the correct order: + # meltano el [--catalog path] [--state-id id] tap-postgres target-nats-kv + # + # Users must create their own ConfigMap containing the catalog JSON. + # Example ConfigMap creation: + # kubectl create configmap my-postgres-catalog --from-file=catalog.json=./my-catalog.json + # Or via YAML manifest: + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: my-postgres-catalog + # data: + # catalog.json: | + # { + # "streams": [ + # { + # "tap_stream_id": "my_table", + # "table_name": "my_table", + # "replication_method": "INCREMENTAL", + # "key_properties": ["id"], + # "schema": {...} + # } + # ] + # } + catalog: + # enabled specifies whether to use a catalog file for schema discovery + # Set to false to rely on Meltano's automatic schema discovery + enabled: false + # configMap specifies an existing ConfigMap containing the catalog (required if enabled) + # Users must create their own ConfigMap with the catalog content before enabling this + configMap: + # name is the name of an existing ConfigMap containing the catalog (required) + name: "" + # key is the key in the ConfigMap containing the catalog JSON + key: catalog.json + # mountPath is where to mount the catalog file in the container + mountPath: /catalogs/tap-postgres diff --git a/cmd/lfx-v1-sync-helper/client_committees.go b/cmd/lfx-v1-sync-helper/client_committees.go index aba0e44..28afca9 100644 --- a/cmd/lfx-v1-sync-helper/client_committees.go +++ b/cmd/lfx-v1-sync-helper/client_committees.go @@ -173,6 +173,62 @@ func updateCommitteeMember(ctx context.Context, payload *committeeservice.Update return nil } +// deleteCommittee deletes a committee by UID. +func deleteCommittee(ctx context.Context, committeeUID string, v1Principal string) error { + // Fetch current committee base to get etag. + _, etag, err := fetchCommitteeBase(ctx, committeeUID) + if err != nil { + return fmt.Errorf("failed to fetch committee base for deletion: %w", err) + } + + token, err := generateCachedJWTToken(ctx, committeeServiceAudience, v1Principal) + if err != nil { + return fmt.Errorf("failed to generate token for committee deletion: %w", err) + } + + payload := &committeeservice.DeleteCommitteePayload{ + BearerToken: &token, + UID: &committeeUID, + IfMatch: stringToStringPtr(etag), + } + + err = committeeClient.DeleteCommittee(ctx, payload) + if err != nil { + return fmt.Errorf("failed to delete committee: %w", err) + } + + return nil +} + +// deleteCommitteeMember deletes a committee member by committee UID and member UID. +func deleteCommitteeMember(ctx context.Context, committeeUID, memberUID string, v1Principal string) error { + // Fetch current committee member to get etag. + _, etag, err := fetchCommitteeMember(ctx, committeeUID, memberUID) + if err != nil { + return fmt.Errorf("failed to fetch committee member for deletion: %w", err) + } + + token, err := generateCachedJWTToken(ctx, committeeServiceAudience, v1Principal) + if err != nil { + return fmt.Errorf("failed to generate token for committee member deletion: %w", err) + } + + payload := &committeeservice.DeleteCommitteeMemberPayload{ + BearerToken: &token, + UID: committeeUID, + MemberUID: memberUID, + Version: "1", + IfMatch: stringToStringPtr(etag), + } + + err = committeeClient.DeleteCommitteeMember(ctx, payload) + if err != nil { + return fmt.Errorf("failed to delete committee member: %w", err) + } + + return nil +} + // committeeMembersEqual compares a committee member with an update payload for equality. func committeeMembersEqual(current *committeeservice.CommitteeMemberFullWithReadonlyAttributes, update *committeeservice.UpdateCommitteeMemberPayload) bool { // Compare basic fields. @@ -182,9 +238,7 @@ func committeeMembersEqual(current *committeeservice.CommitteeMemberFullWithRead stringPtrToString(current.LastName) != stringPtrToString(update.LastName) || stringPtrToString(current.JobTitle) != stringPtrToString(update.JobTitle) || current.AppointedBy != update.AppointedBy || - current.Status != update.Status || - stringPtrToString(current.Agency) != stringPtrToString(update.Agency) || - stringPtrToString(current.Country) != stringPtrToString(update.Country) { + current.Status != update.Status { return false } diff --git a/cmd/lfx-v1-sync-helper/client_projects.go b/cmd/lfx-v1-sync-helper/client_projects.go index c1c2d4f..2b12b4b 100644 --- a/cmd/lfx-v1-sync-helper/client_projects.go +++ b/cmd/lfx-v1-sync-helper/client_projects.go @@ -174,6 +174,33 @@ func updateProject(ctx context.Context, basePayload *projectservice.UpdateProjec return nil } +// deleteProject deletes a project by UID. +func deleteProject(ctx context.Context, projectUID string, v1Principal string) error { + // Fetch current project base to get etag. + _, etag, err := fetchProjectBase(ctx, projectUID) + if err != nil { + return fmt.Errorf("failed to fetch project base for deletion: %w", err) + } + + token, err := generateCachedJWTToken(ctx, projectServiceAudience, v1Principal) + if err != nil { + return fmt.Errorf("failed to generate token for project deletion: %w", err) + } + + payload := &projectservice.DeleteProjectPayload{ + BearerToken: &token, + UID: &projectUID, + IfMatch: stringToStringPtr(etag), + } + + err = projectClient.DeleteProject(ctx, payload) + if err != nil { + return fmt.Errorf("failed to delete project: %w", err) + } + + return nil +} + // projectBasesEqual compares two ProjectBase objects for equality, ignoring system-managed fields. func projectBasesEqual(a, b *projectservice.ProjectBase) bool { return stringPtrToString(a.Name) == stringPtrToString(b.Name) && diff --git a/cmd/lfx-v1-sync-helper/config.go b/cmd/lfx-v1-sync-helper/config.go index 5f25c19..2250e4e 100644 --- a/cmd/lfx-v1-sync-helper/config.go +++ b/cmd/lfx-v1-sync-helper/config.go @@ -8,6 +8,8 @@ import ( "fmt" "net/url" "os" + "slices" + "strings" ) // projectAllowlist contains the list of project slugs that are allowed to be @@ -15,11 +17,11 @@ import ( // entries must be lowercase* (lookups downcase for case-insensitive matching). var projectAllowlist = []string{ "tlf", - // "lfprojects", - // "lf-charities", - // "jdf", - // "jdf-llc", - // "jdf-international", + "lfprojects", + "lf-charities", + "jdf", + "jdf-llc", + "jdf-international", // "lfenergy", } @@ -27,6 +29,7 @@ var projectAllowlist = []string{ // to be synced along with their child projects. *All entries must be // lowercase* (lookups downcase for case-insensitive matching). var projectFamilyAllowlist = []string{ + "test-project-group", // "tazama", // "chiplet", // "cnab", @@ -99,6 +102,9 @@ type Config struct { // Logging Debug bool HTTPDebug bool + + // Data encoding + UseMsgpack bool } // LoadConfig loads configuration from environment variables @@ -118,11 +124,12 @@ func LoadConfig() (*Config, error) { Auth0ClientID: os.Getenv("AUTH0_CLIENT_ID"), Auth0PrivateKey: os.Getenv("AUTH0_PRIVATE_KEY"), // Other configuration - NATSURL: os.Getenv("NATS_URL"), - Port: os.Getenv("PORT"), - Bind: os.Getenv("BIND"), - Debug: os.Getenv("DEBUG") != "", - HTTPDebug: os.Getenv("HTTP_DEBUG") != "", + NATSURL: os.Getenv("NATS_URL"), + Port: os.Getenv("PORT"), + Bind: os.Getenv("BIND"), + Debug: parseBooleanEnv("DEBUG"), + HTTPDebug: parseBooleanEnv("HTTP_DEBUG"), + UseMsgpack: parseBooleanEnv("USE_MSGPACK"), } // Set defaults @@ -197,3 +204,19 @@ func LoadConfig() (*Config, error) { return cfg, nil } + +// parseBooleanEnv parses a boolean environment variable with common truthy values. +// Returns true if the value (case-insensitive) is "true", "yes", "t", "y", or "1". +// Returns false for any other value including empty string. +// +// Examples: +// - parseBooleanEnv("USE_MSGPACK") where USE_MSGPACK="true" returns true +// - parseBooleanEnv("USE_MSGPACK") where USE_MSGPACK="YES" returns true +// - parseBooleanEnv("USE_MSGPACK") where USE_MSGPACK="1" returns true +// - parseBooleanEnv("USE_MSGPACK") where USE_MSGPACK="false" returns false +// - parseBooleanEnv("USE_MSGPACK") where USE_MSGPACK="" returns false +func parseBooleanEnv(envVar string) bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(envVar))) + truthyValues := []string{"true", "yes", "t", "y", "1"} + return slices.Contains(truthyValues, value) +} diff --git a/cmd/lfx-v1-sync-helper/handlers.go b/cmd/lfx-v1-sync-helper/handlers.go index e1db10e..075ff3b 100644 --- a/cmd/lfx-v1-sync-helper/handlers.go +++ b/cmd/lfx-v1-sync-helper/handlers.go @@ -7,12 +7,19 @@ package main import ( "context" "encoding/json" + "fmt" "strings" + "time" "github.com/nats-io/nats.go/jetstream" "github.com/vmihailenco/msgpack/v5" ) +const ( + // tombstoneMarker is used to mark deleted mappings in the KV store. + tombstoneMarker = "!del" +) + // shouldSkipSync checks if the record was last modified by this service and // should be skipped, because it originated in v2, and therefore does not need // to be synced from v1. @@ -28,8 +35,9 @@ func shouldSkipSync(ctx context.Context, v1Data map[string]any) bool { return false } -// kvHandler processes KV bucket updates from Meltano -func kvHandler(entry jetstream.KeyValueEntry) { +// kvHandler processes KV bucket updates from Meltano. +// Returns true if the operation should be retried, false otherwise. +func kvHandler(entry jetstream.KeyValueEntry) bool { ctx := context.Background() key := entry.Key() @@ -40,16 +48,18 @@ func kvHandler(entry jetstream.KeyValueEntry) { // Handle different operations switch operation { case jetstream.KeyValuePut: - handleKVPut(ctx, entry) + return handleKVPut(ctx, entry) case jetstream.KeyValueDelete, jetstream.KeyValuePurge: - handleKVDelete(ctx, entry) + return handleKVDelete(ctx, entry) default: logger.With("key", key, "operation", operation.String()).DebugContext(ctx, "ignoring KV operation") + return false } } -// handleKVPut processes a KV put operation (create/update) -func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) { +// handleKVPut processes a KV put operation (create/update). +// Returns true if the operation should be retried, false otherwise. +func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) bool { key := entry.Key() // Parse the data (try JSON first, then msgpack) @@ -58,16 +68,22 @@ func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) { // JSON failed, try msgpack if msgErr := msgpack.Unmarshal(entry.Value(), &v1Data); msgErr != nil { logger.With(errKey, err, "msgpack_error", msgErr, "key", key).ErrorContext(ctx, "failed to unmarshal KV entry data as JSON or msgpack") - return + return false } logger.With("key", key).DebugContext(ctx, "successfully unmarshalled msgpack data") } else { logger.With("key", key).DebugContext(ctx, "successfully unmarshalled JSON data") } + // Check if this is a soft delete (record has _sdc_deleted_at field). + if deletedAt, exists := v1Data["_sdc_deleted_at"]; exists && deletedAt != nil && deletedAt != "" { + logger.With("key", key, "_sdc_deleted_at", deletedAt).InfoContext(ctx, "processing soft delete from WAL") + return handleKVSoftDelete(ctx, key, v1Data) + } + // Check if we should skip this sync operation. if shouldSkipSync(ctx, v1Data) { - return + return false } // Extract the prefix (everything before the first period) for faster lookup. @@ -80,46 +96,187 @@ func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) { switch prefix { case "salesforce-project__c": handleProjectUpdate(ctx, key, v1Data) + return false case "platform-collaboration__c": handleCommitteeUpdate(ctx, key, v1Data) + return false case "platform-community__c": handleCommitteeMemberUpdate(ctx, key, v1Data) + return false case "itx-zoom-meetings-v2": handleZoomMeetingUpdate(ctx, key, v1Data) + return false case "itx-zoom-meetings-registrants-v2": handleZoomMeetingRegistrantUpdate(ctx, key, v1Data) + return false case "itx-zoom-meetings-mappings-v2": handleZoomMeetingMappingUpdate(ctx, key, v1Data) + return false case "itx-zoom-meetings-invite-responses-v2": handleZoomMeetingInviteResponseUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings-attendees": handleZoomPastMeetingAttendeeUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings-invitees": handleZoomPastMeetingInviteeUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings-mappings": handleZoomPastMeetingMappingUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings-recordings": handleZoomPastMeetingRecordingUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings-summaries": handleZoomPastMeetingSummaryUpdate(ctx, key, v1Data) + return false case "itx-zoom-past-meetings": handleZoomPastMeetingUpdate(ctx, key, v1Data) + return false case "salesforce-merged_user": logger.With("key", key).DebugContext(ctx, "salesforce-merged_user sync not yet implemented") + return false case "salesforce-alternate_email__c": - handleAlternateEmailUpdate(ctx, key, v1Data) + return handleAlternateEmailUpdate(ctx, key, v1Data) default: logger.With("key", key).WarnContext(ctx, "unknown object type, ignoring") + return false } } -// handleKVDelete processes a KV delete operation -func handleKVDelete(ctx context.Context, entry jetstream.KeyValueEntry) { +// handleKVDelete processes a KV delete operation (hard delete from KV bucket). +// Returns true if the operation should be retried, false otherwise. +func handleKVDelete(ctx context.Context, entry jetstream.KeyValueEntry) bool { key := entry.Key() - // For deletes, we would need to look up the mapping and call delete APIs - // This is a simplified implementation - logger.With("key", key).InfoContext(ctx, "delete operation not yet implemented") + logger.With("key", key).InfoContext(ctx, "processing hard delete from KV bucket") + return handleResourceDelete(ctx, key, "") +} + +// handleKVSoftDelete processes a soft delete (record with _sdc_deleted_at field). +// Returns true if the operation should be retried, false otherwise. +func handleKVSoftDelete(ctx context.Context, key string, v1Data map[string]any) bool { + // Extract v1 principal for soft deletes. + v1Principal := extractV1Principal(ctx, v1Data) + return handleResourceDelete(ctx, key, v1Principal) +} + +// handleResourceDelete handles deletion of resources by key prefix with specified principal. +// Returns true if the operation should be retried, false otherwise. +func handleResourceDelete(ctx context.Context, key string, v1Principal string) bool { + // Extract the prefix (everything before the first period) for faster lookup. + prefix := key + if dotIndex := strings.Index(key, "."); dotIndex != -1 { + prefix = key[:dotIndex] + } + + // Extract SFID from key (everything after the first period). + sfid := "" + if dotIndex := strings.Index(key, "."); dotIndex != -1 && dotIndex < len(key)-1 { + sfid = key[dotIndex+1:] + } + + if sfid == "" { + logger.With("key", key).WarnContext(ctx, "cannot extract SFID from key for deletion") + return false + } + + // Determine the object type based on the key prefix and handle deletion. + switch prefix { + case "salesforce-project__c": + return handleProjectDelete(ctx, key, sfid, v1Principal) + case "platform-collaboration__c": + return handleCommitteeDelete(ctx, key, sfid, v1Principal) + case "platform-community__c": + return handleCommitteeMemberDelete(ctx, key, sfid, v1Principal) + case "salesforce-merged_user": + logger.With("key", key).DebugContext(ctx, "salesforce-merged_user delete not yet implemented") + return false + case "salesforce-alternate_email__c": + return handleAlternateEmailDelete(ctx, key, sfid, v1Principal) + case "itx-zoom-meetings-v2", "itx-zoom-meetings-registrants-v2", "itx-zoom-meetings-mappings-v2", "itx-zoom-meetings-invite-responses-v2", "itx-zoom-past-meetings-attendees", "itx-zoom-past-meetings-invitees", "itx-zoom-past-meetings-mappings", "itx-zoom-past-meetings-recordings", "itx-zoom-past-meetings-summaries", "itx-zoom-past-meetings": + logger.With("key", key).DebugContext(ctx, "meeting-related delete not yet implemented") + return false + default: + logger.With("key", key).WarnContext(ctx, "unknown object type for deletion, ignoring") + return false + } +} + +// tombstoneMapping stores a tombstone marker in the mapping KV store. +func tombstoneMapping(ctx context.Context, mappingKey string) error { + if _, err := mappingsKV.Put(ctx, mappingKey, []byte(tombstoneMarker)); err != nil { + return fmt.Errorf("failed to tombstone mapping %s: %w", mappingKey, err) + } + return nil +} + +// isTombstonedMapping checks if a mapping is tombstoned. +func isTombstonedMapping(mappingValue []byte) bool { + return string(mappingValue) == tombstoneMarker +} + +// extractV1Principal extracts the v1 principal from v1 data. +// For soft deletes, only uses lastmodifiedbyid if lastmodifieddate is within 1 second of _sdc_deleted_at. +// For upserts, returns lastmodifiedbyid immediately if _sdc_deleted_at is not present. +func extractV1Principal(ctx context.Context, v1Data map[string]any) string { + lastModifiedBy, hasModifiedBy := v1Data["lastmodifiedbyid"].(string) + + // If no lastmodifiedbyid, return empty (system principal). + if !hasModifiedBy || lastModifiedBy == "" { + return "" + } + + deletedAt, hasDeletedAt := v1Data["_sdc_deleted_at"] + + // If this is not a soft delete (no _sdc_deleted_at), return principal immediately. + if !hasDeletedAt || deletedAt == nil || deletedAt == "" { + logger.With("lastmodifiedbyid", lastModifiedBy). + DebugContext(ctx, "using v1 principal from upsert") + return lastModifiedBy + } + + // This is a soft delete - need to validate timestamps for safety. + lastModifiedDate, hasModifiedDate := v1Data["lastmodifieddate"].(string) + deletedAtStr, isDeletedAtString := deletedAt.(string) + + // If we don't have required timestamp fields for validation, fall back to system principal. + if !hasModifiedDate || !isDeletedAtString { + logger.With("has_modified_date", hasModifiedDate, "has_deleted_at_string", isDeletedAtString). + DebugContext(ctx, "missing timestamp fields for soft delete validation, using system principal") + return "" + } + + // Parse timestamps. + modifiedTime, err := parseTimestamp(lastModifiedDate) + if err != nil { + logger.With(errKey, err, "lastmodifieddate", lastModifiedDate). + WarnContext(ctx, "failed to parse lastmodifieddate, using system principal") + return "" + } + + deletedTime, err := parseTimestamp(deletedAtStr) + if err != nil { + logger.With(errKey, err, "_sdc_deleted_at", deletedAtStr). + WarnContext(ctx, "failed to parse _sdc_deleted_at, using system principal") + return "" + } + + // Check if timestamps are within 1 second of each other. + timeDiff := deletedTime.Sub(modifiedTime) + if timeDiff < 0 { + timeDiff = -timeDiff + } + + if timeDiff <= 1*time.Second { + logger.With("lastmodifiedbyid", lastModifiedBy, "time_diff_seconds", timeDiff.Seconds()). + DebugContext(ctx, "using v1 principal from soft delete") + return lastModifiedBy + } + + logger.With("lastmodifiedbyid", lastModifiedBy, "time_diff_seconds", timeDiff.Seconds()). + DebugContext(ctx, "timestamps too far apart, using system principal for soft delete") + return "" } // extractDateOnly extracts the date part from an ISO 8601 datetime string. diff --git a/cmd/lfx-v1-sync-helper/handlers_committees.go b/cmd/lfx-v1-sync-helper/handlers_committees.go index 9b5ea82..5995457 100644 --- a/cmd/lfx-v1-sync-helper/handlers_committees.go +++ b/cmd/lfx-v1-sync-helper/handlers_committees.go @@ -10,6 +10,7 @@ import ( "strings" committeeservice "github.com/linuxfoundation/lfx-v2-committee-service/gen/committee_service" + "github.com/nats-io/nats.go/jetstream" ) // allowedCommitteeCategories defines the valid values for type__c mapping to category. @@ -138,11 +139,8 @@ func handleCommitteeUpdate(ctx context.Context, key string, v1Data map[string]an return } - // Extract v1Principal from lastmodifiedbyid for JWT generation. - v1Principal := "" - if lastModifiedBy, ok := v1Data["lastmodifiedbyid"].(string); ok && lastModifiedBy != "" { - v1Principal = lastModifiedBy - } + // Extract v1Principal from v1 data for JWT generation. + v1Principal := extractV1Principal(ctx, v1Data) // Extract committee SFID. sfid, ok := v1Data["sfid"].(string) @@ -151,11 +149,22 @@ func handleCommitteeUpdate(ctx context.Context, key string, v1Data map[string]an return } + // Extract project SFID from v1 data for use in project checks and reverse mapping. + projectSFID := "" + if projSFID, ok := v1Data["project_name__c"].(string); ok && projSFID != "" { + projectSFID = projSFID + } + // Check if we have an existing mapping. + // Check if we have an existing mapping using SFID. mappingKey := fmt.Sprintf("committee.sfid.%s", sfid) existingUID := "" if entry, err := mappingsKV.Get(ctx, mappingKey); err == nil { + if isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid).WarnContext(ctx, "skipping committee upsert - mapping is tombstoned (previously deleted)") + return + } existingUID = string(entry.Value()) } @@ -178,7 +187,7 @@ func handleCommitteeUpdate(ctx context.Context, key string, v1Data map[string]an uid = existingUID } else { // Check if parent project exists in mappings before creating new committee. - if projectSFID, ok := v1Data["project_name__c"].(string); ok && projectSFID != "" { + if projectSFID != "" { projectMappingKey := fmt.Sprintf("project.sfid.%s", projectSFID) if _, err := mappingsKV.Get(ctx, projectMappingKey); err != nil { logger.With("project_sfid", projectSFID, "committee_sfid", sfid).InfoContext(ctx, "skipping committee creation - parent project not found in mappings") @@ -209,16 +218,127 @@ func handleCommitteeUpdate(ctx context.Context, key string, v1Data map[string]an return } - // Store the mapping. + // Store the SFID mapping and reverse mapping. if uid != "" { if _, err := mappingsKV.Put(ctx, mappingKey, []byte(uid)); err != nil { logger.With(errKey, err, "sfid", sfid, "uid", uid).WarnContext(ctx, "failed to store committee mapping") } + + // Store reverse mapping (v2 UID -> v1 project:committee SFID). + reverseMappingKey := fmt.Sprintf("committee.uid.%s", uid) + reverseMappingValue := fmt.Sprintf("%s:%s", projectSFID, sfid) + if _, err := mappingsKV.Put(ctx, reverseMappingKey, []byte(reverseMappingValue)); err != nil { + logger.With(errKey, err, "committee_uid", uid, "sfid", sfid).WarnContext(ctx, "failed to store committee reverse mapping") + } } logger.With("committee_uid", uid, "sfid", sfid).InfoContext(ctx, "successfully synced committee") } +// handleCommitteeMemberDelete processes a committee member deletion. +// Returns true if the operation should be retried, false otherwise. +func handleCommitteeMemberDelete(ctx context.Context, key string, sfid string, v1Principal string) bool { + // Check if we have an existing mapping using SFID. + mappingKey := fmt.Sprintf("committee_member.sfid.%s", sfid) + entry, err := mappingsKV.Get(ctx, mappingKey) + if err != nil { + if err == jetstream.ErrKeyNotFound { + logger.With("sfid", sfid, "key", key).InfoContext(ctx, "committee member mapping not found, nothing to delete") + return false + } + logger.With(errKey, err, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to get committee member mapping for deletion") + return true // Retry on error. + } + + mappingValue := string(entry.Value()) + if mappingValue == "" || isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid, "key", key).InfoContext(ctx, "committee member mapping empty or tombstoned, nothing to delete") + return false + } + + // Parse the new mapping format: "{committee_uuid}:{member_uuid}". + parts := strings.Split(mappingValue, ":") + if len(parts) != 2 { + // Old format (no committee ID) means we cannot delete the committee member. Tombstone it anyhow.. + logger.With("member_uid", mappingValue, "sfid", sfid, "key", key).WarnContext(ctx, "committee member deletion with old mapping format, deletion cannot be synced") + if err := tombstoneMapping(ctx, mappingKey); err != nil { + logger.With(errKey, err, "sfid", sfid).WarnContext(ctx, "failed to tombstone old format committee member mapping") + } + return false + } + + committeeUID := parts[0] + memberUID := parts[1] + + // Delete the committee member using the API. + logger.With("committee_uid", committeeUID, "member_uid", memberUID, "sfid", sfid, "key", key, "v1_principal", v1Principal).InfoContext(ctx, "deleting committee member") + + err = deleteCommitteeMember(ctx, committeeUID, memberUID, v1Principal) + if err != nil { + logger.With(errKey, err, "committee_uid", committeeUID, "member_uid", memberUID, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to delete committee member") + return true // Retry on error. + } + + // Tombstone mappings after successful deletion. + if err := tombstoneMapping(ctx, mappingKey); err != nil { + logger.With(errKey, err, "sfid", sfid).WarnContext(ctx, "failed to tombstone committee member SFID mapping") + } + + // Tombstone reverse mapping (member UID -> v1 project:committee:member SFID). + reverseMappingKey := fmt.Sprintf("committee_member.uid.%s", memberUID) + if err := tombstoneMapping(ctx, reverseMappingKey); err != nil { + logger.With(errKey, err, "committee_uid", committeeUID, "member_uid", memberUID).WarnContext(ctx, "failed to tombstone committee member UID mapping") + } + + logger.With("committee_uid", committeeUID, "member_uid", memberUID, "sfid", sfid, "key", key).InfoContext(ctx, "successfully deleted committee member") + return false +} + +// handleCommitteeDelete processes a committee deletion. +// Returns true if the operation should be retried, false otherwise. +func handleCommitteeDelete(ctx context.Context, key string, sfid string, v1Principal string) bool { + // Check if we have an existing mapping using SFID. + mappingKey := fmt.Sprintf("committee.sfid.%s", sfid) + entry, err := mappingsKV.Get(ctx, mappingKey) + if err != nil { + if err == jetstream.ErrKeyNotFound { + logger.With("sfid", sfid, "key", key).WarnContext(ctx, "committee mapping not found, nothing to delete") + return false + } + logger.With(errKey, err, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to get committee mapping for deletion") + return true // Retry on error. + } + + existingUID := string(entry.Value()) + if existingUID == "" || isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid, "key", key).InfoContext(ctx, "committee mapping empty or tombstoned, nothing to delete") + return false + } + + // Delete the committee using provided v1Principal or v1-sync-helper service credentials. + logger.With("committee_uid", existingUID, "sfid", sfid, "key", key, "v1_principal", v1Principal).InfoContext(ctx, "deleting committee") + + err = deleteCommittee(ctx, existingUID, v1Principal) + if err != nil { + logger.With(errKey, err, "committee_uid", existingUID, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to delete committee") + return true // Retry on error. + } + + // Tombstone mappings after successful deletion. + if err := tombstoneMapping(ctx, mappingKey); err != nil { + logger.With(errKey, err, "sfid", sfid, "committee_uid", existingUID).WarnContext(ctx, "failed to tombstone committee SFID mapping") + } + + // Also tombstone reverse mapping (v2 UID -> v1 SFID). + reverseMappingKey := fmt.Sprintf("committee.uid.%s", existingUID) + if err := tombstoneMapping(ctx, reverseMappingKey); err != nil { + logger.With(errKey, err, "committee_uid", existingUID, "sfid", sfid).WarnContext(ctx, "failed to tombstone committee UID mapping") + } + + logger.With("committee_uid", existingUID, "sfid", sfid, "key", key).InfoContext(ctx, "successfully deleted committee") + return false +} + // mapV1DataToCommitteeCreatePayload converts v1 committee data to a CreateCommitteePayload. func mapV1DataToCommitteeCreatePayload(ctx context.Context, v1Data map[string]any) (*committeeservice.CreateCommitteePayload, error) { // Extract required fields. @@ -364,11 +484,8 @@ func handleCommitteeMemberUpdate(ctx context.Context, key string, v1Data map[str return } - // Extract v1Principal from lastmodifiedbyid for JWT generation. - v1Principal := "" - if lastModifiedBy, ok := v1Data["lastmodifiedbyid"].(string); ok && lastModifiedBy != "" { - v1Principal = lastModifiedBy - } + // Extract v1Principal from v1 data for JWT generation. + v1Principal := extractV1Principal(ctx, v1Data) // Extract committee member SFID. sfid, ok := v1Data["sfid"].(string) @@ -407,12 +524,29 @@ func handleCommitteeMemberUpdate(ctx context.Context, key string, v1Data map[str committeeUID := string(committeeEntry.Value()) logger.With("collaboration_sfid", collaborationNameV1, "committee_uid", committeeUID).DebugContext(ctx, "found committee UID from committee SFID mapping") - // Check if we have an existing member mapping. + // Check if we have an existing mapping. memberMappingKey := fmt.Sprintf("committee_member.sfid.%s", sfid) existingMemberUID := "" + needsFormatUpgrade := false if entry, err := mappingsKV.Get(ctx, memberMappingKey); err == nil { - existingMemberUID = string(entry.Value()) + if isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid).WarnContext(ctx, "skipping committee member upsert - mapping is tombstoned (previously deleted)") + return + } + mappingValue := string(entry.Value()) + // Check if it's new format (committee:member) or old format (just member). + if strings.Contains(mappingValue, ":") { + // New format: "{committee_uuid}:{member_uuid}". + parts := strings.Split(mappingValue, ":") + if len(parts) == 2 { + existingMemberUID = parts[1] + } + } else { + // Old format: just member UID - needs upgrade. + existingMemberUID = mappingValue + needsFormatUpgrade = true + } } var memberUID string @@ -457,9 +591,26 @@ func handleCommitteeMemberUpdate(ctx context.Context, key string, v1Data map[str } // Store the member mapping. + // Store the mapping in new format: "{committee_uuid}:{member_uuid}". if memberUID != "" { - if _, err := mappingsKV.Put(ctx, memberMappingKey, []byte(memberUID)); err != nil { + newMappingValue := fmt.Sprintf("%s:%s", committeeUID, memberUID) + if _, err := mappingsKV.Put(ctx, memberMappingKey, []byte(newMappingValue)); err != nil { logger.With(errKey, err, "sfid", sfid, "member_uid", memberUID).WarnContext(ctx, "failed to store committee member mapping") + } else if needsFormatUpgrade { + logger.With("sfid", sfid, "member_uid", memberUID, "committee_uid", committeeUID).InfoContext(ctx, "upgraded committee member mapping to new format") + } + + // Store reverse mapping (committee:member UID -> v1 project:committee:member SFID). + // Extract project SFID from v1 data for reverse mapping. + projectSFID := "" + if projSFID, ok := v1Data["project_name__c"].(string); ok && projSFID != "" { + projectSFID = projSFID + } + + reverseMappingKey := fmt.Sprintf("committee_member.uid.%s", memberUID) + reverseMappingValue := fmt.Sprintf("%s:%s:%s", projectSFID, collaborationNameV1, sfid) + if _, err := mappingsKV.Put(ctx, reverseMappingKey, []byte(reverseMappingValue)); err != nil { + logger.With(errKey, err, "committee_uid", committeeUID, "member_uid", memberUID).WarnContext(ctx, "failed to store committee member reverse mapping") } } @@ -755,15 +906,6 @@ func mapV1DataToCommitteeMemberUpdatePayload(ctx context.Context, committeeUID s } } - // Map GAC-specific fields. - if country, ok := v1Data["country"].(string); ok && country != "" { - payload.Country = &country - } - - if agency, ok := v1Data["agency"].(string); ok && agency != "" { - payload.Agency = &agency - } - // Map organization information. if accountSFID, ok := v1Data["account__c"].(string); ok && accountSFID != "" { // Look up organization information from v1 Organization Service. diff --git a/cmd/lfx-v1-sync-helper/handlers_meetings.go b/cmd/lfx-v1-sync-helper/handlers_meetings.go index 0016387..74295a3 100644 --- a/cmd/lfx-v1-sync-helper/handlers_meetings.go +++ b/cmd/lfx-v1-sync-helper/handlers_meetings.go @@ -847,7 +847,7 @@ func convertMapToInputInviteResponse(v1Data map[string]any) (*inviteResponseInpu func getInviteResponseTags(inviteResponse *inviteResponseInput) []string { tags := []string{ - fmt.Sprintf("%s", inviteResponse.ID), + inviteResponse.ID, fmt.Sprintf("invite_response_uid:%s", inviteResponse.ID), fmt.Sprintf("meeting_uid:%s", inviteResponse.MeetingID), fmt.Sprintf("registrant_uid:%s", inviteResponse.RegistrantID), @@ -1384,7 +1384,7 @@ func convertMapToInputPastMeetingInvitee(v1Data map[string]any) (*pastMeetingInv func getPastMeetingParticipantTags(participant *V2PastMeetingParticipant) []string { tags := []string{ - fmt.Sprintf("%s", participant.UID), + participant.UID, fmt.Sprintf("past_meeting_participant_uid:%s", participant.UID), fmt.Sprintf("past_meeting_uid:%s", participant.PastMeetingUID), fmt.Sprintf("meeting_uid:%s", participant.MeetingUID), @@ -1955,7 +1955,7 @@ func convertMapToInputPastMeetingRecording(v1Data map[string]any) (*pastMeetingR func getPastMeetingRecordingTags(recording *pastMeetingRecordingInput) []string { tags := []string{ - fmt.Sprintf("%s", recording.UID), + recording.UID, fmt.Sprintf("past_meeting_recording_uid:%s", recording.UID), fmt.Sprintf("past_meeting_uid:%s", recording.PastMeetingUID), "platform:Zoom", @@ -1972,7 +1972,7 @@ func getPastMeetingRecordingTags(recording *pastMeetingRecordingInput) []string // Ultimately they are indexed as separate records, so they need their own tags. func getPastMeetingTranscriptTags(recording *pastMeetingRecordingInput) []string { tags := []string{ - fmt.Sprintf("%s", recording.UID), + recording.UID, fmt.Sprintf("past_meeting_transcript_uid:%s", recording.UID), fmt.Sprintf("past_meeting_uid:%s", recording.PastMeetingUID), "platform:Zoom", @@ -2175,7 +2175,7 @@ func convertMapToInputPastMeetingSummary(v1Data map[string]any) (*pastMeetingSum func getPastMeetingSummaryTags(summary *pastMeetingSummaryInput) []string { tags := []string{ - fmt.Sprintf("%s", summary.UID), + summary.UID, fmt.Sprintf("past_meeting_summary_uid:%s", summary.UID), fmt.Sprintf("past_meeting_uid:%s", summary.PastMeetingUID), fmt.Sprintf("meeting_uid:%s", summary.MeetingUID), diff --git a/cmd/lfx-v1-sync-helper/handlers_projects.go b/cmd/lfx-v1-sync-helper/handlers_projects.go index 3f892e5..efcd6c2 100644 --- a/cmd/lfx-v1-sync-helper/handlers_projects.go +++ b/cmd/lfx-v1-sync-helper/handlers_projects.go @@ -11,6 +11,7 @@ import ( "strings" projectservice "github.com/linuxfoundation/lfx-v2-project-service/api/project/v1/gen/project_service" + "github.com/nats-io/nats.go/jetstream" ) // isValidURL checks if a URL value is non-empty and not "nil". @@ -130,11 +131,8 @@ func handleProjectUpdate(ctx context.Context, key string, v1Data map[string]any) return } - // Extract v1Principal from lastmodifiedbyid for JWT generation. - v1Principal := "" - if lastModifiedBy, ok := v1Data["lastmodifiedbyid"].(string); ok && lastModifiedBy != "" { - v1Principal = lastModifiedBy - } + // Extract v1Principal from v1 data for JWT generation. + v1Principal := extractV1Principal(ctx, v1Data) // Extract project SFID (primary key). sfid, ok := v1Data["sfid"].(string) @@ -151,6 +149,10 @@ func handleProjectUpdate(ctx context.Context, key string, v1Data map[string]any) existingUID := "" if entry, err := mappingsKV.Get(ctx, mappingKey); err == nil { + if isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid, "slug", slug).WarnContext(ctx, "skipping project upsert - mapping is tombstoned (previously deleted)") + return + } existingUID = string(entry.Value()) } @@ -210,16 +212,67 @@ func handleProjectUpdate(ctx context.Context, key string, v1Data map[string]any) return } - // Store the SFID mapping. + // Store the SFID mapping and reverse mapping. if uid != "" { if _, err := mappingsKV.Put(ctx, mappingKey, []byte(uid)); err != nil { logger.With(errKey, err, "sfid", sfid, "uid", uid).WarnContext(ctx, "failed to store project mapping") } + + // Store reverse mapping (v2 UID -> v1 SFID). + reverseMappingKey := fmt.Sprintf("project.uid.%s", uid) + if _, err := mappingsKV.Put(ctx, reverseMappingKey, []byte(sfid)); err != nil { + logger.With(errKey, err, "project_uid", uid, "sfid", sfid).WarnContext(ctx, "failed to store project reverse mapping") + } } logger.With("project_uid", uid, "sfid", sfid, "slug", slug).InfoContext(ctx, "successfully synced project") } +// handleProjectDelete processes a project deletion. +// Returns true if the operation should be retried, false otherwise. +func handleProjectDelete(ctx context.Context, key string, sfid string, v1Principal string) bool { + // Check if we have an existing mapping using SFID. + mappingKey := fmt.Sprintf("project.sfid.%s", sfid) + entry, err := mappingsKV.Get(ctx, mappingKey) + if err != nil { + if err == jetstream.ErrKeyNotFound { + logger.With("sfid", sfid, "key", key).InfoContext(ctx, "project mapping not found, nothing to delete") + return false + } + logger.With(errKey, err, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to get project mapping for deletion") + return true // Retry on error. + } + + existingUID := string(entry.Value()) + if existingUID == "" || isTombstonedMapping(entry.Value()) { + logger.With("sfid", sfid, "key", key).InfoContext(ctx, "project mapping empty or tombstoned, nothing to delete") + return false + } + + // Delete the project using provided v1Principal or v1-sync-helper service credentials. + logger.With("project_uid", existingUID, "sfid", sfid, "key", key, "v1_principal", v1Principal).InfoContext(ctx, "deleting project") + + err = deleteProject(ctx, existingUID, v1Principal) + if err != nil { + logger.With(errKey, err, "project_uid", existingUID, "sfid", sfid, "key", key).ErrorContext(ctx, "failed to delete project") + return true // Retry on error. + } + + // Tombstone mappings after successful deletion. + if err := tombstoneMapping(ctx, mappingKey); err != nil { + logger.With(errKey, err, "sfid", sfid, "project_uid", existingUID).WarnContext(ctx, "failed to tombstone project SFID mapping") + } + + // Also tombstone reverse mapping (v2 UID -> v1 SFID). + reverseMappingKey := fmt.Sprintf("project.uid.%s", existingUID) + if err := tombstoneMapping(ctx, reverseMappingKey); err != nil { + logger.With(errKey, err, "project_uid", existingUID, "sfid", sfid).WarnContext(ctx, "failed to tombstone project UID mapping") + } + + logger.With("project_uid", existingUID, "sfid", sfid, "key", key).InfoContext(ctx, "successfully deleted project") + return false +} + // mapV1DataToProjectCreatePayload converts v1 project data to a CreateProjectPayload. func mapV1DataToProjectCreatePayload(ctx context.Context, v1Data map[string]any) (*projectservice.CreateProjectPayload, error) { // Extract required fields. diff --git a/cmd/lfx-v1-sync-helper/handlers_users.go b/cmd/lfx-v1-sync-helper/handlers_users.go index 4e27225..4b2607c 100644 --- a/cmd/lfx-v1-sync-helper/handlers_users.go +++ b/cmd/lfx-v1-sync-helper/handlers_users.go @@ -8,27 +8,26 @@ import ( "context" "encoding/json" "fmt" - "math/rand/v2" - "time" "github.com/nats-io/nats.go/jetstream" ) // handleAlternateEmailUpdate processes alternate email updates and maintains // v1-mapping records for merged users' alternate emails. -func handleAlternateEmailUpdate(ctx context.Context, key string, v1Data map[string]any) { +// Returns true if the operation should be retried, false otherwise. +func handleAlternateEmailUpdate(ctx context.Context, key string, v1Data map[string]any) bool { // Extract the leadorcontactid which references the sfid of merged_user table. leadorcontactid, ok := v1Data["leadorcontactid"].(string) if !ok || leadorcontactid == "" { logger.With("key", key).WarnContext(ctx, "alternate email missing leadorcontactid, skipping") - return + return false } // Extract the sfid of this alternate email record. emailSfid, ok := v1Data["sfid"].(string) if !ok || emailSfid == "" { logger.With("key", key).WarnContext(ctx, "alternate email missing sfid, skipping") - return + return false } // Check if this email is deleted. @@ -37,102 +36,86 @@ func handleAlternateEmailUpdate(ctx context.Context, key string, v1Data map[stri isDeleted = deletedVal } - // Process the update in a goroutine to avoid blocking other handlers. - go func() { - updateUserAlternateEmails(context.WithoutCancel(ctx), leadorcontactid, emailSfid, isDeleted) - }() + // Process the update synchronously and return retry status. + return updateUserAlternateEmails(ctx, leadorcontactid, emailSfid, isDeleted) } // updateUserAlternateEmails updates the v1-mapping record for a user's alternate emails // with concurrency control using atomic KV operations. -func updateUserAlternateEmails(ctx context.Context, userSfid, emailSfid string, isDeleted bool) { +// Returns true if the operation should be retried, false otherwise. +func updateUserAlternateEmails(ctx context.Context, userSfid, emailSfid string, isDeleted bool) bool { mappingKey := fmt.Sprintf("v1-merged-user.alternate-emails.%s", userSfid) - maxRetries := 5 - - for attempt := 1; attempt <= maxRetries; attempt++ { - // Add random splay time up to 1 second to reduce collision chances. - splayTime := time.Duration(rand.IntN(1000)) * time.Millisecond - time.Sleep(splayTime) - - // Get current mapping record. - entry, err := mappingsKV.Get(ctx, mappingKey) - - var currentEmails []string - var revision uint64 - - if err != nil { - if err == jetstream.ErrKeyNotFound { - // Key doesn't exist, we'll create it. - currentEmails = []string{} - revision = 0 - } else { - logger.With("error", err, "key", mappingKey, "attempt", attempt). - ErrorContext(ctx, "failed to get mapping record") - if attempt == maxRetries { - return - } - continue - } - } else { - // Parse existing emails list. - revision = entry.Revision() - if err := json.Unmarshal(entry.Value(), ¤tEmails); err != nil { - logger.With("error", err, "key", mappingKey). - ErrorContext(ctx, "failed to unmarshal existing emails list") - return - } - } - // Update the emails list. - updatedEmails := updateEmailsList(currentEmails, emailSfid, isDeleted) + // Get current mapping record. + entry, err := mappingsKV.Get(ctx, mappingKey) - // Marshal the updated list. - updatedData, err := json.Marshal(updatedEmails) - if err != nil { - logger.With("error", err, "key", mappingKey). - ErrorContext(ctx, "failed to marshal updated emails list") - return - } + var currentEmails []string + var revision uint64 - // Attempt to save with concurrency control. - var saveErr error - if revision == 0 { - // Try to create new record. - _, saveErr = mappingsKV.Create(ctx, mappingKey, updatedData) - if saveErr == jetstream.ErrKeyExists { - // Key was created by another process, retry. - logger.With("key", mappingKey, "attempt", attempt). - DebugContext(ctx, "key created by another process during create attempt, retrying") - continue - } + if err != nil { + if err == jetstream.ErrKeyNotFound { + // Key doesn't exist, we'll create it. + currentEmails = []string{} + revision = 0 } else { - // Try to update existing record. - _, saveErr = mappingsKV.Update(ctx, mappingKey, updatedData, revision) - if saveErr != nil { - // Update failed (likely revision mismatch), retry. - logger.With("error", saveErr, "key", mappingKey, "attempt", attempt). - DebugContext(ctx, "update failed, retrying") - continue - } + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to get mapping record") + return false } - - if saveErr == nil { - // Success! - logger.With("key", mappingKey, "emailSfid", emailSfid, "isDeleted", isDeleted, "attempt", attempt). - DebugContext(ctx, "successfully updated alternate emails mapping") - return + } else { + // Parse existing emails list. + revision = entry.Revision() + if err := json.Unmarshal(entry.Value(), ¤tEmails); err != nil { + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to unmarshal existing emails list") + return false } + } - // If we get here, there was an unexpected error. - logger.With("error", saveErr, "key", mappingKey, "attempt", attempt). - WarnContext(ctx, "unexpected error during save operation") + // Update the emails list. + updatedEmails := updateEmailsList(currentEmails, emailSfid, isDeleted) - if attempt == maxRetries { - logger.With("key", mappingKey, "maxRetries", maxRetries). - ErrorContext(ctx, "max retries exceeded for updating alternate emails mapping") - return + // Marshal the updated list. + updatedData, err := json.Marshal(updatedEmails) + if err != nil { + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to marshal updated emails list") + return false + } + + // Attempt to save with concurrency control. + if revision == 0 { + // Try to create new record. + if _, err := mappingsKV.Create(ctx, mappingKey, updatedData); err != nil { + // Check if this is a revision mismatch (key already exists) that should be retried. + if isRevisionMismatchError(err) || err == jetstream.ErrKeyExists { + logger.With("error", err, "key", mappingKey). + WarnContext(ctx, "key created by another process during create attempt, will retry") + return true + } + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to create mapping record") + return false + } + } else { + // Try to update existing record. + if _, err := mappingsKV.Update(ctx, mappingKey, updatedData, revision); err != nil { + // Check if this is a revision mismatch that should be retried. + if isRevisionMismatchError(err) { + logger.With("error", err, "key", mappingKey, "revision", revision). + WarnContext(ctx, "mapping record revision mismatch, will retry") + return true + } + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to update mapping record") + return false } } + + // Success! + logger.With("key", mappingKey, "emailSfid", emailSfid, "isDeleted", isDeleted). + DebugContext(ctx, "successfully updated alternate emails mapping") + return false } // updateEmailsList adds or removes an email sfid from the list based on deletion status. @@ -162,3 +145,22 @@ func updateEmailsList(currentEmails []string, emailSfid string, isDeleted bool) // Email already in list, nothing to add. return currentEmails } + +// handleAlternateEmailDelete processes an alternate email deletion. +// Returns true if the operation should be retried, false otherwise. +func handleAlternateEmailDelete(ctx context.Context, key string, sfid string, v1Principal string) bool { + // Extract user SFID from the v1 data. + // For alternate emails, we need to parse the relationship to find the user. + // This would typically require looking up the email record to get the user SFID. + + logger.With("key", key, "sfid", sfid).InfoContext(ctx, "alternate email deletion - processing email removal") + + // For now, we'll mark this as handled but not fully implemented. + // A complete implementation would need to: + // 1. Look up which user this alternate email belongs to + // 2. Call updateUserAlternateEmails with isDeleted=true + // 3. Clean up any mappings + + logger.With("key", key, "sfid", sfid, "v1_principal", v1Principal).WarnContext(ctx, "alternate email deletion not fully implemented") + return false +} diff --git a/cmd/lfx-v1-sync-helper/ingest_wal.go b/cmd/lfx-v1-sync-helper/ingest_wal.go new file mode 100644 index 0000000..bab2d54 --- /dev/null +++ b/cmd/lfx-v1-sync-helper/ingest_wal.go @@ -0,0 +1,448 @@ +// Copyright The Linux Foundation and each contributor to LFX. +// SPDX-License-Identifier: MIT + +// The lfx-v1-sync-helper service. +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/vmihailenco/msgpack/v5" +) + +// ActionKind represents the type of WAL action, matching wal-listener's internal ActionKind. +type ActionKind string + +// WAL action types - these constants match the wal-listener's ActionKind values. +const ( + ActionInsert ActionKind = "INSERT" + ActionUpdate ActionKind = "UPDATE" + ActionDelete ActionKind = "DELETE" + ActionTruncate ActionKind = "TRUNCATE" +) + +// String returns the string representation of the ActionKind. +func (k ActionKind) String() string { + return string(k) +} + +// WALEvent represents the structure of a WAL listener event received from the wal_listener stream. +// This structure matches the JSON payload format emitted by the wal-listener service when +// PostgreSQL WAL changes are detected. +type WALEvent struct { + ID string `json:"id"` // Unique event ID + Schema string `json:"schema"` // Database schema name (e.g., "platform") + Table string `json:"table"` // Table name (e.g., "community__c") + Action string `json:"action"` // Action type: "INSERT", "UPDATE", or "DELETE" + Data map[string]interface{} `json:"data"` // New/current record data (empty for DELETE) + DataOld map[string]interface{} `json:"dataOld"` // Previous record data (used for DELETE operations) + CommitTime string `json:"commitTime"` // Transaction commit timestamp +} + +// ActionKind returns the parsed ActionKind from the Action field. +func (w *WALEvent) ActionKind() ActionKind { + return ActionKind(strings.ToUpper(w.Action)) +} + +// IsValid checks if the WAL event has the minimum required fields. +func (w *WALEvent) IsValid() bool { + return w.Schema != "" && w.Table != "" && w.Action != "" +} + +// GetSFID extracts the SFID from the appropriate data field based on the action. +// For DELETE actions, it looks in DataOld; for others, it looks in Data. +func (w *WALEvent) GetSFID() (string, bool) { + var dataSource map[string]interface{} + + switch w.ActionKind() { + case ActionDelete: + dataSource = w.DataOld + default: + dataSource = w.Data + } + + if dataSource == nil { + return "", false + } + + sfidValue, exists := dataSource["sfid"] + if !exists || sfidValue == nil { + return "", false + } + + sfid := fmt.Sprintf("%v", sfidValue) + return sfid, sfid != "" +} + +// walIngestHandler processes WAL listener events from the wal_listener stream. +// It handles INSERT, UPDATE, and DELETE operations by upserting or marking +// records as deleted in the v1-objects KV bucket. This enables real-time +// synchronization of PostgreSQL changes to the KV store for downstream consumption. +// Handles ACK/NAK logic internally based on retry conditions. +func walIngestHandler(msg jetstream.Msg) { + ctx := context.Background() + + subject := msg.Subject() + logger.With("subject", subject).DebugContext(ctx, "received WAL listener message") + + // Parse the WAL event. + var walEvent WALEvent + if err := json.Unmarshal(msg.Data(), &walEvent); err != nil { + logger.With(errKey, err, "subject", subject).ErrorContext(ctx, "failed to unmarshal WAL event") + if ackErr := msg.Ack(); ackErr != nil { + logger.With(errKey, ackErr, "subject", subject).Error("failed to acknowledge WAL JetStream message") + } + return + } + + // Validate the WAL event. + if !walEvent.IsValid() { + logger.With("subject", subject, "event", walEvent).WarnContext(ctx, "invalid WAL event, missing required fields") + if ackErr := msg.Ack(); ackErr != nil { + logger.With(errKey, ackErr, "subject", subject).Error("failed to acknowledge WAL JetStream message") + } + return + } + + // Log the event details. + logger.With( + "subject", subject, + "action", walEvent.Action, + "table", walEvent.Table, + "schema", walEvent.Schema, + ).DebugContext(ctx, "processing WAL event") + + // Handle different actions using typed constants. + var shouldRetry bool + switch walEvent.ActionKind() { + case ActionInsert, ActionUpdate: + shouldRetry = handleWALUpsert(ctx, &walEvent) + case ActionDelete: + shouldRetry = handleWALDelete(ctx, &walEvent) + case ActionTruncate: + logger.With("action", walEvent.Action, "table", walEvent.Table).DebugContext(ctx, "truncate action not supported, ignoring") + shouldRetry = false + default: + logger.With("action", walEvent.Action, "table", walEvent.Table).WarnContext(ctx, "unknown WAL action, ignoring") + shouldRetry = false + } + + // Handle message acknowledgment based on retry decision. + if shouldRetry { + // NAK the message to trigger retry. + if err := msg.Nak(); err != nil { + logger.With(errKey, err, "subject", subject).Error("failed to NAK WAL JetStream message for retry") + } else { + logger.With("subject", subject).Debug("NAKed WAL message for retry") + } + } else { + // Acknowledge the message. + if err := msg.Ack(); err != nil { + logger.With(errKey, err, "subject", subject).Error("failed to acknowledge WAL JetStream message") + } + } +} + +// handleWALUpsert processes INSERT and UPDATE WAL events by upserting to v1-objects KV bucket. +// It dynamically constructs KV keys using the format "{schema}-{table}.{sfid}" (e.g., "platform-community__c.{sfid}"). +// It encodes data using the configured format (JSON by default, or MessagePack if cfg.UseMsgpack is true). +// It implements conditional upsert logic that only updates records if EITHER systemmodstamp +// OR lastmodifieddate in the new data is later than the existing record. This ensures +// that only newer changes are propagated while avoiding unnecessary updates. +// Returns true if the operation should be retried (only for KV revision mismatches), false otherwise. +func handleWALUpsert(ctx context.Context, walEvent *WALEvent) bool { + // Extract the SFID using the helper method. + sfid, exists := walEvent.GetSFID() + if !exists { + logger.With("table", walEvent.Table, "action", walEvent.Action).WarnContext(ctx, "WAL event missing or empty sfid field, skipping") + return false + } + + // Construct the key based on schema and table name. + keyPrefix := fmt.Sprintf("%s-%s", walEvent.Schema, walEvent.Table) + key := fmt.Sprintf("%s.%s", keyPrefix, sfid) + + // Check if the key already exists in the KV bucket. + existing, err := v1KV.Get(ctx, key) + if err != nil && err != jetstream.ErrKeyNotFound { + logger.With(errKey, err, "key", key).ErrorContext(ctx, "failed to get existing KV entry") + return false + } + + var shouldUpdate bool + var lastRevision uint64 + + if err == jetstream.ErrKeyNotFound { + // Key doesn't exist, create it. + shouldUpdate = true + } else { + // Key exists, check if we should update. + lastRevision = existing.Revision() + + // Parse existing data. + var existingData map[string]interface{} + if unmarshalErr := json.Unmarshal(existing.Value(), &existingData); unmarshalErr != nil { + // Try msgpack if JSON fails. + if msgpackErr := msgpack.Unmarshal(existing.Value(), &existingData); msgpackErr != nil { + logger.With(errKey, unmarshalErr, "msgpack_error", msgpackErr, "key", key).ErrorContext(ctx, "failed to unmarshal existing KV entry data") + return false + } + } + + // Check if the record was marked as deleted in the target. + if deletedAt, exists := existingData["_sdc_deleted_at"]; exists && deletedAt != nil { + logger.With("key", key).WarnContext(ctx, "skipping WAL upsert for deleted record") + return false + } + + // Compare timestamps to determine if we should update. + shouldUpdate = shouldUpdateBasedOnTimestamps(ctx, walEvent.Data, existingData, key) + } + + if shouldUpdate { + // Add metadata fields. + walEvent.Data["_sdc_extracted_at"] = walEvent.CommitTime + walEvent.Data["_sdc_received_at"] = time.Now().UTC().Format(time.RFC3339) + + // Encode the data using configured format (JSON or MessagePack). + var dataBytes []byte + var err error + if cfg.UseMsgpack { + dataBytes, err = msgpack.Marshal(walEvent.Data) + } else { + dataBytes, err = json.Marshal(walEvent.Data) + } + if err != nil { + logger.With(errKey, err, "key", key, "use_msgpack", cfg.UseMsgpack).ErrorContext(ctx, "failed to marshal WAL data") + return false + } + + if lastRevision == 0 { + // Create new entry. + if _, err := v1KV.Create(ctx, key, dataBytes); err != nil { + // Check if this is a revision mismatch (key already exists) that should be retried. + if isRevisionMismatchError(err) { + logger.With(errKey, err, "key", key).WarnContext(ctx, "KV create failed due to existing key, will retry") + return true + } + logger.With(errKey, err, "key", key).ErrorContext(ctx, "failed to create KV entry from WAL event") + return false + } + logger.With("key", key, "action", walEvent.Action, "encoding", getEncodingFormat()).InfoContext(ctx, "created KV entry from WAL event") + } else { + // Update existing entry. + if _, err := v1KV.Update(ctx, key, dataBytes, lastRevision); err != nil { + // Check if this is a revision mismatch that should be retried. + if isRevisionMismatchError(err) { + logger.With(errKey, err, "key", key, "revision", lastRevision).WarnContext(ctx, "KV revision mismatch, will retry") + return true + } + logger.With(errKey, err, "key", key, "revision", lastRevision).ErrorContext(ctx, "failed to update KV entry from WAL event") + return false + } + logger.With("key", key, "action", walEvent.Action, "revision", lastRevision, "encoding", getEncodingFormat()).InfoContext(ctx, "updated KV entry from WAL event") + } + } else { + logger.With("key", key, "action", walEvent.Action).DebugContext(ctx, "skipping WAL upsert - existing data is newer or same") + } + + return false +} + +// handleWALDelete processes DELETE WAL events by marking records as deleted in v1-objects KV bucket. +// It uses the same dynamic key format "{schema}-{table}.{sfid}" to locate existing records. +// It encodes the updated data using the configured format (JSON by default, or MessagePack if cfg.UseMsgpack is true). +// Instead of removing the record entirely, it adds a "_sdc_deleted_at" timestamp field +// to maintain an audit trail and allow downstream systems to handle deletion appropriately. +// Returns true if the operation should be retried (only for KV revision mismatches), false otherwise. +func handleWALDelete(ctx context.Context, walEvent *WALEvent) bool { + // Extract the SFID using the helper method (which handles DELETE action correctly). + sfid, exists := walEvent.GetSFID() + if !exists { + logger.With("table", walEvent.Table, "action", walEvent.Action).WarnContext(ctx, "WAL delete event missing or empty sfid field, skipping") + return false + } + + // Construct the key based on schema and table name. + keyPrefix := fmt.Sprintf("%s-%s", walEvent.Schema, walEvent.Table) + key := fmt.Sprintf("%s.%s", keyPrefix, sfid) + + // Check if the key exists in the KV bucket. + existing, err := v1KV.Get(ctx, key) + if err == jetstream.ErrKeyNotFound { + // Key doesn't exist, nothing to delete. + logger.With("key", key).DebugContext(ctx, "WAL delete event for non-existent key, skipping") + return false + } else if err != nil { + logger.With(errKey, err, "key", key).ErrorContext(ctx, "failed to get existing KV entry for delete") + return false + } + + // Parse existing data. + var existingData map[string]interface{} + if unmarshalErr := json.Unmarshal(existing.Value(), &existingData); unmarshalErr != nil { + // Try msgpack if JSON fails. + if msgpackErr := msgpack.Unmarshal(existing.Value(), &existingData); msgpackErr != nil { + logger.With(errKey, unmarshalErr, "msgpack_error", msgpackErr, "key", key).ErrorContext(ctx, "failed to unmarshal existing KV entry data for delete") + return false + } + } + + // Update metadata fields. + existingData["_sdc_extracted_at"] = walEvent.CommitTime + existingData["_sdc_received_at"] = time.Now().UTC().Format(time.RFC3339) + // Mark the record as deleted by adding _sdc_deleted_at field. + existingData["_sdc_deleted_at"] = walEvent.CommitTime + + // Encode the updated data using configured format (JSON or MessagePack). + var dataBytes []byte + if cfg.UseMsgpack { + dataBytes, err = msgpack.Marshal(existingData) + } else { + dataBytes, err = json.Marshal(existingData) + } + if err != nil { + logger.With(errKey, err, "key", key, "use_msgpack", cfg.UseMsgpack).ErrorContext(ctx, "failed to marshal deletion marker data") + return false + } + + // Update the entry with the deletion marker. + if _, err := v1KV.Update(ctx, key, dataBytes, existing.Revision()); err != nil { + // Check if this is a revision mismatch that should be retried. + if isRevisionMismatchError(err) { + logger.With(errKey, err, "key", key, "revision", existing.Revision()).WarnContext(ctx, "KV revision mismatch on delete, will retry") + return true + } + logger.With(errKey, err, "key", key, "revision", existing.Revision()).ErrorContext(ctx, "failed to update KV entry with deletion marker") + return false + } + + logger.With("key", key, "encoding", getEncodingFormat()).InfoContext(ctx, "marked KV entry as deleted from WAL event") + return false +} + +// shouldUpdateBasedOnTimestamps compares timestamps between new and existing data to determine if an update should occur. +// Returns true if EITHER systemmodstamp OR lastmodifieddate in new data is later than existing data. +// This implements the same comparison logic as the Meltano target-nats-kv plugin but allows +// updates based on either timestamp field being newer (not just the bookmark field). +func shouldUpdateBasedOnTimestamps(ctx context.Context, newData, existingData map[string]interface{}, key string) bool { + // Extract timestamps from new data. + newSystemModstamp := getTimestampString(newData, "systemmodstamp") + newLastModified := getTimestampString(newData, "lastmodifieddate") + + // Extract timestamps from existing data. + existingSystemModstamp := getTimestampString(existingData, "systemmodstamp") + existingLastModified := getTimestampString(existingData, "lastmodifieddate") + + logger.With( + "key", key, + "new_systemmodstamp", newSystemModstamp, + "new_lastmodified", newLastModified, + "existing_systemmodstamp", existingSystemModstamp, + "existing_lastmodified", existingLastModified, + ).DebugContext(ctx, "comparing timestamps for WAL upsert decision") + + // Parse timestamps for comparison. + newSystemTime, newSystemErr := parseTimestamp(newSystemModstamp) + newLastModTime, newLastModErr := parseTimestamp(newLastModified) + existingSystemTime, existingSystemErr := parseTimestamp(existingSystemModstamp) + existingLastModTime, existingLastModErr := parseTimestamp(existingLastModified) + + // Compare systemmodstamp if both are valid. + if newSystemErr == nil && existingSystemErr == nil { + if newSystemTime.After(existingSystemTime) { + logger.With("key", key).DebugContext(ctx, "WAL upsert: new systemmodstamp is later") + return true + } + } + + // Compare lastmodifieddate if both are valid. + if newLastModErr == nil && existingLastModErr == nil { + if newLastModTime.After(existingLastModTime) { + logger.With("key", key).DebugContext(ctx, "WAL upsert: new lastmodifieddate is later") + return true + } + } + + // If new timestamps are missing/invalid, warn and don't update. + if newSystemErr != nil && newLastModErr != nil { + logger.With("key", key, "systemmodstamp_err", newSystemErr, "lastmodified_err", newLastModErr).WarnContext(ctx, "WAL event has invalid timestamps, skipping upsert") + return false + } + + // If we have valid new timestamps but missing/invalid existing ones, update. + if (newSystemErr == nil || newLastModErr == nil) && (existingSystemErr != nil && existingLastModErr != nil) { + logger.With("key", key).DebugContext(ctx, "WAL upsert: new data has valid timestamps, existing data does not") + return true + } + + logger.With("key", key).DebugContext(ctx, "WAL upsert: existing data is newer or same, skipping") + return false +} + +// getTimestampString safely extracts a timestamp string from a map. +// Returns an empty string if the field doesn't exist or is nil. +func getTimestampString(data map[string]interface{}, field string) string { + if value, exists := data[field]; exists && value != nil { + return fmt.Sprintf("%v", value) + } + return "" +} + +// parseTimestamp parses a timestamp string in common formats used by Salesforce and PostgreSQL. +// It tries multiple timestamp formats to handle various datetime representations. +func parseTimestamp(timestampStr string) (time.Time, error) { + if timestampStr == "" { + return time.Time{}, fmt.Errorf("empty timestamp") + } + + // Try common timestamp formats. + formats := []string{ + time.RFC3339, + time.RFC3339Nano, + "2006-01-02T15:04:05.000000Z", + "2006-01-02T15:04:05Z", + "2006-01-02 15:04:05", + } + + for _, format := range formats { + if t, err := time.Parse(format, timestampStr); err == nil { + return t, nil + } + } + + return time.Time{}, fmt.Errorf("unable to parse timestamp: %s", timestampStr) +} + +// isRevisionMismatchError checks if an error is a KV revision mismatch that should be retried. +func isRevisionMismatchError(err error) bool { + // Attempt direct JetStreamError comparison. + if jsErr, ok := err.(jetstream.JetStreamError); ok { + if apiErr := jsErr.APIError(); apiErr != nil { + return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence + } + } + + // Check for NATS error strings containing the expected error codes. + errStr := err.Error() + if strings.Contains(errStr, "err_code=10071") || + strings.Contains(errStr, "wrong last sequence") || + strings.Contains(errStr, "key exists") { + return true + } + + return false +} + +// getEncodingFormat returns a string representation of the current encoding format. +func getEncodingFormat() string { + if cfg.UseMsgpack { + return "msgpack" + } + return "json" +} diff --git a/cmd/lfx-v1-sync-helper/kv_watcher.go b/cmd/lfx-v1-sync-helper/kv_watcher.go new file mode 100644 index 0000000..f5390b0 --- /dev/null +++ b/cmd/lfx-v1-sync-helper/kv_watcher.go @@ -0,0 +1,95 @@ +// Copyright The Linux Foundation and each contributor to LFX. +// SPDX-License-Identifier: MIT + +// The lfx-v1-sync-helper service. +package main + +import ( + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// kvEntry implements a mock jetstream.KeyValueEntry interface for the handler. +type kvEntry struct { + key string + value []byte + operation jetstream.KeyValueOp +} + +func (e *kvEntry) Key() string { + return e.key +} + +func (e *kvEntry) Value() []byte { + return e.value +} + +func (e *kvEntry) Operation() jetstream.KeyValueOp { + return e.operation +} + +func (e *kvEntry) Bucket() string { + return "v1-objects" +} + +func (e *kvEntry) Created() time.Time { + return time.Now() +} + +func (e *kvEntry) Delta() uint64 { + return 0 +} + +func (e *kvEntry) Revision() uint64 { + return 0 +} + +// kvMessageHandler processes KV update messages from the consumer. +func kvMessageHandler(msg jetstream.Msg) { + // Parse the message as a KV entry. + headers := msg.Headers() + subject := msg.Subject() + + // Extract key from the subject ($KV.v1-objects.{key}). + key := "" + if len(subject) > len("$KV.v1-objects.") { + key = subject[len("$KV.v1-objects."):] + } + + // Determine operation from headers. + operation := jetstream.KeyValuePut // Default to PUT. + if opHeader := headers.Get("KV-Operation"); opHeader != "" { + switch opHeader { + case "DEL": + operation = jetstream.KeyValueDelete + case "PURGE": + operation = jetstream.KeyValuePurge + } + } + + // Create a mock KV entry for the handler. + entry := &kvEntry{ + key: key, + value: msg.Data(), + operation: operation, + } + + // Process the KV entry and check if retry is needed. + shouldRetry := kvHandler(entry) + + // Handle message acknowledgment based on retry decision. + if shouldRetry { + // NAK the message to trigger retry. + if err := msg.Nak(); err != nil { + logger.With(errKey, err, "key", key).Error("failed to NAK KV JetStream message for retry") + } else { + logger.With("key", key).Debug("NAKed KV message for retry") + } + } else { + // Acknowledge the message. + if err := msg.Ack(); err != nil { + logger.With(errKey, err, "key", key).Error("failed to acknowledge KV JetStream message") + } + } +} diff --git a/cmd/lfx-v1-sync-helper/lfx_v1_client.go b/cmd/lfx-v1-sync-helper/lfx_v1_client.go index 35baef3..017ded9 100644 --- a/cmd/lfx-v1-sync-helper/lfx_v1_client.go +++ b/cmd/lfx-v1-sync-helper/lfx_v1_client.go @@ -62,7 +62,6 @@ const ( var ( v1HTTPClient *http.Client - auth0Config *authentication.Authentication ) // V1User represents a user from the LFX v1 User Service @@ -147,8 +146,6 @@ func initV1Client(cfg *Config) error { return fmt.Errorf("failed to create Auth0 client configuration: %w", err) } - auth0Config = authConfig - // Create HTTP client with Auth0 token source tokenSource := &ClientCredentialsTokenSource{ ctx: context.Background(), diff --git a/cmd/lfx-v1-sync-helper/lfx_v2_client.go b/cmd/lfx-v1-sync-helper/lfx_v2_client.go index 582b32d..091a6fd 100644 --- a/cmd/lfx-v1-sync-helper/lfx_v2_client.go +++ b/cmd/lfx-v1-sync-helper/lfx_v2_client.go @@ -87,11 +87,11 @@ func (dt *debugTransport) RoundTrip(req *http.Request) (*http.Response, error) { } var ( - httpClient *http.Client - jwtPrivateKey *rsa.PrivateKey - jwtKeyID string - jwtClientID string - heimdallConfig *Config + httpClient *http.Client + jwtPrivateKey *rsa.PrivateKey + jwtKeyID string + jwtClientID string + projectClient *projectservice.Client committeeClient *committeeservice.Client jwtTokenCache *cache.Cache @@ -109,7 +109,6 @@ type JWK struct { // initJWTClient initializes the JWT authentication and HTTP client with Goa SDK clients. func initJWTClient(cfg *Config) error { - heimdallConfig = cfg // Parse the private key. block, _ := pem.Decode([]byte(cfg.HeimdallPrivateKey)) if block == nil { @@ -382,15 +381,3 @@ func boolPtrToBool(b *bool) bool { } return *b } - -// stringToTime converts a string to time, parsing ISO 8601 format, returning zero time if empty or invalid. -func stringToTime(s string) time.Time { - if s == "" { - return time.Time{} - } - t, err := time.Parse(time.RFC3339, s) - if err != nil { - return time.Time{} - } - return t -} diff --git a/cmd/lfx-v1-sync-helper/main.go b/cmd/lfx-v1-sync-helper/main.go index a9f5c97..d281701 100644 --- a/cmd/lfx-v1-sync-helper/main.go +++ b/cmd/lfx-v1-sync-helper/main.go @@ -20,41 +20,6 @@ import ( "github.com/nats-io/nats.go/jetstream" ) -// kvEntry implements a mock jetstream.KeyValueEntry interface for the handler -type kvEntry struct { - key string - value []byte - operation jetstream.KeyValueOp -} - -func (e *kvEntry) Key() string { - return e.key -} - -func (e *kvEntry) Value() []byte { - return e.value -} - -func (e *kvEntry) Operation() jetstream.KeyValueOp { - return e.operation -} - -func (e *kvEntry) Bucket() string { - return "v1-objects" -} - -func (e *kvEntry) Created() time.Time { - return time.Now() -} - -func (e *kvEntry) Delta() uint64 { - return 0 -} - -func (e *kvEntry) Revision() uint64 { - return 0 -} - const ( errKey = "error" defaultListenPort = "8080" @@ -62,8 +27,6 @@ const ( // request timeout, and lower than the pod or liveness probe's // terminationGracePeriodSeconds. gracefulShutdownSeconds = 25 - // natsQueue is commented out since WAL-listener handlers are disabled for now. - //natsQueue = "dev.lfx.v1-sync-helper.queue" ) var ( @@ -256,132 +219,65 @@ func main() { os.Exit(1) } - // Process KV updates using the JetStream pull consumer - // Multiple instances will compete for messages automatically - go func() { - for { - select { - case <-ctx.Done(): - return - default: - // Fetch messages with a batch size of 1 and timeout - msgs, err := consumer.Fetch(1, jetstream.FetchMaxWait(5*time.Second)) - if err != nil { - if err == jetstream.ErrNoMessages { - // No messages available, continue polling - continue - } - logger.With(errKey, err, "consumer", consumerName).Error("error fetching messages from consumer") - continue - } - - for msg := range msgs.Messages() { - // Parse the message as a KV entry - headers := msg.Headers() - subject := msg.Subject() - - // Extract key from the subject ($KV.v1-objects.{key}) - key := "" - if len(subject) > len("$KV.v1-objects.") { - key = subject[len("$KV.v1-objects."):] - } - - // Determine operation from headers - operation := jetstream.KeyValuePut // Default to PUT - if opHeader := headers.Get("KV-Operation"); opHeader != "" { - switch opHeader { - case "DEL": - operation = jetstream.KeyValueDelete - case "PURGE": - operation = jetstream.KeyValuePurge - } - } - - // Create a mock KV entry for the handler - entry := &kvEntry{ - key: key, - value: msg.Data(), - operation: operation, - } - - // Process the KV entry - kvHandler(entry) - - // Acknowledge the message - if err := msg.Ack(); err != nil { - logger.With(errKey, err, "key", key).Error("failed to acknowledge JetStream message") - } - } - } - } - }() + // Start consuming KV updates using the JetStream consumer with error handling. + kvConsumerCtx, err := consumer.Consume(kvMessageHandler, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + logger.With(errKey, err).Error("KV consumer error encountered") + })) + if err != nil { + logger.With(errKey, err, "consumer", consumerName).Error("error starting KV consumer") + os.Exit(1) + } + defer kvConsumerCtx.Stop() - // WAL-listener handlers are commented out but kept for future use - // when they need to write to the same v1-objects KV bucket - /* - // Define WAL-listener subjects locally since they're commented out in other files - const ( - walProjectSubject = "wal_listener.salesforce_project__c" - walCollaborationSubject = "wal_listener.salesforce_collaboration__c" - ) - - // Subscribe to wal-listener v1 project events using JetStream - walConsumer, err := jsContext.CreateOrUpdateConsumer(ctx, "wal", jetstream.ConsumerConfig{ - Durable: "v1-sync-helper-wal-project", - FilterSubject: walProjectSubject, - DeliverGroup: natsQueue, - AckPolicy: jetstream.AckExplicitPolicy, - DeliverPolicy: jetstream.DeliverAllPolicy, - }) - if err != nil { - logger.With(errKey, err, "subject", walProjectSubject).Error("error creating consumer for WAL-listener subject") - os.Exit(1) - } - if _, err = walConsumer.Consume(func(msg jetstream.Msg) { - if err := msg.Ack(); err != nil { - logger.With(errKey, err).Error("failed to acknowledge JetStream message") - return - } - // walListenerHandler(&msg.Message, v1KV, mappingsKV) - handler commented out - }); err != nil { - logger.With(errKey, err, "subject", walProjectSubject).Error("error subscribing to WAL-listener subject") - os.Exit(1) - } + // Subscribe to WAL-listener events from the wal_listener stream + walStreamName := "wal_listener" + walConsumerName := "v1-sync-helper-wal-consumer" - // Subscribe to wal-listener v1 committee (collaboration) events using JetStream - walCollabConsumer, err := jsContext.CreateOrUpdateConsumer(ctx, "wal", jetstream.ConsumerConfig{ - Durable: "v1-sync-helper-wal-collab", - FilterSubject: walCollaborationSubject, - DeliverGroup: natsQueue, - AckPolicy: jetstream.AckExplicitPolicy, - DeliverPolicy: jetstream.DeliverAllPolicy, - }) - if err != nil { - logger.With(errKey, err, "subject", walCollaborationSubject).Error("error creating consumer for WAL-listener collaboration subject") - os.Exit(1) - } - if _, err = walCollabConsumer.Consume(func(msg jetstream.Msg) { - if err := msg.Ack(); err != nil { - logger.With(errKey, err).Error("failed to acknowledge JetStream message") - return - } - // walCollaborationHandler(&msg.Message, v1KV, mappingsKV) - handler commented out - }); err != nil { - logger.With(errKey, err, "subject", walCollaborationSubject).Error("error subscribing to WAL-listener collaboration subject") - os.Exit(1) - } - */ + // Create or get consumer for WAL listener events + walConsumer, err := jsContext.CreateOrUpdateConsumer(ctx, walStreamName, jetstream.ConsumerConfig{ + Name: walConsumerName, + Durable: walConsumerName, + DeliverPolicy: jetstream.DeliverAllPolicy, + AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: "wal_listener.*", + MaxDeliver: 3, + AckWait: 30 * time.Second, + MaxAckPending: 100, + Description: "WAL listener consumer for v1-sync-helper", + }) + if err != nil { + logger.With(errKey, err, "consumer", walConsumerName, "stream", walStreamName).Error("error creating WAL listener consumer") + os.Exit(1) + } + + // Start consuming WAL listener messages with error handling. + walConsumerCtx, err := walConsumer.Consume(walIngestHandler, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + logger.With(errKey, err).Error("WAL consumer error encountered") + })) + if err != nil { + logger.With(errKey, err, "consumer", walConsumerName).Error("error starting WAL listener consumer") + os.Exit(1) + } + defer walConsumerCtx.Stop() // This next line blocks until SIGINT or SIGTERM is received, or NATS disconnects. <-done + // Begin graceful shutdown process. + logger.Debug("beginning graceful shutdown") + + // Drain consumers first (non-blocking) to mitigate "nats: connection closed" + // errors in the ConsumeErrHandler. + kvConsumerCtx.Drain() + walConsumerCtx.Drain() + // Cancel the background context. cancel() - // Drain the connection, which will drain all subscriptions, then close the - // connection when complete. + // Drain the connection, which will drain all remaining subscriptions, then + // close the connection when complete (including the consumer draining). if !natsConn.IsClosed() && !natsConn.IsDraining() { - logger.Info("draining NATS connections") + logger.Info("draining NATS connection") if err := natsConn.Drain(); err != nil { logger.With(errKey, err).Error("error draining NATS connection") os.Exit(1) @@ -389,7 +285,9 @@ func main() { } // Wait for the graceful shutdown steps to complete. + logger.Debug("waiting for graceful shutdown steps to complete") gracefulCloseWG.Wait() + logger.Debug("graceful shutdown steps completed") // Immediately close the HTTP server after graceful shutdown has finished. if err = httpServer.Close(); err != nil { diff --git a/go.mod b/go.mod index 1f204b4..cc9d9d2 100644 --- a/go.mod +++ b/go.mod @@ -2,31 +2,31 @@ // SPDX-License-Identifier: MIT module github.com/linuxfoundation/lfx-v1-sync-helper -go 1.25.4 +go 1.25.6 require ( github.com/akamensky/base58 v0.0.0-20210829145138-ce8bf8802e8f - github.com/auth0/go-auth0 v1.32.0 + github.com/auth0/go-auth0 v1.32.1 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/uuid v1.6.0 - github.com/linuxfoundation/lfx-v2-committee-service v0.2.16 - github.com/linuxfoundation/lfx-v2-project-service v0.5.3 - github.com/nats-io/nats.go v1.47.0 + github.com/linuxfoundation/lfx-v2-committee-service v0.2.19 + github.com/linuxfoundation/lfx-v2-project-service v0.5.4 + github.com/nats-io/nats.go v1.48.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/teambition/rrule-go v1.8.2 github.com/vmihailenco/msgpack/v5 v5.4.1 - goa.design/goa/v3 v3.23.2 - golang.org/x/oauth2 v0.33.0 + goa.design/goa/v3 v3.23.4 + golang.org/x/oauth2 v0.34.0 ) require ( github.com/PuerkitoBio/rehttp v1.4.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect - github.com/go-chi/chi/v5 v5.2.3 // indirect + github.com/go-chi/chi/v5 v5.2.4 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect @@ -39,6 +39,6 @@ require ( github.com/segmentio/asm v1.2.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.devnw.com/structs v1.0.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/sys v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index 6c17bda..fbb46b2 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/PuerkitoBio/rehttp v1.4.0 h1:rIN7A2s+O9fmHUM1vUcInvlHj9Ysql4hE+Y0wcl/ github.com/PuerkitoBio/rehttp v1.4.0/go.mod h1:LUwKPoDbDIA2RL5wYZCNsQ90cx4OJ4AWBmq6KzWZL1s= github.com/akamensky/base58 v0.0.0-20210829145138-ce8bf8802e8f h1:z8MkSJCUyTmW5YQlxsMLBlwA7GmjxC7L4ooicxqnhz8= github.com/akamensky/base58 v0.0.0-20210829145138-ce8bf8802e8f/go.mod h1:UdUwYgAXBiL+kLfcqxoQJYkHA/vl937/PbFhZM34aZs= -github.com/auth0/go-auth0 v1.32.0 h1:PuojPRBDQPvFMtXDX7ags8ackLVYXDU7gpTi7/8sEws= -github.com/auth0/go-auth0 v1.32.0/go.mod h1:32sQB1uAn+99fJo6N819EniKq8h785p0ag0lMWhiTaE= +github.com/auth0/go-auth0 v1.32.1 h1:AAXQqaNaFZWkRm2bg5mVVXpqDLmusv7v238uIaxuFpo= +github.com/auth0/go-auth0 v1.32.1/go.mod h1:32sQB1uAn+99fJo6N819EniKq8h785p0ag0lMWhiTaE= github.com/aybabtme/iocontrol v0.0.0-20150809002002-ad15bcfc95a0 h1:0NmehRCgyk5rljDQLKUO+cRJCnduDyn11+zGZIc9Z48= github.com/aybabtme/iocontrol v0.0.0-20150809002002-ad15bcfc95a0/go.mod h1:6L7zgvqo0idzI7IO8de6ZC051AfXb5ipkIJ7bIA2tGA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -14,8 +14,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= -github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4= +github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= @@ -26,8 +26,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -40,12 +40,12 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linuxfoundation/lfx-v2-committee-service v0.2.16 h1:4tVcges7E6Kd6VuQ4wV92yUriKwCqlHdTIwQbR7itic= -github.com/linuxfoundation/lfx-v2-committee-service v0.2.16/go.mod h1:B21xE5Wb7cJlYvWYQotg8pDSS3V30kxDmFa6IeYEmBk= -github.com/linuxfoundation/lfx-v2-project-service v0.5.3 h1:yy/JTEhwxUA47pPGrWhxFLFNYcefzpvT6+8f49nNQeE= -github.com/linuxfoundation/lfx-v2-project-service v0.5.3/go.mod h1:M5iH1soDCJciL32rgOT2PvSCksXKulDa154t8TYOC9I= -github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= -github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/linuxfoundation/lfx-v2-committee-service v0.2.19 h1:xMeqXvf7NLk9JlfAEpNHBpwFtvtNTnwR/bOjxY/5Fa4= +github.com/linuxfoundation/lfx-v2-committee-service v0.2.19/go.mod h1:vhFiTH/CEDcs2kHmDofKzJclW1ZtBlfaRH0brL6lfek= +github.com/linuxfoundation/lfx-v2-project-service v0.5.4 h1:liFMdhd61Ew7D+W0a/fZzUkBX4pjTVWggu5jX1MkaFc= +github.com/linuxfoundation/lfx-v2-project-service v0.5.4/go.mod h1:M5iH1soDCJciL32rgOT2PvSCksXKulDa154t8TYOC9I= +github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= +github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -71,19 +71,19 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.devnw.com/structs v1.0.0 h1:FFkBoBOkapCdxFEIkpOZRmMOMr9b9hxjKTD3bJYl9lk= go.devnw.com/structs v1.0.0/go.mod h1:wHBkdQpNeazdQHszJ2sxwVEpd8zGTEsKkeywDLGbrmg= -goa.design/goa/v3 v3.23.2 h1:i/JWSoD6lLc9O7ckm/+5N5lKw0mzgRPI5KZHmN7wF50= -goa.design/goa/v3 v3.23.2/go.mod h1:DaJ9yv5WoXrpolbzouDj0A0o5Os0rPTTHy4aSebYVuI= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +goa.design/goa/v3 v3.23.4 h1:7d9IAtyC8aP9bAvTdY+YPQaScpoZRd/paDH3PSXaxbM= +goa.design/goa/v3 v3.23.4/go.mod h1:da3W585WfJe9gT+hJCbP8YFB9yc4gmuCwB0MvkbwhXk= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= -golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/manifests/meltano-el-postgres-cronjob.yaml b/manifests/meltano-el-postgres-cronjob.yaml deleted file mode 100644 index 31b6e65..0000000 --- a/manifests/meltano-el-postgres-cronjob.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright The Linux Foundation and each contributor to LFX. -# SPDX-License-Identifier: MIT ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: meltano-el-postgres-cronjob - namespace: v1-sync-helper - labels: - app.kubernetes.io/name: lfx-v1-sync-helper - app.kubernetes.io/component: meltano -spec: - schedule: "*/5 * * * *" # Every 5 minutes - concurrencyPolicy: Forbid # Prevent concurrent execution - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - jobTemplate: - metadata: - labels: - app.kubernetes.io/name: lfx-v1-sync-helper - app.kubernetes.io/component: meltano - spec: - ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour - backoffLimit: 1 - template: - metadata: - labels: - app.kubernetes.io/name: lfx-v1-sync-helper - app.kubernetes.io/component: meltano - spec: - restartPolicy: Never - containers: - - name: meltano - image: ghcr.io/linuxfoundation/lfx-v1-sync-helper/meltano:v0.4.2 - imagePullPolicy: Always - workingDir: /app/meltano - args: - - el - # Catalog from mounted ConfigMap - - "--catalog" - - "/catalogs/tap-postgres/catalog.json" - # State ID for this el job - - "--state-id" - - platform-db-incremental - # Source - - tap-postgres - # Target - - target-nats-kv - env: - # Environment (prod/staging/dev per meltano.yml "environments" list) - - name: MELTANO_ENVIRONMENT - value: "prod" - # S3 state backend (prod/staging/dev suffix from OpenTofu workspace names) - - name: MELTANO_STATE_BACKEND_URI - value: "s3://lfx-v2-meltano-state-prod" - # AWS configuration for DynamoDB access - - name: AWS_DEFAULT_REGION - value: "us-west-2" - - name: AWS_REGION - value: "us-west-2" - # Postgres configuration - - name: TAP_POSTGRES_HOST - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: host - optional: true - - name: TAP_POSTGRES_PORT - value: "5432" - - name: TAP_POSTGRES_DATABASE - value: "sfdc" - - name: TAP_POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: username - optional: true - - name: TAP_POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: password - optional: true - # Target NATS KV configuration - - name: TARGET_NATS_KV_URL - value: "nats://lfx-platform-nats.lfx.svc.cluster.local:4222" - - name: TARGET_NATS_KV_BUCKET - value: "v1-objects" - - name: TARGET_NATS_KV_REFRESH_MODE - value: "newer" - - name: TARGET_NATS_KV_VALIDATE_RECORDS - # (true for PostgreSQL; false for DynamoDB) - value: "true" - - name: TARGET_NATS_KV_MSGPACK - value: "true" - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "2Gi" - cpu: "1000m" - # Health check - Meltano jobs don't typically have health endpoints - # but we can check if the process is running - livenessProbe: - exec: - command: ["pgrep", "-f", "meltano"] - initialDelaySeconds: 30 - periodSeconds: 60 - failureThreshold: 3 - volumeMounts: - - name: tap-postgres-catalog - mountPath: /catalogs/tap-postgres - readOnly: true - volumes: - - name: tap-postgres-catalog - configMap: - name: tap-postgres-catalog - # Use service account with appropriate permissions for AWS - serviceAccountName: v1-sync-helper-sa diff --git a/manifests/meltano-el-postgres.yaml b/manifests/meltano-el-postgres.yaml deleted file mode 100644 index 36ae186..0000000 --- a/manifests/meltano-el-postgres.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright The Linux Foundation and each contributor to LFX. -# SPDX-License-Identifier: MIT ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: meltano-el-postgres - namespace: v1-sync-helper - labels: - app.kubernetes.io/name: lfx-v1-sync-helper - app.kubernetes.io/component: meltano -spec: - ttlSecondsAfterFinished: 3600 # Clean up job after 1 hour - backoffLimit: 3 - template: - metadata: - labels: - app.kubernetes.io/name: lfx-v1-sync-helper - app.kubernetes.io/component: meltano - spec: - restartPolicy: Never - containers: - - name: meltano - image: ghcr.io/linuxfoundation/lfx-v1-sync-helper/meltano:v0.4.2 - imagePullPolicy: Always - workingDir: /app/meltano - args: - - el - # Catalog from mounted ConfigMap - - "--catalog" - - "/catalogs/tap-postgres/catalog.json" - # State ID for this el job - - "--state-id" - - platform-db-incremental - # Source - - tap-postgres - # Target - - target-nats-kv - env: - # Environment (prod/staging/dev per meltano.yml "environments" list) - - name: MELTANO_ENVIRONMENT - value: "prod" - # S3 state backend (prod/staging/dev suffix from OpenTofu workspace names) - - name: MELTANO_STATE_BACKEND_URI - value: "s3://lfx-v2-meltano-state-prod" - # AWS configuration for DynamoDB access - - name: AWS_DEFAULT_REGION - value: "us-west-2" - - name: AWS_REGION - value: "us-west-2" - # Postgres configuration - - name: TAP_POSTGRES_HOST - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: host - optional: true - - name: TAP_POSTGRES_PORT - value: "5432" - - name: TAP_POSTGRES_DATABASE - value: "sfdc" - - name: TAP_POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: username - optional: true - - name: TAP_POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-credentials-ad-hoc - key: password - optional: true - # Target NATS KV configuration - - name: TARGET_NATS_KV_URL - value: "nats://lfx-platform-nats.lfx.svc.cluster.local:4222" - - name: TARGET_NATS_KV_BUCKET - value: "v1-objects" - - name: TARGET_NATS_KV_REFRESH_MODE - value: "newer" - - name: TARGET_NATS_KV_VALIDATE_RECORDS - # (true for PostgreSQL; false for DynamoDB) - value: "true" - - name: TARGET_NATS_KV_MSGPACK - value: "true" - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "2Gi" - cpu: "1000m" - # Health check - Meltano jobs don't typically have health endpoints - # but we can check if the process is running - livenessProbe: - exec: - command: ["pgrep", "-f", "meltano"] - initialDelaySeconds: 30 - periodSeconds: 60 - failureThreshold: 3 - volumeMounts: - - name: tap-postgres-catalog - mountPath: /catalogs/tap-postgres - readOnly: true - volumes: - - name: tap-postgres-catalog - configMap: - name: tap-postgres-catalog - # Use service account with appropriate permissions for AWS - serviceAccountName: v1-sync-helper-sa diff --git a/meltano/load/target-nats-kv/uv.lock b/meltano/load/target-nats-kv/uv.lock index f3f535e..3a5a6d8 100644 --- a/meltano/load/target-nats-kv/uv.lock +++ b/meltano/load/target-nats-kv/uv.lock @@ -85,65 +85,65 @@ wheels = [ [[package]] name = "librt" -version = "0.7.7" +version = "0.7.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/29/47f29026ca17f35cf299290292d5f8331f5077364974b7675a353179afa2/librt-0.7.7.tar.gz", hash = "sha256:81d957b069fed1890953c3b9c3895c7689960f233eea9a1d9607f71ce7f00b2c", size = 145910, upload-time = "2026-01-01T23:52:22.87Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/24/5f3646ff414285e0f7708fa4e946b9bf538345a41d1c375c439467721a5e/librt-0.7.8.tar.gz", hash = "sha256:1a4ede613941d9c3470b0368be851df6bb78ab218635512d0370b27a277a0862", size = 148323, upload-time = "2026-01-14T12:56:16.876Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/56/30b5c342518005546df78841cb0820ae85a17e7d07d521c10ef367306d0d/librt-0.7.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a487b71fbf8a9edb72a8c7a456dda0184642d99cd007bc819c0b7ab93676a8ee", size = 54709, upload-time = "2026-01-01T23:51:02.774Z" }, - { url = "https://files.pythonhosted.org/packages/72/78/9f120e3920b22504d4f3835e28b55acc2cc47c9586d2e1b6ba04c3c1bf01/librt-0.7.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f4d4efb218264ecf0f8516196c9e2d1a0679d9fb3bb15df1155a35220062eba8", size = 56663, upload-time = "2026-01-01T23:51:03.838Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ea/7d7a1ee7dfc1151836028eba25629afcf45b56bbc721293e41aa2e9b8934/librt-0.7.7-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b8bb331aad734b059c4b450cd0a225652f16889e286b2345af5e2c3c625c3d85", size = 161705, upload-time = "2026-01-01T23:51:04.917Z" }, - { url = "https://files.pythonhosted.org/packages/45/a5/952bc840ac8917fbcefd6bc5f51ad02b89721729814f3e2bfcc1337a76d6/librt-0.7.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:467dbd7443bda08338fc8ad701ed38cef48194017554f4c798b0a237904b3f99", size = 171029, upload-time = "2026-01-01T23:51:06.09Z" }, - { url = "https://files.pythonhosted.org/packages/fa/bf/c017ff7da82dc9192cf40d5e802a48a25d00e7639b6465cfdcee5893a22c/librt-0.7.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50d1d1ee813d2d1a3baf2873634ba506b263032418d16287c92ec1cc9c1a00cb", size = 184704, upload-time = "2026-01-01T23:51:07.549Z" }, - { url = "https://files.pythonhosted.org/packages/77/ec/72f3dd39d2cdfd6402ab10836dc9cbf854d145226062a185b419c4f1624a/librt-0.7.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e5070cf3ec92d98f57574da0224f8c73faf1ddd6d8afa0b8c9f6e86997bc74", size = 180719, upload-time = "2026-01-01T23:51:09.062Z" }, - { url = "https://files.pythonhosted.org/packages/78/86/06e7a1a81b246f3313bf515dd9613a1c81583e6fd7843a9f4d625c4e926d/librt-0.7.7-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:bdb9f3d865b2dafe7f9ad7f30ef563c80d0ddd2fdc8cc9b8e4f242f475e34d75", size = 174537, upload-time = "2026-01-01T23:51:10.611Z" }, - { url = "https://files.pythonhosted.org/packages/83/08/f9fb2edc9c7a76e95b2924ce81d545673f5b034e8c5dd92159d1c7dae0c6/librt-0.7.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8185c8497d45164e256376f9da5aed2bb26ff636c798c9dabe313b90e9f25b28", size = 195238, upload-time = "2026-01-01T23:51:11.762Z" }, - { url = "https://files.pythonhosted.org/packages/ba/56/ea2d2489d3ea1f47b301120e03a099e22de7b32c93df9a211e6ff4f9bf38/librt-0.7.7-cp311-cp311-win32.whl", hash = "sha256:44d63ce643f34a903f09ff7ca355aae019a3730c7afd6a3c037d569beeb5d151", size = 42939, upload-time = "2026-01-01T23:51:13.192Z" }, - { url = "https://files.pythonhosted.org/packages/58/7b/c288f417e42ba2a037f1c0753219e277b33090ed4f72f292fb6fe175db4c/librt-0.7.7-cp311-cp311-win_amd64.whl", hash = "sha256:7d13cc340b3b82134f8038a2bfe7137093693dcad8ba5773da18f95ad6b77a8a", size = 49240, upload-time = "2026-01-01T23:51:14.264Z" }, - { url = "https://files.pythonhosted.org/packages/7c/24/738eb33a6c1516fdb2dfd2a35db6e5300f7616679b573585be0409bc6890/librt-0.7.7-cp311-cp311-win_arm64.whl", hash = "sha256:983de36b5a83fe9222f4f7dcd071f9b1ac6f3f17c0af0238dadfb8229588f890", size = 42613, upload-time = "2026-01-01T23:51:15.268Z" }, - { url = "https://files.pythonhosted.org/packages/56/72/1cd9d752070011641e8aee046c851912d5f196ecd726fffa7aed2070f3e0/librt-0.7.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a85a1fc4ed11ea0eb0a632459ce004a2d14afc085a50ae3463cd3dfe1ce43fc", size = 55687, upload-time = "2026-01-01T23:51:16.291Z" }, - { url = "https://files.pythonhosted.org/packages/50/aa/d5a1d4221c4fe7e76ae1459d24d6037783cb83c7645164c07d7daf1576ec/librt-0.7.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c87654e29a35938baead1c4559858f346f4a2a7588574a14d784f300ffba0efd", size = 57136, upload-time = "2026-01-01T23:51:17.363Z" }, - { url = "https://files.pythonhosted.org/packages/23/6f/0c86b5cb5e7ef63208c8cc22534df10ecc5278efc0d47fb8815577f3ca2f/librt-0.7.7-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c9faaebb1c6212c20afd8043cd6ed9de0a47d77f91a6b5b48f4e46ed470703fe", size = 165320, upload-time = "2026-01-01T23:51:18.455Z" }, - { url = "https://files.pythonhosted.org/packages/16/37/df4652690c29f645ffe405b58285a4109e9fe855c5bb56e817e3e75840b3/librt-0.7.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1908c3e5a5ef86b23391448b47759298f87f997c3bd153a770828f58c2bb4630", size = 174216, upload-time = "2026-01-01T23:51:19.599Z" }, - { url = "https://files.pythonhosted.org/packages/9a/d6/d3afe071910a43133ec9c0f3e4ce99ee6df0d4e44e4bddf4b9e1c6ed41cc/librt-0.7.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dbc4900e95a98fc0729523be9d93a8fedebb026f32ed9ffc08acd82e3e181503", size = 189005, upload-time = "2026-01-01T23:51:21.052Z" }, - { url = "https://files.pythonhosted.org/packages/d5/18/74060a870fe2d9fd9f47824eba6717ce7ce03124a0d1e85498e0e7efc1b2/librt-0.7.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a7ea4e1fbd253e5c68ea0fe63d08577f9d288a73f17d82f652ebc61fa48d878d", size = 183961, upload-time = "2026-01-01T23:51:22.493Z" }, - { url = "https://files.pythonhosted.org/packages/7c/5e/918a86c66304af66a3c1d46d54df1b2d0b8894babc42a14fb6f25511497f/librt-0.7.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ef7699b7a5a244b1119f85c5bbc13f152cd38240cbb2baa19b769433bae98e50", size = 177610, upload-time = "2026-01-01T23:51:23.874Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d7/b5e58dc2d570f162e99201b8c0151acf40a03a39c32ab824dd4febf12736/librt-0.7.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:955c62571de0b181d9e9e0a0303c8bc90d47670a5eff54cf71bf5da61d1899cf", size = 199272, upload-time = "2026-01-01T23:51:25.341Z" }, - { url = "https://files.pythonhosted.org/packages/18/87/8202c9bd0968bdddc188ec3811985f47f58ed161b3749299f2c0dd0f63fb/librt-0.7.7-cp312-cp312-win32.whl", hash = "sha256:1bcd79be209313b270b0e1a51c67ae1af28adad0e0c7e84c3ad4b5cb57aaa75b", size = 43189, upload-time = "2026-01-01T23:51:26.799Z" }, - { url = "https://files.pythonhosted.org/packages/61/8d/80244b267b585e7aa79ffdac19f66c4861effc3a24598e77909ecdd0850e/librt-0.7.7-cp312-cp312-win_amd64.whl", hash = "sha256:4353ee891a1834567e0302d4bd5e60f531912179578c36f3d0430f8c5e16b456", size = 49462, upload-time = "2026-01-01T23:51:27.813Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1f/75db802d6a4992d95e8a889682601af9b49d5a13bbfa246d414eede1b56c/librt-0.7.7-cp312-cp312-win_arm64.whl", hash = "sha256:a76f1d679beccccdf8c1958e732a1dfcd6e749f8821ee59d7bec009ac308c029", size = 42828, upload-time = "2026-01-01T23:51:28.804Z" }, - { url = "https://files.pythonhosted.org/packages/8d/5e/d979ccb0a81407ec47c14ea68fb217ff4315521730033e1dd9faa4f3e2c1/librt-0.7.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f4a0b0a3c86ba9193a8e23bb18f100d647bf192390ae195d84dfa0a10fb6244", size = 55746, upload-time = "2026-01-01T23:51:29.828Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2c/3b65861fb32f802c3783d6ac66fc5589564d07452a47a8cf9980d531cad3/librt-0.7.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5335890fea9f9e6c4fdf8683061b9ccdcbe47c6dc03ab8e9b68c10acf78be78d", size = 57174, upload-time = "2026-01-01T23:51:31.226Z" }, - { url = "https://files.pythonhosted.org/packages/50/df/030b50614b29e443607220097ebaf438531ea218c7a9a3e21ea862a919cd/librt-0.7.7-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b4346b1225be26def3ccc6c965751c74868f0578cbcba293c8ae9168483d811", size = 165834, upload-time = "2026-01-01T23:51:32.278Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e1/bd8d1eacacb24be26a47f157719553bbd1b3fe812c30dddf121c0436fd0b/librt-0.7.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a10b8eebdaca6e9fdbaf88b5aefc0e324b763a5f40b1266532590d5afb268a4c", size = 174819, upload-time = "2026-01-01T23:51:33.461Z" }, - { url = "https://files.pythonhosted.org/packages/46/7d/91d6c3372acf54a019c1ad8da4c9ecf4fc27d039708880bf95f48dbe426a/librt-0.7.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:067be973d90d9e319e6eb4ee2a9b9307f0ecd648b8a9002fa237289a4a07a9e7", size = 189607, upload-time = "2026-01-01T23:51:34.604Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ac/44604d6d3886f791fbd1c6ae12d5a782a8f4aca927484731979f5e92c200/librt-0.7.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23d2299ed007812cccc1ecef018db7d922733382561230de1f3954db28433977", size = 184586, upload-time = "2026-01-01T23:51:35.845Z" }, - { url = "https://files.pythonhosted.org/packages/5c/26/d8a6e4c17117b7f9b83301319d9a9de862ae56b133efb4bad8b3aa0808c9/librt-0.7.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6b6f8ea465524aa4c7420c7cc4ca7d46fe00981de8debc67b1cc2e9957bb5b9d", size = 178251, upload-time = "2026-01-01T23:51:37.018Z" }, - { url = "https://files.pythonhosted.org/packages/99/ab/98d857e254376f8e2f668e807daccc1f445e4b4fc2f6f9c1cc08866b0227/librt-0.7.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8df32a99cc46eb0ee90afd9ada113ae2cafe7e8d673686cf03ec53e49635439", size = 199853, upload-time = "2026-01-01T23:51:38.195Z" }, - { url = "https://files.pythonhosted.org/packages/7c/55/4523210d6ae5134a5da959900be43ad8bab2e4206687b6620befddb5b5fd/librt-0.7.7-cp313-cp313-win32.whl", hash = "sha256:86f86b3b785487c7760247bcdac0b11aa8bf13245a13ed05206286135877564b", size = 43247, upload-time = "2026-01-01T23:51:39.629Z" }, - { url = "https://files.pythonhosted.org/packages/25/40/3ec0fed5e8e9297b1cf1a3836fb589d3de55f9930e3aba988d379e8ef67c/librt-0.7.7-cp313-cp313-win_amd64.whl", hash = "sha256:4862cb2c702b1f905c0503b72d9d4daf65a7fdf5a9e84560e563471e57a56949", size = 49419, upload-time = "2026-01-01T23:51:40.674Z" }, - { url = "https://files.pythonhosted.org/packages/1c/7a/aab5f0fb122822e2acbc776addf8b9abfb4944a9056c00c393e46e543177/librt-0.7.7-cp313-cp313-win_arm64.whl", hash = "sha256:0996c83b1cb43c00e8c87835a284f9057bc647abd42b5871e5f941d30010c832", size = 42828, upload-time = "2026-01-01T23:51:41.731Z" }, - { url = "https://files.pythonhosted.org/packages/69/9c/228a5c1224bd23809a635490a162e9cbdc68d99f0eeb4a696f07886b8206/librt-0.7.7-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:23daa1ab0512bafdd677eb1bfc9611d8ffbe2e328895671e64cb34166bc1b8c8", size = 55188, upload-time = "2026-01-01T23:51:43.14Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c2/0e7c6067e2b32a156308205e5728f4ed6478c501947e9142f525afbc6bd2/librt-0.7.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:558a9e5a6f3cc1e20b3168fb1dc802d0d8fa40731f6e9932dcc52bbcfbd37111", size = 56895, upload-time = "2026-01-01T23:51:44.534Z" }, - { url = "https://files.pythonhosted.org/packages/0e/77/de50ff70c80855eb79d1d74035ef06f664dd073fb7fb9d9fb4429651b8eb/librt-0.7.7-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2567cb48dc03e5b246927ab35cbb343376e24501260a9b5e30b8e255dca0d1d2", size = 163724, upload-time = "2026-01-01T23:51:45.571Z" }, - { url = "https://files.pythonhosted.org/packages/6e/19/f8e4bf537899bdef9e0bb9f0e4b18912c2d0f858ad02091b6019864c9a6d/librt-0.7.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6066c638cdf85ff92fc6f932d2d73c93a0e03492cdfa8778e6d58c489a3d7259", size = 172470, upload-time = "2026-01-01T23:51:46.823Z" }, - { url = "https://files.pythonhosted.org/packages/42/4c/dcc575b69d99076768e8dd6141d9aecd4234cba7f0e09217937f52edb6ed/librt-0.7.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a609849aca463074c17de9cda173c276eb8fee9e441053529e7b9e249dc8b8ee", size = 186806, upload-time = "2026-01-01T23:51:48.009Z" }, - { url = "https://files.pythonhosted.org/packages/fe/f8/4094a2b7816c88de81239a83ede6e87f1138477d7ee956c30f136009eb29/librt-0.7.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:add4e0a000858fe9bb39ed55f31085506a5c38363e6eb4a1e5943a10c2bfc3d1", size = 181809, upload-time = "2026-01-01T23:51:49.35Z" }, - { url = "https://files.pythonhosted.org/packages/1b/ac/821b7c0ab1b5a6cd9aee7ace8309c91545a2607185101827f79122219a7e/librt-0.7.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a3bfe73a32bd0bdb9a87d586b05a23c0a1729205d79df66dee65bb2e40d671ba", size = 175597, upload-time = "2026-01-01T23:51:50.636Z" }, - { url = "https://files.pythonhosted.org/packages/71/f9/27f6bfbcc764805864c04211c6ed636fe1d58f57a7b68d1f4ae5ed74e0e0/librt-0.7.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0ecce0544d3db91a40f8b57ae26928c02130a997b540f908cefd4d279d6c5848", size = 196506, upload-time = "2026-01-01T23:51:52.535Z" }, - { url = "https://files.pythonhosted.org/packages/46/ba/c9b9c6fc931dd7ea856c573174ccaf48714905b1a7499904db2552e3bbaf/librt-0.7.7-cp314-cp314-win32.whl", hash = "sha256:8f7a74cf3a80f0c3b0ec75b0c650b2f0a894a2cec57ef75f6f72c1e82cdac61d", size = 39747, upload-time = "2026-01-01T23:51:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/c5/69/cd1269337c4cde3ee70176ee611ab0058aa42fc8ce5c9dce55f48facfcd8/librt-0.7.7-cp314-cp314-win_amd64.whl", hash = "sha256:3d1fe2e8df3268dd6734dba33ededae72ad5c3a859b9577bc00b715759c5aaab", size = 45971, upload-time = "2026-01-01T23:51:54.697Z" }, - { url = "https://files.pythonhosted.org/packages/79/fd/e0844794423f5583108c5991313c15e2b400995f44f6ec6871f8aaf8243c/librt-0.7.7-cp314-cp314-win_arm64.whl", hash = "sha256:2987cf827011907d3dfd109f1be0d61e173d68b1270107bb0e89f2fca7f2ed6b", size = 39075, upload-time = "2026-01-01T23:51:55.726Z" }, - { url = "https://files.pythonhosted.org/packages/42/02/211fd8f7c381e7b2a11d0fdfcd410f409e89967be2e705983f7c6342209a/librt-0.7.7-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8e92c8de62b40bfce91d5e12c6e8b15434da268979b1af1a6589463549d491e6", size = 57368, upload-time = "2026-01-01T23:51:56.706Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/aca257affae73ece26041ae76032153266d110453173f67d7603058e708c/librt-0.7.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f683dcd49e2494a7535e30f779aa1ad6e3732a019d80abe1309ea91ccd3230e3", size = 59238, upload-time = "2026-01-01T23:51:58.066Z" }, - { url = "https://files.pythonhosted.org/packages/96/47/7383a507d8e0c11c78ca34c9d36eab9000db5989d446a2f05dc40e76c64f/librt-0.7.7-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b15e5d17812d4d629ff576699954f74e2cc24a02a4fc401882dd94f81daba45", size = 183870, upload-time = "2026-01-01T23:51:59.204Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/50f3d8eec8efdaf79443963624175c92cec0ba84827a66b7fcfa78598e51/librt-0.7.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c084841b879c4d9b9fa34e5d5263994f21aea7fd9c6add29194dbb41a6210536", size = 194608, upload-time = "2026-01-01T23:52:00.419Z" }, - { url = "https://files.pythonhosted.org/packages/23/d9/1b6520793aadb59d891e3b98ee057a75de7f737e4a8b4b37fdbecb10d60f/librt-0.7.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c8fb9966f84737115513fecbaf257f9553d067a7dd45a69c2c7e5339e6a8dc", size = 206776, upload-time = "2026-01-01T23:52:01.705Z" }, - { url = "https://files.pythonhosted.org/packages/ff/db/331edc3bba929d2756fa335bfcf736f36eff4efcb4f2600b545a35c2ae58/librt-0.7.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5fb1ecb2c35362eab2dbd354fd1efa5a8440d3e73a68be11921042a0edc0ff", size = 203206, upload-time = "2026-01-01T23:52:03.315Z" }, - { url = "https://files.pythonhosted.org/packages/b2/e1/6af79ec77204e85f6f2294fc171a30a91bb0e35d78493532ed680f5d98be/librt-0.7.7-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d1454899909d63cc9199a89fcc4f81bdd9004aef577d4ffc022e600c412d57f3", size = 196697, upload-time = "2026-01-01T23:52:04.857Z" }, - { url = "https://files.pythonhosted.org/packages/f3/46/de55ecce4b2796d6d243295c221082ca3a944dc2fb3a52dcc8660ce7727d/librt-0.7.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7ef28f2e7a016b29792fe0a2dd04dec75725b32a1264e390c366103f834a9c3a", size = 217193, upload-time = "2026-01-01T23:52:06.159Z" }, - { url = "https://files.pythonhosted.org/packages/41/61/33063e271949787a2f8dd33c5260357e3d512a114fc82ca7890b65a76e2d/librt-0.7.7-cp314-cp314t-win32.whl", hash = "sha256:5e419e0db70991b6ba037b70c1d5bbe92b20ddf82f31ad01d77a347ed9781398", size = 40277, upload-time = "2026-01-01T23:52:07.625Z" }, - { url = "https://files.pythonhosted.org/packages/06/21/1abd972349f83a696ea73159ac964e63e2d14086fdd9bc7ca878c25fced4/librt-0.7.7-cp314-cp314t-win_amd64.whl", hash = "sha256:d6b7d93657332c817b8d674ef6bf1ab7796b4f7ce05e420fd45bd258a72ac804", size = 46765, upload-time = "2026-01-01T23:52:08.647Z" }, - { url = "https://files.pythonhosted.org/packages/51/0e/b756c7708143a63fca65a51ca07990fa647db2cc8fcd65177b9e96680255/librt-0.7.7-cp314-cp314t-win_arm64.whl", hash = "sha256:142c2cd91794b79fd0ce113bd658993b7ede0fe93057668c2f98a45ca00b7e91", size = 39724, upload-time = "2026-01-01T23:52:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a3/87ea9c1049f2c781177496ebee29430e4631f439b8553a4969c88747d5d8/librt-0.7.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ff3e9c11aa260c31493d4b3197d1e28dd07768594a4f92bec4506849d736248f", size = 56507, upload-time = "2026-01-14T12:54:54.156Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4a/23bcef149f37f771ad30203d561fcfd45b02bc54947b91f7a9ac34815747/librt-0.7.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ddb52499d0b3ed4aa88746aaf6f36a08314677d5c346234c3987ddc506404eac", size = 58455, upload-time = "2026-01-14T12:54:55.978Z" }, + { url = "https://files.pythonhosted.org/packages/22/6e/46eb9b85c1b9761e0f42b6e6311e1cc544843ac897457062b9d5d0b21df4/librt-0.7.8-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e9c0afebbe6ce177ae8edba0c7c4d626f2a0fc12c33bb993d163817c41a7a05c", size = 164956, upload-time = "2026-01-14T12:54:57.311Z" }, + { url = "https://files.pythonhosted.org/packages/7a/3f/aa7c7f6829fb83989feb7ba9aa11c662b34b4bd4bd5b262f2876ba3db58d/librt-0.7.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:631599598e2c76ded400c0a8722dec09217c89ff64dc54b060f598ed68e7d2a8", size = 174364, upload-time = "2026-01-14T12:54:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2d/d57d154b40b11f2cb851c4df0d4c4456bacd9b1ccc4ecb593ddec56c1a8b/librt-0.7.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c1ba843ae20db09b9d5c80475376168feb2640ce91cd9906414f23cc267a1ff", size = 188034, upload-time = "2026-01-14T12:55:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/59/f9/36c4dad00925c16cd69d744b87f7001792691857d3b79187e7a673e812fb/librt-0.7.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b5b007bb22ea4b255d3ee39dfd06d12534de2fcc3438567d9f48cdaf67ae1ae3", size = 186295, upload-time = "2026-01-14T12:55:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/23/9b/8a9889d3df5efb67695a67785028ccd58e661c3018237b73ad081691d0cb/librt-0.7.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dbd79caaf77a3f590cbe32dc2447f718772d6eea59656a7dcb9311161b10fa75", size = 181470, upload-time = "2026-01-14T12:55:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/43/64/54d6ef11afca01fef8af78c230726a9394759f2addfbf7afc5e3cc032a45/librt-0.7.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:87808a8d1e0bd62a01cafc41f0fd6818b5a5d0ca0d8a55326a81643cdda8f873", size = 201713, upload-time = "2026-01-14T12:55:03.919Z" }, + { url = "https://files.pythonhosted.org/packages/2d/29/73e7ed2991330b28919387656f54109139b49e19cd72902f466bd44415fd/librt-0.7.8-cp311-cp311-win32.whl", hash = "sha256:31724b93baa91512bd0a376e7cf0b59d8b631ee17923b1218a65456fa9bda2e7", size = 43803, upload-time = "2026-01-14T12:55:04.996Z" }, + { url = "https://files.pythonhosted.org/packages/3f/de/66766ff48ed02b4d78deea30392ae200bcbd99ae61ba2418b49fd50a4831/librt-0.7.8-cp311-cp311-win_amd64.whl", hash = "sha256:978e8b5f13e52cf23a9e80f3286d7546baa70bc4ef35b51d97a709d0b28e537c", size = 50080, upload-time = "2026-01-14T12:55:06.489Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e3/33450438ff3a8c581d4ed7f798a70b07c3206d298cf0b87d3806e72e3ed8/librt-0.7.8-cp311-cp311-win_arm64.whl", hash = "sha256:20e3946863d872f7cabf7f77c6c9d370b8b3d74333d3a32471c50d3a86c0a232", size = 43383, upload-time = "2026-01-14T12:55:07.49Z" }, + { url = "https://files.pythonhosted.org/packages/56/04/79d8fcb43cae376c7adbab7b2b9f65e48432c9eced62ac96703bcc16e09b/librt-0.7.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9b6943885b2d49c48d0cff23b16be830ba46b0152d98f62de49e735c6e655a63", size = 57472, upload-time = "2026-01-14T12:55:08.528Z" }, + { url = "https://files.pythonhosted.org/packages/b4/ba/60b96e93043d3d659da91752689023a73981336446ae82078cddf706249e/librt-0.7.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46ef1f4b9b6cc364b11eea0ecc0897314447a66029ee1e55859acb3dd8757c93", size = 58986, upload-time = "2026-01-14T12:55:09.466Z" }, + { url = "https://files.pythonhosted.org/packages/7c/26/5215e4cdcc26e7be7eee21955a7e13cbf1f6d7d7311461a6014544596fac/librt-0.7.8-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:907ad09cfab21e3c86e8f1f87858f7049d1097f77196959c033612f532b4e592", size = 168422, upload-time = "2026-01-14T12:55:10.499Z" }, + { url = "https://files.pythonhosted.org/packages/0f/84/e8d1bc86fa0159bfc24f3d798d92cafd3897e84c7fea7fe61b3220915d76/librt-0.7.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2991b6c3775383752b3ca0204842743256f3ad3deeb1d0adc227d56b78a9a850", size = 177478, upload-time = "2026-01-14T12:55:11.577Z" }, + { url = "https://files.pythonhosted.org/packages/57/11/d0268c4b94717a18aa91df1100e767b010f87b7ae444dafaa5a2d80f33a6/librt-0.7.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03679b9856932b8c8f674e87aa3c55ea11c9274301f76ae8dc4d281bda55cf62", size = 192439, upload-time = "2026-01-14T12:55:12.7Z" }, + { url = "https://files.pythonhosted.org/packages/8d/56/1e8e833b95fe684f80f8894ae4d8b7d36acc9203e60478fcae599120a975/librt-0.7.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3968762fec1b2ad34ce57458b6de25dbb4142713e9ca6279a0d352fa4e9f452b", size = 191483, upload-time = "2026-01-14T12:55:13.838Z" }, + { url = "https://files.pythonhosted.org/packages/17/48/f11cf28a2cb6c31f282009e2208312aa84a5ee2732859f7856ee306176d5/librt-0.7.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:bb7a7807523a31f03061288cc4ffc065d684c39db7644c676b47d89553c0d714", size = 185376, upload-time = "2026-01-14T12:55:15.017Z" }, + { url = "https://files.pythonhosted.org/packages/b8/6a/d7c116c6da561b9155b184354a60a3d5cdbf08fc7f3678d09c95679d13d9/librt-0.7.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad64a14b1e56e702e19b24aae108f18ad1bf7777f3af5fcd39f87d0c5a814449", size = 206234, upload-time = "2026-01-14T12:55:16.571Z" }, + { url = "https://files.pythonhosted.org/packages/61/de/1975200bb0285fc921c5981d9978ce6ce11ae6d797df815add94a5a848a3/librt-0.7.8-cp312-cp312-win32.whl", hash = "sha256:0241a6ed65e6666236ea78203a73d800dbed896cf12ae25d026d75dc1fcd1dac", size = 44057, upload-time = "2026-01-14T12:55:18.077Z" }, + { url = "https://files.pythonhosted.org/packages/8e/cd/724f2d0b3461426730d4877754b65d39f06a41ac9d0a92d5c6840f72b9ae/librt-0.7.8-cp312-cp312-win_amd64.whl", hash = "sha256:6db5faf064b5bab9675c32a873436b31e01d66ca6984c6f7f92621656033a708", size = 50293, upload-time = "2026-01-14T12:55:19.179Z" }, + { url = "https://files.pythonhosted.org/packages/bd/cf/7e899acd9ee5727ad8160fdcc9994954e79fab371c66535c60e13b968ffc/librt-0.7.8-cp312-cp312-win_arm64.whl", hash = "sha256:57175aa93f804d2c08d2edb7213e09276bd49097611aefc37e3fa38d1fb99ad0", size = 43574, upload-time = "2026-01-14T12:55:20.185Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fe/b1f9de2829cf7fc7649c1dcd202cfd873837c5cc2fc9e526b0e7f716c3d2/librt-0.7.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4c3995abbbb60b3c129490fa985dfe6cac11d88fc3c36eeb4fb1449efbbb04fc", size = 57500, upload-time = "2026-01-14T12:55:21.219Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d4/4a60fbe2e53b825f5d9a77325071d61cd8af8506255067bf0c8527530745/librt-0.7.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:44e0c2cbc9bebd074cf2cdbe472ca185e824be4e74b1c63a8e934cea674bebf2", size = 59019, upload-time = "2026-01-14T12:55:22.256Z" }, + { url = "https://files.pythonhosted.org/packages/6a/37/61ff80341ba5159afa524445f2d984c30e2821f31f7c73cf166dcafa5564/librt-0.7.8-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d2f1e492cae964b3463a03dc77a7fe8742f7855d7258c7643f0ee32b6651dd3", size = 169015, upload-time = "2026-01-14T12:55:23.24Z" }, + { url = "https://files.pythonhosted.org/packages/1c/86/13d4f2d6a93f181ebf2fc953868826653ede494559da8268023fe567fca3/librt-0.7.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:451e7ffcef8f785831fdb791bd69211f47e95dc4c6ddff68e589058806f044c6", size = 178161, upload-time = "2026-01-14T12:55:24.826Z" }, + { url = "https://files.pythonhosted.org/packages/88/26/e24ef01305954fc4d771f1f09f3dd682f9eb610e1bec188ffb719374d26e/librt-0.7.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3469e1af9f1380e093ae06bedcbdd11e407ac0b303a56bbe9afb1d6824d4982d", size = 193015, upload-time = "2026-01-14T12:55:26.04Z" }, + { url = "https://files.pythonhosted.org/packages/88/a0/92b6bd060e720d7a31ed474d046a69bd55334ec05e9c446d228c4b806ae3/librt-0.7.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f11b300027ce19a34f6d24ebb0a25fd0e24a9d53353225a5c1e6cadbf2916b2e", size = 192038, upload-time = "2026-01-14T12:55:27.208Z" }, + { url = "https://files.pythonhosted.org/packages/06/bb/6f4c650253704279c3a214dad188101d1b5ea23be0606628bc6739456624/librt-0.7.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4adc73614f0d3c97874f02f2c7fd2a27854e7e24ad532ea6b965459c5b757eca", size = 186006, upload-time = "2026-01-14T12:55:28.594Z" }, + { url = "https://files.pythonhosted.org/packages/dc/00/1c409618248d43240cadf45f3efb866837fa77e9a12a71481912135eb481/librt-0.7.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:60c299e555f87e4c01b2eca085dfccda1dde87f5a604bb45c2906b8305819a93", size = 206888, upload-time = "2026-01-14T12:55:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/d9/83/b2cfe8e76ff5c1c77f8a53da3d5de62d04b5ebf7cf913e37f8bca43b5d07/librt-0.7.8-cp313-cp313-win32.whl", hash = "sha256:b09c52ed43a461994716082ee7d87618096851319bf695d57ec123f2ab708951", size = 44126, upload-time = "2026-01-14T12:55:31.44Z" }, + { url = "https://files.pythonhosted.org/packages/a9/0b/c59d45de56a51bd2d3a401fc63449c0ac163e4ef7f523ea8b0c0dee86ec5/librt-0.7.8-cp313-cp313-win_amd64.whl", hash = "sha256:f8f4a901a3fa28969d6e4519deceab56c55a09d691ea7b12ca830e2fa3461e34", size = 50262, upload-time = "2026-01-14T12:55:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b9/973455cec0a1ec592395250c474164c4a58ebf3e0651ee920fef1a2623f1/librt-0.7.8-cp313-cp313-win_arm64.whl", hash = "sha256:43d4e71b50763fcdcf64725ac680d8cfa1706c928b844794a7aa0fa9ac8e5f09", size = 43600, upload-time = "2026-01-14T12:55:34.054Z" }, + { url = "https://files.pythonhosted.org/packages/1a/73/fa8814c6ce2d49c3827829cadaa1589b0bf4391660bd4510899393a23ebc/librt-0.7.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:be927c3c94c74b05128089a955fba86501c3b544d1d300282cc1b4bd370cb418", size = 57049, upload-time = "2026-01-14T12:55:35.056Z" }, + { url = "https://files.pythonhosted.org/packages/53/fe/f6c70956da23ea235fd2e3cc16f4f0b4ebdfd72252b02d1164dd58b4e6c3/librt-0.7.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7b0803e9008c62a7ef79058233db7ff6f37a9933b8f2573c05b07ddafa226611", size = 58689, upload-time = "2026-01-14T12:55:36.078Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4d/7a2481444ac5fba63050d9abe823e6bc16896f575bfc9c1e5068d516cdce/librt-0.7.8-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:79feb4d00b2a4e0e05c9c56df707934f41fcb5fe53fd9efb7549068d0495b758", size = 166808, upload-time = "2026-01-14T12:55:37.595Z" }, + { url = "https://files.pythonhosted.org/packages/ac/3c/10901d9e18639f8953f57c8986796cfbf4c1c514844a41c9197cf87cb707/librt-0.7.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9122094e3f24aa759c38f46bd8863433820654927370250f460ae75488b66ea", size = 175614, upload-time = "2026-01-14T12:55:38.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/01/5cbdde0951a5090a80e5ba44e6357d375048123c572a23eecfb9326993a7/librt-0.7.8-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e03bea66af33c95ce3addf87a9bf1fcad8d33e757bc479957ddbc0e4f7207ac", size = 189955, upload-time = "2026-01-14T12:55:39.939Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b4/e80528d2f4b7eaf1d437fcbd6fc6ba4cbeb3e2a0cb9ed5a79f47c7318706/librt-0.7.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f1ade7f31675db00b514b98f9ab9a7698c7282dad4be7492589109471852d398", size = 189370, upload-time = "2026-01-14T12:55:41.057Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ab/938368f8ce31a9787ecd4becb1e795954782e4312095daf8fd22420227c8/librt-0.7.8-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a14229ac62adcf1b90a15992f1ab9c69ae8b99ffb23cb64a90878a6e8a2f5b81", size = 183224, upload-time = "2026-01-14T12:55:42.328Z" }, + { url = "https://files.pythonhosted.org/packages/3c/10/559c310e7a6e4014ac44867d359ef8238465fb499e7eb31b6bfe3e3f86f5/librt-0.7.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5bcaaf624fd24e6a0cb14beac37677f90793a96864c67c064a91458611446e83", size = 203541, upload-time = "2026-01-14T12:55:43.501Z" }, + { url = "https://files.pythonhosted.org/packages/f8/db/a0db7acdb6290c215f343835c6efda5b491bb05c3ddc675af558f50fdba3/librt-0.7.8-cp314-cp314-win32.whl", hash = "sha256:7aa7d5457b6c542ecaed79cec4ad98534373c9757383973e638ccced0f11f46d", size = 40657, upload-time = "2026-01-14T12:55:44.668Z" }, + { url = "https://files.pythonhosted.org/packages/72/e0/4f9bdc2a98a798511e81edcd6b54fe82767a715e05d1921115ac70717f6f/librt-0.7.8-cp314-cp314-win_amd64.whl", hash = "sha256:3d1322800771bee4a91f3b4bd4e49abc7d35e65166821086e5afd1e6c0d9be44", size = 46835, upload-time = "2026-01-14T12:55:45.655Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3d/59c6402e3dec2719655a41ad027a7371f8e2334aa794ed11533ad5f34969/librt-0.7.8-cp314-cp314-win_arm64.whl", hash = "sha256:5363427bc6a8c3b1719f8f3845ea53553d301382928a86e8fab7984426949bce", size = 39885, upload-time = "2026-01-14T12:55:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9c/2481d80950b83085fb14ba3c595db56330d21bbc7d88a19f20165f3538db/librt-0.7.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ca916919793a77e4a98d4a1701e345d337ce53be4a16620f063191f7322ac80f", size = 59161, upload-time = "2026-01-14T12:55:48.45Z" }, + { url = "https://files.pythonhosted.org/packages/96/79/108df2cfc4e672336765d54e3ff887294c1cc36ea4335c73588875775527/librt-0.7.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:54feb7b4f2f6706bb82325e836a01be805770443e2400f706e824e91f6441dde", size = 61008, upload-time = "2026-01-14T12:55:49.527Z" }, + { url = "https://files.pythonhosted.org/packages/46/f2/30179898f9994a5637459d6e169b6abdc982012c0a4b2d4c26f50c06f911/librt-0.7.8-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:39a4c76fee41007070f872b648cc2f711f9abf9a13d0c7162478043377b52c8e", size = 187199, upload-time = "2026-01-14T12:55:50.587Z" }, + { url = "https://files.pythonhosted.org/packages/b4/da/f7563db55cebdc884f518ba3791ad033becc25ff68eb70902b1747dc0d70/librt-0.7.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac9c8a458245c7de80bc1b9765b177055efff5803f08e548dd4bb9ab9a8d789b", size = 198317, upload-time = "2026-01-14T12:55:51.991Z" }, + { url = "https://files.pythonhosted.org/packages/b3/6c/4289acf076ad371471fa86718c30ae353e690d3de6167f7db36f429272f1/librt-0.7.8-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b67aa7eff150f075fda09d11f6bfb26edffd300f6ab1666759547581e8f666", size = 210334, upload-time = "2026-01-14T12:55:53.682Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7f/377521ac25b78ac0a5ff44127a0360ee6d5ddd3ce7327949876a30533daa/librt-0.7.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:535929b6eff670c593c34ff435d5440c3096f20fa72d63444608a5aef64dd581", size = 211031, upload-time = "2026-01-14T12:55:54.827Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/e1e96c3e20b23d00cf90f4aad48f0deb4cdfec2f0ed8380d0d85acf98bbf/librt-0.7.8-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:63937bd0f4d1cb56653dc7ae900d6c52c41f0015e25aaf9902481ee79943b33a", size = 204581, upload-time = "2026-01-14T12:55:56.811Z" }, + { url = "https://files.pythonhosted.org/packages/43/71/0f5d010e92ed9747e14bef35e91b6580533510f1e36a8a09eb79ee70b2f0/librt-0.7.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cf243da9e42d914036fd362ac3fa77d80a41cadcd11ad789b1b5eec4daaf67ca", size = 224731, upload-time = "2026-01-14T12:55:58.175Z" }, + { url = "https://files.pythonhosted.org/packages/22/f0/07fb6ab5c39a4ca9af3e37554f9d42f25c464829254d72e4ebbd81da351c/librt-0.7.8-cp314-cp314t-win32.whl", hash = "sha256:171ca3a0a06c643bd0a2f62a8944e1902c94aa8e5da4db1ea9a8daf872685365", size = 41173, upload-time = "2026-01-14T12:55:59.315Z" }, + { url = "https://files.pythonhosted.org/packages/24/d4/7e4be20993dc6a782639625bd2f97f3c66125c7aa80c82426956811cfccf/librt-0.7.8-cp314-cp314t-win_amd64.whl", hash = "sha256:445b7304145e24c60288a2f172b5ce2ca35c0f81605f5299f3fa567e189d2e32", size = 47668, upload-time = "2026-01-14T12:56:00.261Z" }, + { url = "https://files.pythonhosted.org/packages/fc/85/69f92b2a7b3c0f88ffe107c86b952b397004b5b8ea5a81da3d9c04c04422/librt-0.7.8-cp314-cp314t-win_arm64.whl", hash = "sha256:8766ece9de08527deabcd7cb1b4f1a967a385d26e33e536d6d8913db6ef74f06", size = 40550, upload-time = "2026-01-14T12:56:01.542Z" }, ] [[package]] @@ -250,11 +250,11 @@ sdist = { url = "https://files.pythonhosted.org/packages/71/c5/2564d917503fe8d68 [[package]] name = "pathspec" -version = "0.12.1" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/b2/bb8e495d5262bfec41ab5cb18f522f1012933347fb5d9e62452d446baca2/pathspec-1.0.3.tar.gz", hash = "sha256:bac5cf97ae2c2876e2d25ebb15078eb04d76e4b98921ee31c6f85ade8b59444d", size = 130841, upload-time = "2026-01-09T15:46:46.009Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/121e912bd60eebd623f873fd090de0e84f322972ab25a7f9044c056804ed/pathspec-1.0.3-py3-none-any.whl", hash = "sha256:e80767021c1cc524aa3fb14bedda9c34406591343cc42797b386ce7b9354fb6c", size = 55021, upload-time = "2026-01-09T15:46:44.652Z" }, ] [[package]] @@ -510,14 +510,14 @@ dev = [ [[package]] name = "types-jsonschema" -version = "4.25.1.20251009" +version = "4.26.0.20260109" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "referencing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/da/5b901088da5f710690b422137e8ae74197fb1ca471e4aa84dd3ef0d6e295/types_jsonschema-4.25.1.20251009.tar.gz", hash = "sha256:75d0f5c5dd18dc23b664437a0c1a625743e8d2e665ceaf3aecb29841f3a5f97f", size = 15661, upload-time = "2025-10-09T02:54:36.963Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/03/a1509b0c13fc7a1fca1494c84bde8cce8a5c0582b6255b9640ebd3034017/types_jsonschema-4.26.0.20260109.tar.gz", hash = "sha256:340fe91e6ea517900d6ababb6262a86c176473b5bf8455b96e85a89e3cfb5daa", size = 15886, upload-time = "2026-01-09T03:21:45.464Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/6a/e5146754c0dfc272f176db9c245bc43cc19030262d891a5a85d472797e60/types_jsonschema-4.25.1.20251009-py3-none-any.whl", hash = "sha256:f30b329037b78e7a60146b1146feb0b6fb0b71628637584409bada83968dad3e", size = 15925, upload-time = "2025-10-09T02:54:35.847Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d6/3db3134f35f1e4bf9a13517d759c1ae64086eddb8ad0047caee140021e64/types_jsonschema-4.26.0.20260109-py3-none-any.whl", hash = "sha256:e0276640d228732fb75d883905d607359b24a4ff745ba7f9a5f50e6fda891926", size = 15923, upload-time = "2026-01-09T03:21:43.828Z" }, ] [[package]] diff --git a/meltano/meltano.yml b/meltano/meltano.yml index cee4305..3d8bba7 100644 --- a/meltano/meltano.yml +++ b/meltano/meltano.yml @@ -55,6 +55,9 @@ plugins: - salesforce-merged_user.lastmodifiedbyid metadata: '*': + replication-method: INCREMENTAL + replication-key: systemmodstamp + 'platform-collaboration__c': replication-method: INCREMENTAL replication-key: lastmodifieddate - name: tap-dynamodb diff --git a/pyproject.toml b/pyproject.toml index 759f444..5ff6a61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,5 +7,5 @@ version = "0.1.1" description = "LFX v1<>v2 data sync components" requires-python = ">=3.12,<3.13" dependencies = [ - "meltano[s3]==4.0.6", + "meltano[s3]==4.0.8", ] diff --git a/uv.lock b/uv.lock index ec44192..b16069f 100644 --- a/uv.lock +++ b/uv.lock @@ -4,29 +4,29 @@ requires-python = "==3.12.*" [[package]] name = "alembic" -version = "1.17.2" +version = "1.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/a6/74c8cadc2882977d80ad756a13857857dbcf9bd405bc80b662eb10651282/alembic-1.17.2.tar.gz", hash = "sha256:bbe9751705c5e0f14877f02d46c53d10885e377e3d90eda810a016f9baa19e8e", size = 1988064, upload-time = "2025-11-14T20:35:04.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/cc/aca263693b2ece99fa99a09b6d092acb89973eb2bb575faef1777e04f8b4/alembic-1.18.1.tar.gz", hash = "sha256:83ac6b81359596816fb3b893099841a0862f2117b2963258e965d70dc62fb866", size = 2044319, upload-time = "2026-01-14T18:53:14.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl", hash = "sha256:f483dd1fe93f6c5d49217055e4d15b905b425b6af906746abb35b69c1996c4e6", size = 248554, upload-time = "2025-11-14T20:35:05.699Z" }, + { url = "https://files.pythonhosted.org/packages/83/36/cd9cb6101e81e39076b2fbe303bfa3c85ca34e55142b0324fcbf22c5c6e2/alembic-1.18.1-py3-none-any.whl", hash = "sha256:f1c3b0920b87134e851c25f1f7f236d8a332c34b75416802d06971df5d1b7810", size = 260973, upload-time = "2026-01-14T18:53:17.533Z" }, ] [[package]] name = "anyio" -version = "4.12.0" +version = "4.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/ce/8a777047513153587e5434fd752e89334ac33e379aa3497db860eeb60377/anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0", size = 228266, upload-time = "2025-11-28T23:37:38.911Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362, upload-time = "2025-11-28T23:36:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] [[package]] @@ -40,39 +40,39 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.76" +version = "1.42.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/04/8cf6cf7e6390c71b9c958f3bfedc45d1182b51a35f7789354bf7b2ff4e8c/boto3-1.40.76.tar.gz", hash = "sha256:16f4cf97f8dd8e0aae015f4dc66219bd7716a91a40d1e2daa0dafa241a4761c5", size = 111598, upload-time = "2025-11-18T20:23:10.938Z" } +sdist = { url = "https://files.pythonhosted.org/packages/42/79/2dac8b7cb075cfa43908ee9af3f8ee06880d84b86013854c5cca8945afac/boto3-1.42.30.tar.gz", hash = "sha256:ba9cd2f7819637d15bfbeb63af4c567fcc8a7dcd7b93dd12734ec58601169538", size = 112809, upload-time = "2026-01-16T20:37:23.636Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/8e/966263696eb441e8d1c4daa5fdfb3b4be10a96a23c418cc74c80b0b03d4e/boto3-1.40.76-py3-none-any.whl", hash = "sha256:8df6df755727be40ad9e309cfda07f9a12c147e17b639430c55d4e4feee8a167", size = 139359, upload-time = "2025-11-18T20:23:08.75Z" }, + { url = "https://files.pythonhosted.org/packages/52/b3/2c0d828c9f668292e277ca5232e6160dd5b4b660a3f076f20dd5378baa1e/boto3-1.42.30-py3-none-any.whl", hash = "sha256:d7e548bea65e0ae2c465c77de937bc686b591aee6a352d5a19a16bc751e591c1", size = 140573, upload-time = "2026-01-16T20:37:22.089Z" }, ] [[package]] name = "botocore" -version = "1.40.76" +version = "1.42.30" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/eb/50e2d280589a3c20c3b649bb66262d2b53a25c03262e4cc492048ac7540a/botocore-1.40.76.tar.gz", hash = "sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc", size = 14494001, upload-time = "2025-11-18T20:22:59.131Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/38/23862628a0eb044c8b8b3d7a9ad1920b3bfd6bce6d746d5a871e8382c7e4/botocore-1.42.30.tar.gz", hash = "sha256:9bf1662b8273d5cc3828a49f71ca85abf4e021011c1f0a71f41a2ea5769a5116", size = 14891439, upload-time = "2026-01-16T20:37:13.77Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8d/6d7b016383b1f74dd93611b1c5078bbaddaca901553ab886dcda87cae365/botocore-1.42.30-py3-none-any.whl", hash = "sha256:97070a438cac92430bb7b65f8ebd7075224f4a289719da4ee293d22d1e98db02", size = 14566340, upload-time = "2026-01-16T20:37:10.94Z" }, ] [[package]] name = "certifi" -version = "2025.11.12" +version = "2026.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/8c/58f469717fa48465e4a50c014a0400602d3c437d7c0c468e17ada824da3a/certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316", size = 160538, upload-time = "2025-11-12T02:54:51.517Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, ] [[package]] @@ -140,18 +140,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/1a/aff8bb287a4b1400f69e09a53bd65de96aa5cee5691925b38731c67fc695/click_default_group-1.2.4-py2.py3-none-any.whl", hash = "sha256:9b60486923720e7fc61731bdb32b617039aba820e22e1c88766b1125592eaa5f", size = 4123, upload-time = "2023-08-04T07:54:56.875Z" }, ] -[[package]] -name = "click-didyoumean" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -196,11 +184,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.20.1" +version = "3.20.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/23/ce7a1126827cedeb958fc043d61745754464eb56c5937c35bbf2b8e26f34/filelock-3.20.1.tar.gz", hash = "sha256:b8360948b351b80f420878d8516519a2204b07aefcdcfd24912a5d33127f188c", size = 19476, upload-time = "2025-12-15T23:54:28.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/7f/a1a97644e39e7316d850784c642093c99df1290a460df4ede27659056834/filelock-3.20.1-py3-none-any.whl", hash = "sha256:15d9e9a67306188a44baa72f569d2bfd803076269365fdea0934385da4dc361a", size = 16666, upload-time = "2025-12-15T23:54:26.874Z" }, + { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] [[package]] @@ -212,7 +200,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, - { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -251,7 +238,7 @@ wheels = [ [[package]] name = "jsonschema" -version = "4.25.1" +version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -259,9 +246,9 @@ dependencies = [ { name = "referencing" }, { name = "rpds-py" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] [[package]] @@ -285,7 +272,7 @@ dependencies = [ ] [package.metadata] -requires-dist = [{ name = "meltano", extras = ["s3"], specifier = "==4.0.6" }] +requires-dist = [{ name = "meltano", extras = ["s3"], specifier = "==4.0.8" }] [[package]] name = "mako" @@ -341,7 +328,7 @@ wheels = [ [[package]] name = "meltano" -version = "4.0.6" +version = "4.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alembic" }, @@ -349,7 +336,6 @@ dependencies = [ { name = "check-jsonschema" }, { name = "click" }, { name = "click-default-group" }, - { name = "click-didyoumean" }, { name = "dateparser" }, { name = "fasteners" }, { name = "jinja2" }, @@ -373,9 +359,9 @@ dependencies = [ { name = "uv" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/57/762f941a151ec7e81b5cf3dd4b0b03b4aac38ad2c01066013fcfb231098f/meltano-4.0.6.tar.gz", hash = "sha256:24cb2166f60d56aa340ee6ad07cbb4d1fc4e2895fc35c7a8c5d539ac30f51f0d", size = 12759697, upload-time = "2025-11-20T00:09:27.394Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/41/2c1cb004f744b5844c885dc40c4039b5f59bd672a26ca252bc237a0af467/meltano-4.0.8.tar.gz", hash = "sha256:78dda8af3b6db01d6f7ac3369441bdfcb8f7519f1e5ca6e3f60e77c5f5f4aae6", size = 12763967, upload-time = "2025-12-17T19:57:10.296Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/c5/38296f22f765af7d8934de04bbd57ba22aae00698139f1a7e0aee2471551/meltano-4.0.6-py3-none-any.whl", hash = "sha256:5d22b5b494775c7d602789e92dd165db53d94d2e9f693bee73c8950adf098646", size = 394433, upload-time = "2025-11-20T00:09:25.039Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1b/6d8a3c0a8ca8a3dab9ea4a4e1b161830e8e79482c06266a75f736135d600/meltano-4.0.8-py3-none-any.whl", hash = "sha256:538a25d2d8c7a91cd25620b0721455ca6881618221dd7ce2bfe5f4249436e556", size = 396017, upload-time = "2025-12-17T19:57:08.507Z" }, ] [package.optional-dependencies] @@ -412,16 +398,18 @@ wheels = [ [[package]] name = "psutil" -version = "7.1.3" +version = "7.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz", hash = "sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74", size = 489059, upload-time = "2025-11-02T12:25:54.619Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz", hash = "sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3", size = 490253, upload-time = "2025-12-29T08:26:00.169Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab", size = 238359, upload-time = "2025-11-02T12:26:25.284Z" }, - { url = "https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880", size = 239171, upload-time = "2025-11-02T12:26:27.23Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3", size = 263261, upload-time = "2025-11-02T12:26:29.48Z" }, - { url = "https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b", size = 264635, upload-time = "2025-11-02T12:26:31.74Z" }, - { url = "https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd", size = 247633, upload-time = "2025-11-02T12:26:33.887Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl", hash = "sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1", size = 244608, upload-time = "2025-11-02T12:26:36.136Z" }, + { url = "https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42", size = 128137, upload-time = "2025-12-29T08:26:27.759Z" }, + { url = "https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1", size = 128947, upload-time = "2025-12-29T08:26:29.548Z" }, + { url = "https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8", size = 154694, upload-time = "2025-12-29T08:26:32.147Z" }, + { url = "https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6", size = 156136, upload-time = "2025-12-29T08:26:34.079Z" }, + { url = "https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8", size = 148108, upload-time = "2025-12-29T08:26:36.225Z" }, + { url = "https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67", size = 147402, upload-time = "2025-12-29T08:26:39.21Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/d9317542e3f2b180c4306e3f45d3c922d7e86d8ce39f941bb9e2e9d8599e/psutil-7.2.1-cp37-abi3-win_amd64.whl", hash = "sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17", size = 136938, upload-time = "2025-12-29T08:26:41.036Z" }, + { url = "https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl", hash = "sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442", size = 133836, upload-time = "2025-12-29T08:26:43.086Z" }, ] [[package]] @@ -497,24 +485,26 @@ wheels = [ [[package]] name = "regex" -version = "2025.11.3" +version = "2026.1.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/a9/546676f25e573a4cf00fe8e119b78a37b6a8fe2dc95cda877b30889c9c45/regex-2025.11.3.tar.gz", hash = "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", size = 414669, upload-time = "2025-11-03T21:34:22.089Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/86/07d5056945f9ec4590b518171c4254a5925832eb727b56d3c38a7476f316/regex-2026.1.15.tar.gz", hash = "sha256:164759aa25575cbc0651bef59a0b18353e54300d79ace8084c818ad8ac72b7d5", size = 414811, upload-time = "2026-01-14T23:18:02.775Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/74/18f04cb53e58e3fb107439699bd8375cf5a835eec81084e0bddbd122e4c2/regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", size = 489312, upload-time = "2025-11-03T21:31:34.343Z" }, - { url = "https://files.pythonhosted.org/packages/78/3f/37fcdd0d2b1e78909108a876580485ea37c91e1acf66d3bb8e736348f441/regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", size = 291256, upload-time = "2025-11-03T21:31:35.675Z" }, - { url = "https://files.pythonhosted.org/packages/bf/26/0a575f58eb23b7ebd67a45fccbc02ac030b737b896b7e7a909ffe43ffd6a/regex-2025.11.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", size = 288921, upload-time = "2025-11-03T21:31:37.07Z" }, - { url = "https://files.pythonhosted.org/packages/ea/98/6a8dff667d1af907150432cf5abc05a17ccd32c72a3615410d5365ac167a/regex-2025.11.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", size = 798568, upload-time = "2025-11-03T21:31:38.784Z" }, - { url = "https://files.pythonhosted.org/packages/64/15/92c1db4fa4e12733dd5a526c2dd2b6edcbfe13257e135fc0f6c57f34c173/regex-2025.11.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", size = 864165, upload-time = "2025-11-03T21:31:40.559Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e7/3ad7da8cdee1ce66c7cd37ab5ab05c463a86ffeb52b1a25fe7bd9293b36c/regex-2025.11.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", size = 912182, upload-time = "2025-11-03T21:31:42.002Z" }, - { url = "https://files.pythonhosted.org/packages/84/bd/9ce9f629fcb714ffc2c3faf62b6766ecb7a585e1e885eb699bcf130a5209/regex-2025.11.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", size = 803501, upload-time = "2025-11-03T21:31:43.815Z" }, - { url = "https://files.pythonhosted.org/packages/7c/0f/8dc2e4349d8e877283e6edd6c12bdcebc20f03744e86f197ab6e4492bf08/regex-2025.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", size = 787842, upload-time = "2025-11-03T21:31:45.353Z" }, - { url = "https://files.pythonhosted.org/packages/f9/73/cff02702960bc185164d5619c0c62a2f598a6abff6695d391b096237d4ab/regex-2025.11.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", size = 858519, upload-time = "2025-11-03T21:31:46.814Z" }, - { url = "https://files.pythonhosted.org/packages/61/83/0e8d1ae71e15bc1dc36231c90b46ee35f9d52fab2e226b0e039e7ea9c10a/regex-2025.11.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", size = 850611, upload-time = "2025-11-03T21:31:48.289Z" }, - { url = "https://files.pythonhosted.org/packages/c8/f5/70a5cdd781dcfaa12556f2955bf170cd603cb1c96a1827479f8faea2df97/regex-2025.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", size = 789759, upload-time = "2025-11-03T21:31:49.759Z" }, - { url = "https://files.pythonhosted.org/packages/59/9b/7c29be7903c318488983e7d97abcf8ebd3830e4c956c4c540005fcfb0462/regex-2025.11.3-cp312-cp312-win32.whl", hash = "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", size = 266194, upload-time = "2025-11-03T21:31:51.53Z" }, - { url = "https://files.pythonhosted.org/packages/1a/67/3b92df89f179d7c367be654ab5626ae311cb28f7d5c237b6bb976cd5fbbb/regex-2025.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", size = 277069, upload-time = "2025-11-03T21:31:53.151Z" }, - { url = "https://files.pythonhosted.org/packages/d7/55/85ba4c066fe5094d35b249c3ce8df0ba623cfd35afb22d6764f23a52a1c5/regex-2025.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", size = 270330, upload-time = "2025-11-03T21:31:54.514Z" }, + { url = "https://files.pythonhosted.org/packages/92/81/10d8cf43c807d0326efe874c1b79f22bfb0fb226027b0b19ebc26d301408/regex-2026.1.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4c8fcc5793dde01641a35905d6731ee1548f02b956815f8f1cab89e515a5bdf1", size = 489398, upload-time = "2026-01-14T23:14:43.741Z" }, + { url = "https://files.pythonhosted.org/packages/90/b0/7c2a74e74ef2a7c32de724658a69a862880e3e4155cba992ba04d1c70400/regex-2026.1.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bfd876041a956e6a90ad7cdb3f6a630c07d491280bfeed4544053cd434901681", size = 291339, upload-time = "2026-01-14T23:14:45.183Z" }, + { url = "https://files.pythonhosted.org/packages/19/4d/16d0773d0c818417f4cc20aa0da90064b966d22cd62a8c46765b5bd2d643/regex-2026.1.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9250d087bc92b7d4899ccd5539a1b2334e44eee85d848c4c1aef8e221d3f8c8f", size = 289003, upload-time = "2026-01-14T23:14:47.25Z" }, + { url = "https://files.pythonhosted.org/packages/c6/e4/1fc4599450c9f0863d9406e944592d968b8d6dfd0d552a7d569e43bceada/regex-2026.1.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8a154cf6537ebbc110e24dabe53095e714245c272da9c1be05734bdad4a61aa", size = 798656, upload-time = "2026-01-14T23:14:48.77Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e6/59650d73a73fa8a60b3a590545bfcf1172b4384a7df2e7fe7b9aab4e2da9/regex-2026.1.15-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8050ba2e3ea1d8731a549e83c18d2f0999fbc99a5f6bd06b4c91449f55291804", size = 864252, upload-time = "2026-01-14T23:14:50.528Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ab/1d0f4d50a1638849a97d731364c9a80fa304fec46325e48330c170ee8e80/regex-2026.1.15-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf065240704cb8951cc04972cf107063917022511273e0969bdb34fc173456c", size = 912268, upload-time = "2026-01-14T23:14:52.952Z" }, + { url = "https://files.pythonhosted.org/packages/dd/df/0d722c030c82faa1d331d1921ee268a4e8fb55ca8b9042c9341c352f17fa/regex-2026.1.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c32bef3e7aeee75746748643667668ef941d28b003bfc89994ecf09a10f7a1b5", size = 803589, upload-time = "2026-01-14T23:14:55.182Z" }, + { url = "https://files.pythonhosted.org/packages/66/23/33289beba7ccb8b805c6610a8913d0131f834928afc555b241caabd422a9/regex-2026.1.15-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5eaa4a4c5b1906bd0d2508d68927f15b81821f85092e06f1a34a4254b0e1af3", size = 775700, upload-time = "2026-01-14T23:14:56.707Z" }, + { url = "https://files.pythonhosted.org/packages/e7/65/bf3a42fa6897a0d3afa81acb25c42f4b71c274f698ceabd75523259f6688/regex-2026.1.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:86c1077a3cc60d453d4084d5b9649065f3bf1184e22992bd322e1f081d3117fb", size = 787928, upload-time = "2026-01-14T23:14:58.312Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f5/13bf65864fc314f68cdd6d8ca94adcab064d4d39dbd0b10fef29a9da48fc/regex-2026.1.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:2b091aefc05c78d286657cd4db95f2e6313375ff65dcf085e42e4c04d9c8d410", size = 858607, upload-time = "2026-01-14T23:15:00.657Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/040e589834d7a439ee43fb0e1e902bc81bd58a5ba81acffe586bb3321d35/regex-2026.1.15-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:57e7d17f59f9ebfa9667e6e5a1c0127b96b87cb9cede8335482451ed00788ba4", size = 763729, upload-time = "2026-01-14T23:15:02.248Z" }, + { url = "https://files.pythonhosted.org/packages/9b/84/6921e8129687a427edf25a34a5594b588b6d88f491320b9de5b6339a4fcb/regex-2026.1.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:c6c4dcdfff2c08509faa15d36ba7e5ef5fcfab25f1e8f85a0c8f45bc3a30725d", size = 850697, upload-time = "2026-01-14T23:15:03.878Z" }, + { url = "https://files.pythonhosted.org/packages/8a/87/3d06143d4b128f4229158f2de5de6c8f2485170c7221e61bf381313314b2/regex-2026.1.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cf8ff04c642716a7f2048713ddc6278c5fd41faa3b9cab12607c7abecd012c22", size = 789849, upload-time = "2026-01-14T23:15:06.102Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/c50a63842b6bd48850ebc7ab22d46e7a2a32d824ad6c605b218441814639/regex-2026.1.15-cp312-cp312-win32.whl", hash = "sha256:82345326b1d8d56afbe41d881fdf62f1926d7264b2fc1537f99ae5da9aad7913", size = 266279, upload-time = "2026-01-14T23:15:07.678Z" }, + { url = "https://files.pythonhosted.org/packages/f2/36/39d0b29d087e2b11fd8191e15e81cce1b635fcc845297c67f11d0d19274d/regex-2026.1.15-cp312-cp312-win_amd64.whl", hash = "sha256:4def140aa6156bc64ee9912383d4038f3fdd18fee03a6f222abd4de6357ce42a", size = 277166, upload-time = "2026-01-14T23:15:09.257Z" }, + { url = "https://files.pythonhosted.org/packages/28/32/5b8e476a12262748851fa8ab1b0be540360692325975b094e594dfebbb52/regex-2026.1.15-cp312-cp312-win_arm64.whl", hash = "sha256:c6c565d9a6e1a8d783c1948937ffc377dd5771e83bd56de8317c450a954d2056", size = 270415, upload-time = "2026-01-14T23:15:10.743Z" }, ] [[package]] @@ -622,14 +612,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.14.0" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } +sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, + { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" }, ] [[package]] @@ -726,51 +716,51 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.2" +version = "2.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/24/a2a2ed9addd907787d7aa0355ba36a6cadf1768b934c652ea78acbd59dcd/urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797", size = 432930, upload-time = "2025-12-11T15:56:40.252Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/b9/4095b668ea3678bf6a0af005527f39de12fb026516fb3df17495a733b7f8/urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd", size = 131182, upload-time = "2025-12-11T15:56:38.584Z" }, + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] [[package]] name = "uv" -version = "0.9.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/03/1afff9e6362dc9d3a9e03743da0a4b4c7a0809f859c79eb52bbae31ea582/uv-0.9.18.tar.gz", hash = "sha256:17b5502f7689c4dc1fdeee9d8437a9a6664dcaa8476e70046b5f4753559533f5", size = 3824466, upload-time = "2025-12-16T15:45:11.81Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/9c/92fad10fcee8ea170b66442d95fd2af308fe9a107909ded4b3cc384fdc69/uv-0.9.18-py3-none-linux_armv6l.whl", hash = "sha256:e9e4915bb280c1f79b9a1c16021e79f61ed7c6382856ceaa99d53258cb0b4951", size = 21345538, upload-time = "2025-12-16T15:45:13.992Z" }, - { url = "https://files.pythonhosted.org/packages/81/b1/b0e5808e05acb54aa118c625d9f7b117df614703b0cbb89d419d03d117f3/uv-0.9.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d91abfd2649987996e3778729140c305ef0f6ff5909f55aac35c3c372544a24f", size = 20439572, upload-time = "2025-12-16T15:45:26.397Z" }, - { url = "https://files.pythonhosted.org/packages/b7/0b/9487d83adf5b7fd1e20ced33f78adf84cb18239c3d7e91f224cedba46c08/uv-0.9.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cf33f4146fd97e94cdebe6afc5122208eea8c55b65ca4127f5a5643c9717c8b8", size = 18952907, upload-time = "2025-12-16T15:44:48.399Z" }, - { url = "https://files.pythonhosted.org/packages/58/92/c8f7ae8900eff8e4ce1f7826d2e1e2ad5a95a5f141abdb539865aff79930/uv-0.9.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:edf965e9a5c55f74020ac82285eb0dfe7fac4f325ad0a7afc816290269ecfec1", size = 20772495, upload-time = "2025-12-16T15:45:29.614Z" }, - { url = "https://files.pythonhosted.org/packages/5a/28/9831500317c1dd6cde5099e3eb3b22b88ac75e47df7b502f6aef4df5750e/uv-0.9.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae10a941bd7ca1ee69edbe3998c34dce0a9fc2d2406d98198343daf7d2078493", size = 20949623, upload-time = "2025-12-16T15:44:57.482Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ff/1fe1ffa69c8910e54dd11f01fb0765d4fd537ceaeb0c05fa584b6b635b82/uv-0.9.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1669a95b588f613b13dd10e08ced6d5bcd79169bba29a2240eee87532648790", size = 21920580, upload-time = "2025-12-16T15:44:39.009Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ee/eed3ec7679ee80e16316cfc95ed28ef6851700bcc66edacfc583cbd2cc47/uv-0.9.18-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:11e1e406590d3159138288203a41ff8a8904600b8628a57462f04ff87d62c477", size = 23491234, upload-time = "2025-12-16T15:45:32.59Z" }, - { url = "https://files.pythonhosted.org/packages/78/58/64b15df743c79ad03ea7fbcbd27b146ba16a116c57f557425dd4e44d6684/uv-0.9.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e82078d3c622cb4c60da87f156168ffa78b9911136db7ffeb8e5b0a040bf30e", size = 23095438, upload-time = "2025-12-16T15:45:17.916Z" }, - { url = "https://files.pythonhosted.org/packages/43/6d/3d3dae71796961603c3871699e10d6b9de2e65a3c327b58d4750610a5f93/uv-0.9.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704abaf6e76b4d293fc1f24bef2c289021f1df0de9ed351f476cbbf67a7edae0", size = 22140992, upload-time = "2025-12-16T15:44:45.527Z" }, - { url = "https://files.pythonhosted.org/packages/31/91/1042d0966a30e937df500daed63e1f61018714406ce4023c8a6e6d2dcf7c/uv-0.9.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3332188fd8d96a68e5001409a52156dced910bf1bc41ec3066534cffcd46eb68", size = 22229626, upload-time = "2025-12-16T15:45:20.712Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1f/0a4a979bb2bf6e1292cc57882955bf1d7757cad40b1862d524c59c2a2ad8/uv-0.9.18-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:b7295e6d505f1fd61c54b1219e3b18e11907396333a9fa61cefe489c08fc7995", size = 20896524, upload-time = "2025-12-16T15:45:06.799Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3c/24f92e56af00cac7d9bed2888d99a580f8093c8745395ccf6213bfccf20b/uv-0.9.18-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:62ea0e518dd4ab76e6f06c0f43a25898a6342a3ecf996c12f27f08eb801ef7f1", size = 22077340, upload-time = "2025-12-16T15:44:51.271Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3e/73163116f748800e676bf30cee838448e74ac4cc2f716c750e1705bc3fe4/uv-0.9.18-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:8bd073e30030211ba01206caa57b4d63714e1adee2c76a1678987dd52f72d44d", size = 20932956, upload-time = "2025-12-16T15:45:00.3Z" }, - { url = "https://files.pythonhosted.org/packages/59/1b/a26990b51a17de1ffe41fbf2e30de3a98f0e0bce40cc60829fb9d9ed1a8a/uv-0.9.18-py3-none-musllinux_1_1_i686.whl", hash = "sha256:f248e013d10e1fc7a41f94310628b4a8130886b6d683c7c85c42b5b36d1bcd02", size = 21357247, upload-time = "2025-12-16T15:45:23.575Z" }, - { url = "https://files.pythonhosted.org/packages/5f/20/b6ba14fdd671e9237b22060d7422aba4a34503e3e42d914dbf925eff19aa/uv-0.9.18-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:17bedf2b0791e87d889e1c7f125bd5de77e4b7579aec372fa06ba832e07c957e", size = 22443585, upload-time = "2025-12-16T15:44:42.213Z" }, - { url = "https://files.pythonhosted.org/packages/5e/da/1b3dd596964f90a122cfe94dcf5b6b89cf5670eb84434b8c23864382576f/uv-0.9.18-py3-none-win32.whl", hash = "sha256:de6f0bb3e9c18e484545bd1549ec3c956968a141a393d42e2efb25281cb62787", size = 20091088, upload-time = "2025-12-16T15:45:03.225Z" }, - { url = "https://files.pythonhosted.org/packages/11/0b/50e13ebc1eedb36d88524b7740f78351be33213073e3faf81ac8925d0c6e/uv-0.9.18-py3-none-win_amd64.whl", hash = "sha256:c82b0e2e36b33e2146fba5f0ae6906b9679b3b5fe6a712e5d624e45e441e58e9", size = 22181193, upload-time = "2025-12-16T15:44:54.394Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d4/0bf338d863a3d9e5545e268d77a8e6afdd75d26bffc939603042f2e739f9/uv-0.9.18-py3-none-win_arm64.whl", hash = "sha256:4c4ce0ed080440bbda2377488575d426867f94f5922323af6d4728a1cd4d091d", size = 20564933, upload-time = "2025-12-16T15:45:09.819Z" }, +version = "0.9.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/6a/ef4ea19097ecdfd7df6e608f93874536af045c68fd70aa628c667815c458/uv-0.9.26.tar.gz", hash = "sha256:8b7017a01cc48847a7ae26733383a2456dd060fc50d21d58de5ee14f6b6984d7", size = 3790483, upload-time = "2026-01-15T20:51:33.582Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/e1/5c0b17833d5e3b51a897957348ff8d937a3cdfc5eea5c4a7075d8d7b9870/uv-0.9.26-py3-none-linux_armv6l.whl", hash = "sha256:7dba609e32b7bd13ef81788d580970c6ff3a8874d942755b442cffa8f25dba57", size = 22638031, upload-time = "2026-01-15T20:51:44.187Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8b/68ac5825a615a8697e324f52ac0b92feb47a0ec36a63759c5f2931f0c3a0/uv-0.9.26-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b815e3b26eeed00e00f831343daba7a9d99c1506883c189453bb4d215f54faac", size = 21507805, upload-time = "2026-01-15T20:50:42.574Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a2/664a338aefe009f6e38e47455ee2f64a21da7ad431dbcaf8b45d8b1a2b7a/uv-0.9.26-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1b012e6c4dfe767f818cbb6f47d02c207c9b0c82fee69a5de6d26ffb26a3ef3c", size = 20249791, upload-time = "2026-01-15T20:50:49.835Z" }, + { url = "https://files.pythonhosted.org/packages/ba/3d/b8186a7dec1346ca4630c674b760517d28bffa813a01965f4b57596bacf3/uv-0.9.26-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:ea296b700d7c4c27acdfd23ffaef2b0ecdd0aa1b58d942c62ee87df3b30f06ac", size = 22039108, upload-time = "2026-01-15T20:51:00.675Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a9/687fd587e7a3c2c826afe72214fb24b7f07b0d8b0b0300e6a53b554180ea/uv-0.9.26-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:1ba860d2988efc27e9c19f8537a2f9fa499a8b7ebe4afbe2d3d323d72f9aee61", size = 22174763, upload-time = "2026-01-15T20:50:46.471Z" }, + { url = "https://files.pythonhosted.org/packages/38/69/7fa03ee7d59e562fca1426436f15a8c107447d41b34e0899e25ee69abfad/uv-0.9.26-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8610bdfc282a681a0a40b90495a478599aa3484c12503ef79ef42cd271fd80fe", size = 22189861, upload-time = "2026-01-15T20:51:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/10/2d/4be446a2ec09f3c428632b00a138750af47c76b0b9f987e9a5b52fef0405/uv-0.9.26-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4bf700bd071bd595084b9ee0a8d77c6a0a10ca3773d3771346a2599f306bd9c", size = 23005589, upload-time = "2026-01-15T20:50:57.185Z" }, + { url = "https://files.pythonhosted.org/packages/c3/16/860990b812136695a63a8da9fb5f819c3cf18ea37dcf5852e0e1b795ca0d/uv-0.9.26-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:89a7beea1c692f76a6f8da13beff3cbb43f7123609e48e03517cc0db5c5de87c", size = 24713505, upload-time = "2026-01-15T20:51:04.366Z" }, + { url = "https://files.pythonhosted.org/packages/01/43/5d7f360d551e62d8f8bf6624b8fca9895cea49ebe5fce8891232d7ed2321/uv-0.9.26-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:182f5c086c7d03ad447e522b70fa29a0302a70bcfefad4b8cd08496828a0e179", size = 24342500, upload-time = "2026-01-15T20:51:47.863Z" }, + { url = "https://files.pythonhosted.org/packages/9b/9c/2bae010a189e7d8e5dc555edcfd053b11ce96fad2301b919ba0d9dd23659/uv-0.9.26-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d8c62a501f13425b4b0ce1dd4c6b82f3ce5a5179e2549c55f4bb27cc0eb8ef8", size = 23222578, upload-time = "2026-01-15T20:51:36.85Z" }, + { url = "https://files.pythonhosted.org/packages/38/16/a07593a040fe6403c36f3b0a99b309f295cbfe19a1074dbadb671d5d4ef7/uv-0.9.26-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7e89798bd3df7dcc4b2b4ac4e2fc11d6b3ff4fe7d764aa3012d664c635e2922", size = 23250201, upload-time = "2026-01-15T20:51:19.117Z" }, + { url = "https://files.pythonhosted.org/packages/23/a0/45893e15ad3ab842db27c1eb3b8605b9b4023baa5d414e67cfa559a0bff0/uv-0.9.26-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:60a66f1783ec4efc87b7e1f9bd66e8fd2de3e3b30d122b31cb1487f63a3ea8b7", size = 22229160, upload-time = "2026-01-15T20:51:22.931Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c0/20a597a5c253702a223b5e745cf8c16cd5dd053080f896bb10717b3bedec/uv-0.9.26-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:63c6a1f1187facba1fb45a2fa45396980631a3427ac11b0e3d9aa3ebcf2c73cf", size = 23090730, upload-time = "2026-01-15T20:51:26.611Z" }, + { url = "https://files.pythonhosted.org/packages/40/c9/744537867d9ab593fea108638b57cca1165a0889cfd989981c942b6de9a5/uv-0.9.26-py3-none-musllinux_1_1_i686.whl", hash = "sha256:c6d8650fbc980ccb348b168266143a9bd4deebc86437537caaf8ff2a39b6ea50", size = 22436632, upload-time = "2026-01-15T20:51:12.045Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e2/be683e30262f2cf02dcb41b6c32910a6939517d50ec45f502614d239feb7/uv-0.9.26-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:25278f9298aa4dade38241a93d036739b0c87278dcfad1ec1f57e803536bfc49", size = 23480064, upload-time = "2026-01-15T20:50:53.333Z" }, + { url = "https://files.pythonhosted.org/packages/50/3e/4a7e6bc5db2beac9c4966f212805f1903d37d233f2e160737f0b24780ada/uv-0.9.26-py3-none-win32.whl", hash = "sha256:10d075e0193e3a0e6c54f830731c4cb965d6f4e11956e84a7bed7ed61d42aa27", size = 21000052, upload-time = "2026-01-15T20:51:40.753Z" }, + { url = "https://files.pythonhosted.org/packages/07/5d/eb80c6eff2a9f7d5cf35ec84fda323b74aa0054145db28baf72d35a7a301/uv-0.9.26-py3-none-win_amd64.whl", hash = "sha256:0315fc321f5644b12118f9928086513363ed9b29d74d99f1539fda1b6b5478ab", size = 23684930, upload-time = "2026-01-15T20:51:08.448Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9d/3b2631931649b1783f5024796ca8ad2b42a01a829b9ce1202d973cc7bce5/uv-0.9.26-py3-none-win_arm64.whl", hash = "sha256:344ff38749b6cd7b7dfdfb382536f168cafe917ae3a5aa78b7a63746ba2a905b", size = 22158123, upload-time = "2026-01-15T20:51:30.939Z" }, ] [[package]] name = "virtualenv" -version = "20.35.4" +version = "20.36.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/28/e6f1a6f655d620846bd9df527390ecc26b3805a0c5989048c210e22c5ca9/virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c", size = 6028799, upload-time = "2025-10-29T06:57:40.511Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/a3/4d310fa5f00863544e1d0f4de93bddec248499ccf97d4791bc3122c9d4f3/virtualenv-20.36.1.tar.gz", hash = "sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba", size = 6032239, upload-time = "2026-01-09T18:21:01.296Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] [[package]]