diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..67218a8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,48 @@ +# Copyright The Linux Foundation and each contributor to LFX. +# SPDX-License-Identifier: MIT + +.git +.gitignore +.dockerignore +/bin/ +*.sh +*.pem +.env +*.env + +# Helm chart dependencies +/charts/*/charts/ +*.tgz + +# Local and temporary files +.DS_Store +.idea/ +.vscode/ +*.swp +*~ + +# Rendered templates +**/templates/*.rendered +**/templates/*.generated.yaml +**/templates/_*.tpl.rendered + +# MegaLinter reports +/megalinter-reports/ + +# Python environment for meltano +.venv/ +__pycache__/ +.mypy_cache/ +.ruff_cache/ + +# Documentation +README.md +*.md +LICENSE* + +# Test files +*_test.go +testdata/ + +# Development files +Makefile diff --git a/.github/workflows/ko-build-main.yaml b/.github/workflows/ko-build-main.yaml index 44cced1..8d3ac1e 100644 --- a/.github/workflows/ko-build-main.yaml +++ b/.github/workflows/ko-build-main.yaml @@ -23,13 +23,12 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: - go-version-file: v1-sync-helper/go.mod + go-version-file: go.mod - uses: ko-build/setup-ko@d006021bd0c28d1ce33a07e7943d48b079944c8d # v0.9 with: version: v0.18.0 - - working-directory: ./v1-sync-helper - run: | - ko build github.com/linuxfoundation/lfx-v1-sync-helper \ + - run: | + ko build github.com/linuxfoundation/lfx-v1-sync-helper/cmd/lfx-v1-sync-helper \ -B \ --platform linux/amd64,linux/arm64 \ -t ${{ github.sha }} \ diff --git a/.github/workflows/ko-build-tag.yaml b/.github/workflows/ko-build-tag.yaml index 5dad677..e2b0620 100644 --- a/.github/workflows/ko-build-tag.yaml +++ b/.github/workflows/ko-build-tag.yaml @@ -45,7 +45,7 @@ jobs: - name: Setup Go uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: - go-version-file: v1-sync-helper/go.mod + go-version-file: go.mod - name: Setup Ko uses: ko-build/setup-ko@d006021bd0c28d1ce33a07e7943d48b079944c8d # v0.9 @@ -53,9 +53,8 @@ jobs: version: v0.18.0 - name: Build and publish container image - working-directory: ./v1-sync-helper run: | - ko build github.com/linuxfoundation/lfx-v1-sync-helper \ + ko build github.com/linuxfoundation/lfx-v1-sync-helper/cmd/lfx-v1-sync-helper \ -B \ --platform linux/amd64,linux/arm64 \ -t ${{ github.ref_name }} \ diff --git a/.gitignore b/.gitignore index 9e5241c..6081c3c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ *~ .env *.env +*.pem # Rendered templates **/templates/*.rendered @@ -27,3 +28,6 @@ __pycache__/ .mypy_cache/ .ruff_cache/ + +# Go build artifacts +/bin/ diff --git a/.yamllint.yml b/.yamllint.yml index 1155737..41486ec 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -8,9 +8,11 @@ ignore: | styles venv .venv - charts - meltano + charts/lfx-v1-sync-helper/templates rules: line-length: max: 120 level: warning + indentation: + # Meltano doesn't indent sequences, but we otherwise do. + indent-sequences: whatever diff --git a/v1-sync-helper/Makefile b/Makefile similarity index 83% rename from v1-sync-helper/Makefile rename to Makefile index b8617cd..6ca95bd 100644 --- a/v1-sync-helper/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ # SPDX-License-Identifier: MIT # Binary name -BINARY_NAME=v1-sync-helper +BINARY_NAME=lfx-v1-sync-helper BINARY_PATH=bin/$(BINARY_NAME) +CMD_PATH=cmd/lfx-v1-sync-helper # Go parameters GOCMD=go @@ -28,13 +29,13 @@ all: clean deps fmt lint test build build: @echo "Building $(BINARY_NAME)..." @mkdir -p bin - $(GOBUILD) $(BUILD_FLAGS) -o $(BINARY_PATH) . + $(GOBUILD) $(BUILD_FLAGS) -o $(BINARY_PATH) ./$(CMD_PATH) # Build with debug symbols and race detection debug: @echo "Building $(BINARY_NAME) with debug symbols..." @mkdir -p bin - $(GOBUILD) $(DEBUG_FLAGS) -o $(BINARY_PATH) . + $(GOBUILD) $(DEBUG_FLAGS) -o $(BINARY_PATH) ./$(CMD_PATH) # Clean build artifacts clean: @@ -101,20 +102,17 @@ run-debug: debug # Build Docker image docker-build: @echo "Building Docker image..." - docker build -t ghcr.io/linuxfoundation/lfx-v1-sync-helper/$(BINARY_NAME):latest . + docker build -f docker/Dockerfile.v1-sync-helper -t ghcr.io/linuxfoundation/lfx-v1-sync-helper/$(BINARY_NAME):latest . # Run Docker container docker-run: docker-build @echo "Running Docker container..." + @if [ ! -f .env ]; then \ + echo "Error: .env file not found. Please create one from cmd/lfx-v1-sync-helper/config.example.env"; \ + exit 1; \ + fi docker run --rm -p 8080:8080 \ - -e NATS_URL=nats://localhost:4222 \ - -e PROJECT_SERVICE_URL=http://localhost:8080 \ - -e HEIMDALL_CLIENT_ID=v1_sync_helper \ - -e HEIMDALL_PRIVATE_KEY="$(shell cat /path/to/heimdall-private-key.pem)" \ - -e AUTH0_TENANT=linuxfoundation-dev \ - -e AUTH0_CLIENT_ID=your-auth0-client-id \ - -e AUTH0_PRIVATE_KEY="$(shell cat /path/to/auth0-private-key.pem)" \ - -e LFX_API_GW=https://api-gw.dev.platform.linuxfoundation.org/ \ + --env-file .env \ ghcr.io/linuxfoundation/lfx-v1-sync-helper/$(BINARY_NAME):latest # Update dependencies @@ -141,6 +139,6 @@ help: @echo " run - Build and run the application" @echo " run-debug - Build with debug and run the application" @echo " docker-build - Build Docker image" - @echo " docker-run - Build and run Docker container" + @echo " docker-run - Build and run Docker container (requires .env file)" @echo " update-deps - Update all dependencies" @echo " help - Show this help message" diff --git a/README.md b/README.md index c821529..fd36187 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ This repository contains three main components: ### [Meltano](./meltano/README.md) Data extraction and loading pipeline that extracts data from LFX v1 sources (DynamoDB for meetings, PostgreSQL for projects/committees) and loads it into NATS KV stores for processing by the v2 platform. -### [v1-sync-helper](./v1-sync-helper/README.md) +### [v1-sync-helper](./cmd/lfx-v1-sync-helper/README.md) Go service that monitors NATS KV stores for replicated v1 data and synchronizes it with the LFX v2 platform APIs, handling data transformation and conflict resolution. ### [Helm charts](./charts/lfx-v1-sync-helper/README.md) diff --git a/charts/lfx-v1-sync-helper/README.md b/charts/lfx-v1-sync-helper/README.md index d0958c4..f5860b0 100644 --- a/charts/lfx-v1-sync-helper/README.md +++ b/charts/lfx-v1-sync-helper/README.md @@ -111,7 +111,7 @@ The following environment variables have defaults configured in the chart's `app | `PORT` | `8080` | HTTP server port | | `BIND` | `*` | Interface to bind on | -For a complete list of all supported environment variables, including required ones like `AUTH0_TENANT`, see the [v1-sync-helper README](../../v1-sync-helper/README.md#environment-variables). +For a complete list of all supported environment variables, including required ones like `AUTH0_TENANT`, see the [v1-sync-helper README](../../cmd/lfx-v1-sync-helper/README.md#environment-variables). ### Additional Configuration diff --git a/charts/lfx-v1-sync-helper/values.yaml b/charts/lfx-v1-sync-helper/values.yaml index e21ad56..74b8026 100644 --- a/charts/lfx-v1-sync-helper/values.yaml +++ b/charts/lfx-v1-sync-helper/values.yaml @@ -73,9 +73,9 @@ nats: # storage is the storage type for the KV bucket storage: file # maxValueSize is the maximum size of a value in the KV bucket - maxValueSize: 10485760 # 10MB + maxValueSize: 10485760 # 10MB # maxBytes is the maximum number of bytes in the KV bucket - maxBytes: 10737418240 # 10GB + maxBytes: 10737418240 # 10GB # compression is a boolean to determine if the KV bucket should be compressed compression: true @@ -94,9 +94,9 @@ nats: # storage is the storage type for the KV bucket storage: file # maxValueSize is the maximum size of a value in the KV bucket - maxValueSize: 10485760 # 10MB + maxValueSize: 10485760 # 10MB # maxBytes is the maximum number of bytes in the KV bucket - maxBytes: 2147483648 # 2GB + maxBytes: 2147483648 # 2GB # compression is a boolean to determine if the KV bucket should be compressed compression: true diff --git a/v1-sync-helper/README.md b/cmd/lfx-v1-sync-helper/README.md similarity index 99% rename from v1-sync-helper/README.md rename to cmd/lfx-v1-sync-helper/README.md index f47a1ec..da86560 100644 --- a/v1-sync-helper/README.md +++ b/cmd/lfx-v1-sync-helper/README.md @@ -13,7 +13,7 @@ The LFX v1 Sync Helper is a Go microservice that synchronizes v1 data from NATS ### Data Flow -For a more detail view, see the root [README.md](../README.md) diagrams. +For a more detail view, see the root [README.md](../../README.md) diagrams. ```mermaid flowchart LR diff --git a/v1-sync-helper/client_committees.go b/cmd/lfx-v1-sync-helper/client_committees.go similarity index 99% rename from v1-sync-helper/client_committees.go rename to cmd/lfx-v1-sync-helper/client_committees.go index 8ce5065..aba0e44 100644 --- a/v1-sync-helper/client_committees.go +++ b/cmd/lfx-v1-sync-helper/client_committees.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Committee-specific client operations for the v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/client_projects.go b/cmd/lfx-v1-sync-helper/client_projects.go similarity index 99% rename from v1-sync-helper/client_projects.go rename to cmd/lfx-v1-sync-helper/client_projects.go index 7131d22..c1c2d4f 100644 --- a/v1-sync-helper/client_projects.go +++ b/cmd/lfx-v1-sync-helper/client_projects.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Project-specific client operations for the v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/config.go b/cmd/lfx-v1-sync-helper/config.go similarity index 99% rename from v1-sync-helper/config.go rename to cmd/lfx-v1-sync-helper/config.go index f612201..5f25c19 100644 --- a/v1-sync-helper/config.go +++ b/cmd/lfx-v1-sync-helper/config.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Configuration management for v1-sync-helper service +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/handlers.go b/cmd/lfx-v1-sync-helper/handlers.go similarity index 94% rename from v1-sync-helper/handlers.go rename to cmd/lfx-v1-sync-helper/handlers.go index be8af7b..7bf93ff 100644 --- a/v1-sync-helper/handlers.go +++ b/cmd/lfx-v1-sync-helper/handlers.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// The v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( @@ -95,6 +95,10 @@ func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) { handleZoomPastMeetingSummaryUpdate(ctx, key, v1Data) case "itx-zoom-past-meetings": handleZoomPastMeetingUpdate(ctx, key, v1Data) + case "salesforce-merged_user": + logger.With("key", key).DebugContext(ctx, "salesforce-merged_user sync not yet implemented") + case "salesforce-alternate_email__c": + handleAlternateEmailUpdate(ctx, key, v1Data) default: logger.With("key", key).WarnContext(ctx, "unknown object type, ignoring") } diff --git a/v1-sync-helper/handlers_committees.go b/cmd/lfx-v1-sync-helper/handlers_committees.go similarity index 99% rename from v1-sync-helper/handlers_committees.go rename to cmd/lfx-v1-sync-helper/handlers_committees.go index 4f3be76..9b5ea82 100644 --- a/v1-sync-helper/handlers_committees.go +++ b/cmd/lfx-v1-sync-helper/handlers_committees.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Committee-specific handlers for the v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/handlers_meetings.go b/cmd/lfx-v1-sync-helper/handlers_meetings.go similarity index 99% rename from v1-sync-helper/handlers_meetings.go rename to cmd/lfx-v1-sync-helper/handlers_meetings.go index ae1fe2f..79e2e60 100644 --- a/v1-sync-helper/handlers_meetings.go +++ b/cmd/lfx-v1-sync-helper/handlers_meetings.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Package main provides HTTP handlers for meeting-related operations. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/handlers_projects.go b/cmd/lfx-v1-sync-helper/handlers_projects.go similarity index 99% rename from v1-sync-helper/handlers_projects.go rename to cmd/lfx-v1-sync-helper/handlers_projects.go index 4b13c47..3f892e5 100644 --- a/v1-sync-helper/handlers_projects.go +++ b/cmd/lfx-v1-sync-helper/handlers_projects.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Project-specific handlers for the v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/cmd/lfx-v1-sync-helper/handlers_users.go b/cmd/lfx-v1-sync-helper/handlers_users.go new file mode 100644 index 0000000..0ebb2c0 --- /dev/null +++ b/cmd/lfx-v1-sync-helper/handlers_users.go @@ -0,0 +1,164 @@ +// Copyright The Linux Foundation and each contributor to LFX. +// SPDX-License-Identifier: MIT + +// The lfx-v1-sync-helper service. +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/rand/v2" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// handleAlternateEmailUpdate processes alternate email updates and maintains +// v1-mapping records for merged users' alternate emails. +func handleAlternateEmailUpdate(ctx context.Context, key string, v1Data map[string]any) { + // Extract the leadorcontactid which references the sfid of merged_user table. + leadorcontactid, ok := v1Data["leadorcontactid"].(string) + if !ok || leadorcontactid == "" { + logger.With("key", key).WarnContext(ctx, "alternate email missing leadorcontactid, skipping") + return + } + + // Extract the sfid of this alternate email record. + emailSfid, ok := v1Data["sfid"].(string) + if !ok || emailSfid == "" { + logger.With("key", key).WarnContext(ctx, "alternate email missing sfid, skipping") + return + } + + // Check if this email is deleted. + isDeleted := false + if deletedVal, ok := v1Data["isdeleted"].(bool); ok { + isDeleted = deletedVal + } + + // Process the update in a goroutine to avoid blocking other handlers. + go func() { + updateUserAlternateEmails(context.WithoutCancel(ctx), leadorcontactid, emailSfid, isDeleted) + }() +} + +// updateUserAlternateEmails updates the v1-mapping record for a user's alternate emails +// with concurrency control using atomic KV operations. +func updateUserAlternateEmails(ctx context.Context, userSfid, emailSfid string, isDeleted bool) { + mappingKey := fmt.Sprintf("v1-merged-user.alternate-emails.%s", userSfid) + maxRetries := 5 + + for attempt := 1; attempt <= maxRetries; attempt++ { + // Add random splay time up to 1 second to reduce collision chances. + splayTime := time.Duration(rand.IntN(1000)) * time.Millisecond + time.Sleep(splayTime) + + // Get current mapping record. + entry, err := mappingsKV.Get(ctx, mappingKey) + + var currentEmails []string + var revision uint64 + + if err != nil { + if err == jetstream.ErrKeyNotFound { + // Key doesn't exist, we'll create it. + currentEmails = []string{} + revision = 0 + } else { + logger.With("error", err, "key", mappingKey, "attempt", attempt). + ErrorContext(ctx, "failed to get mapping record") + if attempt == maxRetries { + return + } + continue + } + } else { + // Parse existing emails list. + revision = entry.Revision() + if err := json.Unmarshal(entry.Value(), ¤tEmails); err != nil { + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to unmarshal existing emails list") + return + } + } + + // Update the emails list. + updatedEmails := updateEmailsList(currentEmails, emailSfid, isDeleted) + + // Marshal the updated list. + updatedData, err := json.Marshal(updatedEmails) + if err != nil { + logger.With("error", err, "key", mappingKey). + ErrorContext(ctx, "failed to marshal updated emails list") + return + } + + // Attempt to save with concurrency control. + var saveErr error + if revision == 0 { + // Try to create new record. + _, saveErr = mappingsKV.Create(ctx, mappingKey, updatedData) + if saveErr == jetstream.ErrKeyExists { + // Key was created by another process, retry. + logger.With("key", mappingKey, "attempt", attempt). + DebugContext(ctx, "key created by another process during create attempt, retrying") + continue + } + } else { + // Try to update existing record. + _, saveErr = mappingsKV.Update(ctx, mappingKey, updatedData, revision) + if saveErr != nil { + // Update failed (likely revision mismatch), retry. + logger.With("error", saveErr, "key", mappingKey, "attempt", attempt). + DebugContext(ctx, "update failed, retrying") + continue + } + } + + if saveErr == nil { + // Success! + logger.With("key", mappingKey, "emailSfid", emailSfid, "isDeleted", isDeleted, "attempt", attempt). + DebugContext(ctx, "successfully updated alternate emails mapping") + return + } + + // If we get here, there was an unexpected error. + logger.With("error", saveErr, "key", mappingKey, "attempt", attempt). + ErrorContext(ctx, "unexpected error during save operation") + + if attempt == maxRetries { + logger.With("key", mappingKey, "maxRetries", maxRetries). + ErrorContext(ctx, "max retries exceeded for updating alternate emails mapping") + return + } + } +} + +// updateEmailsList adds or removes an email sfid from the list based on deletion status. +func updateEmailsList(currentEmails []string, emailSfid string, isDeleted bool) []string { + // Find if the email already exists in the list. + index := -1 + for i, email := range currentEmails { + if email == emailSfid { + index = i + break + } + } + + if isDeleted { + // Remove from list if it exists. + if index != -1 { + // Remove element at index. + return append(currentEmails[:index], currentEmails[index+1:]...) + } + // Email not in list, nothing to remove. + return currentEmails + } + // Add to list if it doesn't exist. + if index == -1 { + return append(currentEmails, emailSfid) + } + // Email already in list, nothing to add. + return currentEmails +} diff --git a/v1-sync-helper/helper_meetings.go b/cmd/lfx-v1-sync-helper/helper_meetings.go similarity index 99% rename from v1-sync-helper/helper_meetings.go rename to cmd/lfx-v1-sync-helper/helper_meetings.go index 7055e4e..c6127b9 100644 --- a/v1-sync-helper/helper_meetings.go +++ b/cmd/lfx-v1-sync-helper/helper_meetings.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// package main is the main package for the v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/lfx_v1_client.go b/cmd/lfx-v1-sync-helper/lfx_v1_client.go similarity index 99% rename from v1-sync-helper/lfx_v1_client.go rename to cmd/lfx-v1-sync-helper/lfx_v1_client.go index 5245699..35baef3 100644 --- a/v1-sync-helper/lfx_v1_client.go +++ b/cmd/lfx-v1-sync-helper/lfx_v1_client.go @@ -1,6 +1,9 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT +// The lfx-v1-sync-helper service. +package main + // Auth0 authentication and HTTP client for LFX v1 API Gateway calls // // This client handles: @@ -20,7 +23,6 @@ // - Machine users: platform IDs with "@clients" suffix (no API lookup) // - Platform users: regular platform IDs requiring v1 API lookup // - Invalid users: empty usernames or API errors (cached to prevent retries) -package main import ( "context" diff --git a/v1-sync-helper/lfx_v2_client.go b/cmd/lfx-v1-sync-helper/lfx_v2_client.go similarity index 99% rename from v1-sync-helper/lfx_v2_client.go rename to cmd/lfx-v1-sync-helper/lfx_v2_client.go index ba33d9d..582b32d 100644 --- a/v1-sync-helper/lfx_v2_client.go +++ b/cmd/lfx-v1-sync-helper/lfx_v2_client.go @@ -1,9 +1,11 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// JWT authentication and HTTP client for LFX v2 service calls with user impersonation +// The lfx-v1-sync-helper service. package main +// JWT authentication and HTTP client for LFX v2 service calls with user impersonation + import ( "context" "crypto/rsa" diff --git a/v1-sync-helper/main.go b/cmd/lfx-v1-sync-helper/main.go similarity index 99% rename from v1-sync-helper/main.go rename to cmd/lfx-v1-sync-helper/main.go index 2f49d06..a9f5c97 100644 --- a/v1-sync-helper/main.go +++ b/cmd/lfx-v1-sync-helper/main.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// The v1-sync-helper service. +// The lfx-v1-sync-helper service. package main import ( diff --git a/v1-sync-helper/models_meetings.go b/cmd/lfx-v1-sync-helper/models_meetings.go similarity index 99% rename from v1-sync-helper/models_meetings.go rename to cmd/lfx-v1-sync-helper/models_meetings.go index a2a32f2..dcfbb5c 100644 --- a/v1-sync-helper/models_meetings.go +++ b/cmd/lfx-v1-sync-helper/models_meetings.go @@ -1,7 +1,7 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT -// Package main provides data models and structures for meeting-related operations. +// The lfx-v1-sync-helper service. package main import ( @@ -909,7 +909,7 @@ type V2PastMeetingParticipant struct { // Participants can have multiple sessions if they join and leave multiple times type ParticipantSession struct { UID string `json:"uid"` - JoinTime time.Time `json:"join_time"` + JoinTime *time.Time `json:"join_time,omitempty"` LeaveTime *time.Time `json:"leave_time,omitempty"` LeaveReason string `json:"leave_reason,omitempty"` } diff --git a/v1-sync-helper/nats_client.go b/cmd/lfx-v1-sync-helper/nats_client.go similarity index 100% rename from v1-sync-helper/nats_client.go rename to cmd/lfx-v1-sync-helper/nats_client.go diff --git a/v1-sync-helper/username_mapping.go b/cmd/lfx-v1-sync-helper/username_mapping.go similarity index 94% rename from v1-sync-helper/username_mapping.go rename to cmd/lfx-v1-sync-helper/username_mapping.go index 84f68b1..4b3d191 100644 --- a/v1-sync-helper/username_mapping.go +++ b/cmd/lfx-v1-sync-helper/username_mapping.go @@ -1,11 +1,13 @@ // Copyright The Linux Foundation and each contributor to LFX. // SPDX-License-Identifier: MIT +// The lfx-v1-sync-helper service. +package main + // Username mapping utility for converting usernames to Auth0 "sub" format. // -// This module handles the conversion of usernames to the "sub" claim format +// This file handles the conversion of usernames to the "sub" claim format // expected by v2 services, which uses "auth0|{ldap exported safe ID}" format. -package main import ( "crypto/sha512" diff --git a/v1-sync-helper/Dockerfile b/docker/Dockerfile.v1-sync-helper similarity index 76% rename from v1-sync-helper/Dockerfile rename to docker/Dockerfile.v1-sync-helper index a5072a1..82d367b 100644 --- a/v1-sync-helper/Dockerfile +++ b/docker/Dockerfile.v1-sync-helper @@ -20,10 +20,10 @@ COPY go.mod go.sum ./ RUN go mod download # Copy the code into the container -COPY . . +COPY cmd/lfx-v1-sync-helper/ ./cmd/lfx-v1-sync-helper/ # Build the application -RUN go build -o /go/bin/v1-sync-helper -trimpath -ldflags="-w -s" . +RUN go build -o /go/bin/lfx-v1-sync-helper -trimpath -ldflags="-w -s" ./cmd/lfx-v1-sync-helper # Run our go binary standalone FROM cgr.dev/chainguard/static:latest @@ -31,6 +31,6 @@ FROM cgr.dev/chainguard/static:latest # Implicit with base image; setting explicitly for linters. USER nonroot -COPY --from=builder /go/bin/v1-sync-helper /v1-sync-helper +COPY --from=builder /go/bin/lfx-v1-sync-helper /lfx-v1-sync-helper -ENTRYPOINT ["/v1-sync-helper"] +ENTRYPOINT ["/lfx-v1-sync-helper"] diff --git a/v1-sync-helper/go.mod b/go.mod similarity index 100% rename from v1-sync-helper/go.mod rename to go.mod diff --git a/v1-sync-helper/go.sum b/go.sum similarity index 100% rename from v1-sync-helper/go.sum rename to go.sum diff --git a/meltano/meltano.yml b/meltano/meltano.yml index 71b253b..63957de 100644 --- a/meltano/meltano.yml +++ b/meltano/meltano.yml @@ -19,8 +19,39 @@ plugins: - platform-collaboration__c.* - platform-community__c.* - salesforce-project__c.* + - salesforce-alternate_email__c.* + - salesforce-merged_user.sfid + - salesforce-merged_user.isdeleted + - salesforce-merged_user.lastname + - salesforce-merged_user.firstname + - salesforce-merged_user.salutation + - salesforce-merged_user.middlename + - salesforce-merged_user.suffix + - salesforce-merged_user.name + - salesforce-merged_user.title + - salesforce-merged_user.photourl + - salesforce-merged_user.description + - salesforce-merged_user.createddate + - salesforce-merged_user.lastmodifieddate + - salesforce-merged_user.username__c + - salesforce-merged_user.facebook_id__c + - salesforce-merged_user.github_id__c + - salesforce-merged_user.linkedin_id__c + - salesforce-merged_user.twitter_id__c + - salesforce-merged_user.gitlab_id__c + - salesforce-merged_user.bio__c + - salesforce-merged_user.photo_url__c + - salesforce-merged_user.technology__c + - salesforce-merged_user.timezone__c + - salesforce-merged_user.company + - salesforce-merged_user.website + - salesforce-merged_user.preferred_language__c + - salesforce-merged_user.accountid + - salesforce-merged_user.sfid_b2b + - salesforce-merged_user.createdbyid + - salesforce-merged_user.lastmodifiedbyid metadata: - '*': + "*": replication-method: INCREMENTAL replication-key: lastmodifieddate - name: tap-dynamodb @@ -29,17 +60,17 @@ plugins: config: aws_default_region: us-west-2 tables: - - itx-zoom-meetings-mappings-v2 - - itx-zoom-meetings-v2 - - itx-zoom-past-meetings-mappings - - itx-zoom-past-meetings - - itx-zoom-past-meetings-attendees - - itx-zoom-past-meetings-invitees - - itx-zoom-past-meetings-recordings - - itx-zoom-past-meetings-summaries - - itx-zoom-meetings-registrants-v2 + - itx-zoom-meetings-mappings-v2 + - itx-zoom-meetings-v2 + - itx-zoom-past-meetings-mappings + - itx-zoom-past-meetings + - itx-zoom-past-meetings-attendees + - itx-zoom-past-meetings-invitees + - itx-zoom-past-meetings-recordings + - itx-zoom-past-meetings-summaries + - itx-zoom-meetings-registrants-v2 metadata: - '*': + "*": # Setting the replication key allows conditional loads for # target-nats-kv, even though Meltano's tap-dynamodb doesn't support # INCREMENTAL or LOG_BASED replication methods. @@ -81,7 +112,6 @@ plugins: settings: - name: url value: nats://lfx-platform-nats.lfx.svc.cluster.local:4222 - alias: host - name: bucket value: v1-objects - name: user diff --git a/v1-sync-helper/.dockerignore b/v1-sync-helper/.dockerignore deleted file mode 100644 index 7f8c14e..0000000 --- a/v1-sync-helper/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.git -/bin/ -*.sh diff --git a/v1-sync-helper/.gitignore b/v1-sync-helper/.gitignore deleted file mode 100644 index 0c515ea..0000000 --- a/v1-sync-helper/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright The Linux Foundation and each contributor to LFX. -# SPDX-License-Identifier: MIT - -/bin/ -*.pem