diff --git a/db/README_TESTING.md b/db/README_TESTING.md new file mode 100644 index 0000000..006c4c6 --- /dev/null +++ b/db/README_TESTING.md @@ -0,0 +1,98 @@ +# Database Testing Guide + +## Overview + +Unit tests for the database layer use an **in-memory SQLite database** for fast, isolated testing without requiring external dependencies. + +## Approach + +### In-Memory SQLite Database +- Tests create a fresh `:memory:` SQLite database for each test +- Schema is migrated using GORM's AutoMigrate +- No cleanup required - database is garbage collected after test completion + +### Test Structure + +1. **`setupTestDB(t)`** - Creates and migrates a clean in-memory database +2. **`seedTestData(t, db)`** - Populates the database with test fixtures +3. **Individual test functions** - Test specific functionality + +## Example: TestGetMaintainersByProject + +```go +func TestGetMaintainersByProject(t *testing.T) { + db := setupTestDB(t) + company, project1, project2, m1, m2, m3 := seedTestData(t, db) + store := NewSQLStore(db) + + // Test cases... +} +``` + +### Test Fixtures + +The `seedTestData` function creates: +- 1 company ("Test Company") +- 2 projects (kubernetes, prometheus) +- 3 maintainers (Alice, Bob, Charlie) +- Associations: + - kubernetes → Alice, Bob + - prometheus → Bob, Charlie + +### What Gets Tested + +1. ✅ Returns correct maintainers for a project +2. ✅ Company relationships are preloaded +3. ✅ Empty results for projects with no maintainers +4. ✅ Empty results for non-existent projects +5. ✅ All maintainer fields are populated correctly +6. ✅ Projects field is NOT preloaded (as expected) + +## Running Tests + +```bash +# Run all db tests +go test ./db + +# Run specific test +go test ./db -run TestGetMaintainersByProject + +# Run with verbose output +go test -v ./db + +# Run with coverage +go test -cover ./db +``` + +## Benefits of This Approach + +1. **Fast** - In-memory database is extremely fast +2. **Isolated** - Each test gets a fresh database +3. **No dependencies** - No need for Docker, external DB, or test infrastructure +4. **Deterministic** - Tests always start with the same state +5. **Parallel-safe** - Each test has its own database instance + +## Adding New Tests + +To add a new test: + +1. Use `setupTestDB(t)` to get a fresh database +2. Create your own fixtures or use `seedTestData(t, db)` if appropriate +3. Test your function +4. Assert expected behavior + +Example: +```go +func TestYourFunction(t *testing.T) { + db := setupTestDB(t) + // Create custom test data if needed + project := model.Project{Name: "test"} + require.NoError(t, db.Create(&project).Error) + + store := NewSQLStore(db) + result, err := store.YourFunction(project.ID) + + require.NoError(t, err) + assert.Equal(t, expectedValue, result) +} +``` diff --git a/deploy/README.MD b/deploy/README.MD new file mode 100644 index 0000000..7c504e3 --- /dev/null +++ b/deploy/README.MD @@ -0,0 +1,237 @@ +# Deployment notes + +MaintainerD - input data (List of Projects and associated Maintainers) + +## Kubernetes resources overview +```mermaid +graph RL + subgraph maintainerd["Namespace: maintainerd"] + subgraph pod["Pod"] + subgraph initContainerBootstrap["initContainer: bootstrap"] + bootstrap["Loads Project → Maintainer data
into sqlite3 database
from CNCF-Internal Google Sheet"] + end + subgraph maintainerdServer["maintainerd-server"] + server["
GitHub WebHooks
labels can start on-boarding processes
issue comments can progress workflows
/healthz establishes availability by maintainerd-service."] + end + end + + pvc[(PersistentVolumeClaim
sqlite3 maintainerd-db)] + svc{{Service
maintainerd-service
type=LoadBalancer}} + svc -->|"ports 80/443"| ext[(External LB IP
github-events.cncf.io/webhook)] + end + + bootstrap -->|"SQL INSERTs into PROJECT and MAINTAINERS tables"| pvc +``` + + + deploy -->|creates| pod + deploy -->|mounts| pvc + deploy -->|envFrom| secrets + deploy -->|imagePullSecrets| ghcr + pod -->|exposes 2525| svc + svc -->|TLS| tls + pod -->|volumeMount| pvc + + + + +## Maintainer Database ER diagram + +```mermaid +erDiagram +companies { +INTEGER id PK +DATETIME created_at +DATETIME updated_at +DATETIME deleted_at +TEXT name +} + + projects { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + TEXT name + INTEGER parent_project_id "FK to projects.id" + TEXT maturity + TEXT maintainer_ref + TEXT onboarding_issue + TEXT mailing_list + } + + services { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + TEXT name + TEXT description + } + + service_projects { + INTEGER project_id PK "FK to projects.id" + INTEGER service_id PK "FK to services.id" + } + + maintainers { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + TEXT name + TEXT email + TEXT git_hub_account + TEXT git_hub_email + TEXT maintainer_status + TEXT import_warnings + DATETIME registered_at + INTEGER company_id "FK to companies.id" + } + + maintainer_projects { + INTEGER maintainer_id PK "FK to maintainers.id" + INTEGER project_id PK "FK to projects.id" + DATETIME joined_at + } + + collaborators { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + TEXT name + TEXT email + TEXT git_hub_email + TEXT git_hub_account + DATETIME last_login + DATETIME registered_at + } + + service_teams { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + INTEGER project_id "FK to projects.id" + INTEGER service_id "FK to services.id" + TEXT service_team_name + TEXT project_name + } + + service_users { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + INTEGER service_id "FK to services.id" + INTEGER service_user_id + TEXT service_email + TEXT service_ref + TEXT service_git_hub_name + } + + service_user_teams { + INTEGER id PK + DATETIME created_at + DATETIME updated_at + DATETIME deleted_at + INTEGER service_id "FK to services.id" + INTEGER service_user_id "FK to service_users.id" + INTEGER service_team_id "FK to service_teams.id" + INTEGER maintainer_id "FK to maintainers.id" + INTEGER collaborator_id "FK to collaborators.id" + } + + companies ||--o{ maintainers : employs + projects ||--o{ projects : parent_of + projects ||--o{ service_projects : includes + services ||--o{ service_projects : provides + maintainers ||--o{ maintainer_projects : assigned_to + projects ||--o{ maintainer_projects : hosts + service_teams ||--o{ service_user_teams : has + services ||--o{ service_users : has +``` + + + +## init-container + - bootstrap process loads project and maintainer data from + - a CNCF-Internal worksheet + - in future, should be loaded from a combination of PCC user profiles (keyed by GitHub user account) and a registered list of project.yaml files on a per-project basis. + +## GitHub Event Listener + +### cert-manager +Manages the server cert associated with the maintainer-d event listener that listens for changes to onboarding issues. + +Steps to integrate with Let's Encrypt on OKE: + +1. Create the OCI DNS credentials secret: + ```bash + kubectl create secret generic oci-dns-credentials -n cert-manager \ + --from-literal=tenancyOCID= \ + --from-literal=userOCID= \ + --from-literal=fingerprint= \ + --from-file=privateKey= + ``` +2. Apply a `ClusterIssuer` using the OCI DNS solver (update compartment OCID, DNS zone, and email): + ```yaml + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: letsencrypt-dns + spec: + acme: + email: you@example.com + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-dns-account-key + solvers: + - dns01: + oci: + compartmentOCID: ocid1.compartment.oc1..aaaaaaaa22icap66vxktktubjlhf6oxvfhev6n7udgje2chahyrtq65ga63a + dnsZoneName: cncf.io. + secretRef: + name: oci-dns-credentials + key: privateKey + tenancyOCID: ocid1.tenancy.oc1... # must match the secret + userOCID: ocid1.user.oc1... # must match the secret + fingerprint: + ``` +3. Request the certificate in the `maintainerd` namespace: + ```yaml + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: maintainerd-cert + namespace: maintainerd + spec: + secretName: maintainerd-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - github-events.cncf.io + ``` +4. Update `deploy/manifests/service.yaml` to expose HTTPS and reference the secret: + ```yaml + metadata: + annotations: + service.beta.kubernetes.io/oci-load-balancer-ssl-ports: https + service.beta.kubernetes.io/oci-load-balancer-tls-secret: maintainerd/maintainerd-tls + spec: + ports: + - name: http + port: 80 + targetPort: 2525 + - name: https + port: 443 + targetPort: 2525 + ``` +5. Apply the manifests (`kubectl apply -f deploy/manifests/service.yaml`) and verify: + - `kubectl describe certificate maintainerd-cert -n maintainerd` + - `kubectl get secret maintainerd-tls -n maintainerd` + - `curl -vk https://github-events.cncf.io/healthz` + +Ensure your OCI IAM policies allow the OKE dynamic group to manage DNS records in the relevant compartment; otherwise the ACME solver cannot create the TXT challenges. diff --git a/deploy/cert-ops.md b/deploy/cert-ops.md new file mode 100644 index 0000000..21ce984 --- /dev/null +++ b/deploy/cert-ops.md @@ -0,0 +1,43 @@ +# Certificate Maintenance for maintainer-d + +maintainer-d uses a manually issued Let’s Encrypt certificate stored in the `maintainerd-tls` secret. + +Repeat these steps before the cert expires (every ~60–90 days): + +1. **Request a new certificate via certbot** + ```bash + sudo certbot certonly \ + --manual \ + --preferred-challenges dns \ + --key-type rsa \ + -d github-events.cncf.io \ + --email \ + --agree-tos + ``` + Certbot prints a `_acme-challenge.github-events.cncf.io` TXT record. Ask the DNSimple admin to create it (TTL 60). Press Enter once DNS propagates. + +2. **Load the cert into Kubernetes** + ```bash + kubectl create secret tls maintainerd-tls \ + --cert=/etc/letsencrypt/live/github-events.cncf.io/fullchain.pem \ + --key=/etc/letsencrypt/live/github-events.cncf.io/privkey.pem \ + -n maintainerd \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + +3. **Recreate the Service so OCI reloads the cert** + ```bash + kubectl delete svc maintainerd -n maintainerd + kubectl apply -f deploy/manifests/service.yaml + kubectl get svc maintainerd -n maintainerd --watch + ``` + Wait until `EXTERNAL-IP` shows `170.9.21.206` again. + +4. **Verify** + ```bash + kubectl describe svc maintainerd -n maintainerd + curl -vk https://github-events.cncf.io/healthz + openssl s_client -connect github-events.cncf.io:443 -servername github-events.cncf.io -tls1_2 + ``` + +Keep `/etc/letsencrypt` backed up or document the certbot host. If you ever automate DNS updates, you can replace the manual step with cert-manager and remove the monthly coordination with DNSimple. diff --git a/github-events-service.md b/github-events-service.md new file mode 100644 index 0000000..ea344f6 --- /dev/null +++ b/github-events-service.md @@ -0,0 +1,38 @@ +# GitHub Events Service Architecture + +```mermaid +flowchart LR + subgraph GitHub Cloud + GH[GitHub Webhooks] + end + + subgraph Internet + DNS[DNS: github-events.cncf.io -> Reserved IP] + end + + subgraph "Oracle Cloud (OCI)" + subgraph Cluster Network + LBService[Ingress Controller Service\n(type=LoadBalancer, loadBalancerIP=Reserved IP)] + Ingress[Ingress-NGINX Controller] + Service[maintainerd Service\n(ClusterIP)] + Pod[maintainerd Pod] + end + subgraph MetalLB System + Pool[IPAddress Pool\ncontains Reserved IP] + Speakers[Speaker Pods\nannounce IP via BGP/ARP] + end + end + + GH -->|HTTPS webhook| DNS --> LBService + LBService -->|requests IP| Pool + Speakers -->|advertise IP| Internet + LBService --> Ingress --> Service --> Pod +``` + +## Flow Summary +- GitHub delivers webhook events to `github-events.cncf.io`. +- DNS resolves the hostname to the reserved OCI public IP. +- The ingress controller Service is configured with `loadBalancerIP=`, so MetalLB binds that address. +- MetalLB speakers advertise the IP; traffic enters the cluster via the ingress controller. +- The ingress routes traffic to the internal `maintainerd` Service (ClusterIP) and on to the pod. + diff --git a/onboarding/fossa_mock.go b/onboarding/fossa_mock.go index 6db7f4e..5770b68 100644 --- a/onboarding/fossa_mock.go +++ b/onboarding/fossa_mock.go @@ -22,6 +22,8 @@ type MockFossaClient struct { invitationsSent []string teamsCreated []string membersAdded map[int][]string // teamID -> emails added + + createTeamErr error } // NewMockFossaClient creates a new mock FOSSA client @@ -44,6 +46,10 @@ func (m *MockFossaClient) CreateTeam(name string) (*fossa.Team, error) { m.mu.Lock() defer m.mu.Unlock() + if m.createTeamErr != nil { + return nil, m.createTeamErr + } + if _, exists := m.teams[name]; exists { return nil, fossa.ErrTeamAlreadyExists } @@ -239,4 +245,12 @@ func (m *MockFossaClient) Reset() { m.teamsCreated = nil m.membersAdded = make(map[int][]string) m.importedRepos = make(map[int]fossa.ImportedProjects) + m.createTeamErr = nil +} + +// SetCreateTeamError configures the mock to fail team creation requests. +func (m *MockFossaClient) SetCreateTeamError(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.createTeamErr = err } diff --git a/onboarding/server.go b/onboarding/server.go index 8b360f8..0e3ead4 100644 --- a/onboarding/server.go +++ b/onboarding/server.go @@ -8,6 +8,7 @@ import ( "maintainerd/model" "net/http" "os" + "strings" "time" "golang.org/x/oauth2" @@ -81,7 +82,7 @@ func (s *EventListener) Run(addr string) error { Addr: addr, Handler: mux, ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, + WriteTimeout: 120 * time.Second, IdleTimeout: 60 * time.Second, } return server.ListenAndServe() @@ -296,46 +297,52 @@ func (s *EventListener) signProjectUpForFOSSA(project model.Project) ([]string, } else { // create the team on FOSSA, add the team to the ServiceTeams team, err := s.FossaClient.CreateTeam(project.Name) - if err != nil { actions = append(actions, fmt.Sprintf(":x: Problem creating team on FOSSA for %s: %v", project.Name, err)) - } else { - log.Printf("team created: %s", team.Name) - actions = append(actions, - fmt.Sprintf("👥 [%s team](https://app.fossa.com/account/settings/organization/teams/%d) has been created in FOSSA", - team.Name, team.ID)) - _, err := s.Store.CreateServiceTeam(project.ID, project.Name, team.ID, team.Name) - if err != nil { - log.Printf("handleWebhook: WRN, failed to create service team: %v", err) - } + return actions, fmt.Errorf("create team on FOSSA: %w", err) } + + log.Printf("team created: %s", team.Name) + actions = append(actions, + fmt.Sprintf("👥 [%s team](https://app.fossa.com/account/settings/organization/teams/%d) has been created in FOSSA", + team.Name, team.ID)) + _, err = s.Store.CreateServiceTeam(project.ID, project.Name, team.ID, team.Name) if err != nil { - log.Printf("signProjectUpForFOSSA: Error creating team on FOSSA for %s: %v", project.Name, err) + log.Printf("handleWebhook: WRN, failed to create service team: %v", err) } + st = &model.ServiceTeam{ServiceTeamID: team.ID} } if len(maintainers) == 0 { actions = append(actions, fmt.Sprintf("Maintainers not yet registered, for project %s", project.Name)) return actions, fmt.Errorf(":x: no maintainers found for project %d", project.ID) } - var invitedMaintainers string // track who we've invited so we can mention them in a single line comment + var invitedMaintainers []string // track who we've invited so we can mention them in a single line comment + var existingMaintainers []string // track who is already a member over on CNCF FOSSA for _, maintainer := range maintainers { err := s.FossaClient.SendUserInvitation(maintainer.Email) // TODO See if I can Name the User on FOSSA! - if errors.Is(err, fossa.ErrInviteAlreadyExists) { - actions = append(actions, fmt.Sprintf("@%s : you have a pending invitation to join CNCF FOSSA. Please check your registered email and accept the invitation within 48 hours.", maintainer.GitHubAccount)) + invitedMaintainers = append(invitedMaintainers, maintainer.GitHubAccount) // invited already } else if errors.Is(err, fossa.ErrUserAlreadyMember) { - // TODO Edge case - maintainers already signed up to CNCF FOSSA, maintainer on an another project? - actions = append(actions, fmt.Sprintf("@%s : You are CNCF FOSSA User", maintainer.GitHubAccount)) - // TODO call fc.AddUserToTeamByEmail() - log.Printf("user is already a member, skipping") + err := s.FossaClient.AddUserToTeamByEmail(st.ServiceTeamID, maintainer.Email, 3) + if err != nil { + actions = append(actions, fmt.Sprintf("@%s : error adding you to your team on CNCF FOSSA", maintainer.GitHubAccount)) + } else { + existingMaintainers = append(existingMaintainers, maintainer.GitHubAccount) + } } else if err != nil { log.Printf("error sending invite: %v", err) - actions = append(actions, fmt.Sprintf("@%s : there was a problem sending out a CNCF FOSSA invitation to you, a CNCF Staff member will contact you.", maintainer.GitHubAccount)) + actions = append(actions, fmt.Sprintf("@%s there was a problem sending you a CNCF FOSSA invitation. A CNCF Staff member will contact you.", maintainer.GitHubAccount)) } else { - invitedMaintainers = invitedMaintainers + " @" + maintainer.GitHubAccount + invitedMaintainers = append(invitedMaintainers, maintainer.GitHubAccount) // invited just now } } - actions = append(actions, fmt.Sprintf("✅ Invitation(s) to join CNCF FOSSA sent to%s", invitedMaintainers)) + + if len(invitedMaintainers) > 0 { + actions = append(actions, fmt.Sprintf("✅ Invitation(s) to join CNCF FOSSA sent to %s", formatHandles(invitedMaintainers))) + } + if len(existingMaintainers) != 0 { + actions = append(actions, fmt.Sprintf("✅ CNCF FOSSA Users added to the team as Team Admins %s", formatHandles(existingMaintainers))) + } // check if the project team has imported their repos. If we label an onboarding issue with 'fossa' and the project // has been manually setup in the past, better to report that repos have been imported into FOSSA. @@ -355,10 +362,16 @@ func (s *EventListener) signProjectUpForFOSSA(project model.Project) ([]string, } else { actions = append(actions, fmt.Sprintf("The %s project team have imported %d repo(s)
%s", project.Name, count, importedRepos)) } - return actions, nil } +func formatHandles(handles []string) string { + if len(handles) == 0 { + return "" + } + return "@" + strings.Join(handles, " @") +} + func (s *EventListener) updateIssue(ctx context.Context, owner, repo string, issueNumber int, comment string) error { issueComment := &github.IssueComment{ Body: github.String(comment), diff --git a/onboarding/server_test.go b/onboarding/server_test.go index a39c340..7d628d2 100644 --- a/onboarding/server_test.go +++ b/onboarding/server_test.go @@ -1,6 +1,7 @@ package onboarding import ( + "errors" "net/http" "testing" @@ -163,11 +164,11 @@ func TestFossaChosen_Basic(t *testing.T) { // Execute server.fossaChosen(project.Name, req, issueEvent) - // Verify GitHub comment mentions pending invitation + // Verify GitHub comment includes aggregated invitation summary comments := mockGitHub.GetCreatedComments() require.Len(t, comments, 1) assert.Contains(t, comments[0].Body, "@alice") - assert.Contains(t, comments[0].Body, "pending invitation") + assert.Contains(t, comments[0].Body, "Invitation(s) to join CNCF FOSSA sent to") }) t.Run("maintainer already exists in FOSSA", func(t *testing.T) { @@ -188,10 +189,30 @@ func TestFossaChosen_Basic(t *testing.T) { // Execute server.fossaChosen(project.Name, req, issueEvent) - // Verify GitHub comment mentions user is already member + // Verify GitHub comment mentions aggregated existing member info comments := mockGitHub.GetCreatedComments() require.Len(t, comments, 1) assert.Contains(t, comments[0].Body, "@alice") - assert.Contains(t, comments[0].Body, "CNCF FOSSA User") + assert.Contains(t, comments[0].Body, "CNCF FOSSA Users added to the team as Team Admins") + }) +} + +func TestSignProjectUpForFOSSA_CreateTeamFailure(t *testing.T) { + db := setupTestDB(t) + project, maintainers := seedProjectData(t, db) + + mockFossa := NewMockFossaClient() + mockFossa.SetCreateTeamError(errors.New("boom")) + + for _, maintainer := range maintainers { + mockFossa.SetUserExists(maintainer.Email, true) + } + + mockGitHub := NewMockGitHubTransport() + server := createTestServer(t, db, mockFossa, mockGitHub) + + assert.NotPanics(t, func() { + _, err := server.signProjectUpForFOSSA(project) + assert.Error(t, err) }) } diff --git a/plugins/fossa/user-invites.md b/plugins/fossa/user-invites.md new file mode 100644 index 0000000..b2e32b0 --- /dev/null +++ b/plugins/fossa/user-invites.md @@ -0,0 +1,24 @@ +# FOSSA User Invitation Endpoints + +## List Pending Invitations +- **Method**: GET +- **Path**: /api/user-invitations +- **Description**: Returns all active invitations that have not yet expired (48-hour lifetime). Includes invitee email, creator information, and relevant timestamps. + +## Create Invitations +- **Method**: POST +- **Path**: /api/organizations/:id/invite +- **Description**: Creates new user invitations, supporting both single and bulk operations for the specified organization. + +## Delete Invitation +- **Method**: DELETE +- **Path**: /api/user-invitations/:email +- **Description**: Cancels a pending invitation identified by the invitee email address. + +## Pending SSO Domain Invitations +- **Method**: GET +- **Path**: /api/organizations/:id/pending-sso-domains +- **Description**: Lists outstanding SSO domain verification invitations for the specified organization. + +## Authorization +- Proper user invitation permissions are required to call these endpoints successfully.