Skip to content

Commit caa2a8b

Browse files
detjdetj
authored
feat(backend): add ingest service (#3249)
- Spins off a new ingest service carved out of API service - Create a new module for sharing common functionality - Fixed an incorrect table name in cleanup service - Refactor out global server.Server.* usage from relevant sites in `backend/api - Session detail query now uses query caching - Fix an issue where bug report status change would fail - Improved app/span filters query to fetch latest state - Show ingest service URL otherwise fallback to API service URL on apps page. - Update self hosting guide to reflect newer ingest endpoint setup - Sessionator now defaults to new ingest service - API service ingest now proxies to ingest service for self hosted users who have not setup an ingest endpoint yet - API service ingest remains fully function for backward compatibility fixes #3065 --------- Signed-off-by: detj <detj@users.noreply.github.com>
1 parent c81027b commit caa2a8b

File tree

140 files changed

+7435
-687
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+7435
-687
lines changed

‎.github/workflows/ingest.yml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: Ingest
2+
on:
3+
pull_request:
4+
paths:
5+
- .github/workflows/ingest.yml
6+
- backend/ingest/dockerfile
7+
- backend/ingest/**/*.go
8+
- backend/ingest/**/*.mod
9+
- backend/ingest/**/*.sum
10+
- backend/ingest/**/*.go
11+
push:
12+
branches:
13+
- "main"
14+
paths:
15+
- .github/workflows/ingest.yml
16+
- backend/ingest/dockerfile
17+
- backend/ingest/**/*.go
18+
- backend/ingest/**/*.mod
19+
- backend/ingest/**/*.sum
20+
- backend/ingest/**/*.go
21+
tags:
22+
- "v*"
23+
24+
permissions:
25+
contents: read
26+
27+
env:
28+
REGISTRY: ghcr.io
29+
IMAGE_NAME: measure-sh/ingest
30+
31+
jobs:
32+
build:
33+
name: Build
34+
runs-on: ubuntu-latest
35+
strategy:
36+
matrix:
37+
go-version: ["1.24.x", "1.25.x"]
38+
defaults:
39+
run:
40+
working-directory: backend/ingest
41+
steps:
42+
- name: Checkout repository
43+
uses: actions/checkout@v4
44+
- name: Setup go
45+
uses: actions/setup-go@v5
46+
with:
47+
go-version: ${{ matrix.go-version }}
48+
check-latest: true
49+
cache-dependency-path: backend/ingest/go.sum
50+
- name: Display go version
51+
run: go version
52+
- name: Install dependencies
53+
run: go get .
54+
- name: Build with ${{ matrix.go-version }}
55+
run: go build -v ./...
56+
57+
test:
58+
name: Test
59+
runs-on: ubuntu-latest
60+
strategy:
61+
matrix:
62+
go-version: ["1.24.x", "1.25.x"]
63+
defaults:
64+
run:
65+
working-directory: backend/ingest
66+
steps:
67+
- name: Checkout repository
68+
uses: actions/checkout@v4
69+
- name: Setup go
70+
uses: actions/setup-go@v5
71+
with:
72+
go-version: ${{ matrix.go-version }}
73+
check-latest: true
74+
cache-dependency-path: backend/ingest/go.sum
75+
- name: Display go version
76+
run: go version
77+
- name: Install dependencies
78+
run: go get .
79+
- name: Test with ${{ matrix.go-version }}
80+
run: go test ./...
81+
82+
push:
83+
name: Build and push image
84+
runs-on: ubuntu-latest
85+
if: github.ref_type == 'tag' && startsWith(github.ref, 'refs/tags/v')
86+
needs: [build, test]
87+
permissions:
88+
contents: read
89+
packages: write
90+
attestations: write
91+
id-token: write
92+
steps:
93+
- name: Checkout repository
94+
uses: actions/checkout@v4
95+
96+
- name: Login to Container Registry
97+
uses: docker/login-action@v3
98+
with:
99+
registry: ${{ env.REGISTRY }}
100+
username: ${{ github.actor }}
101+
password: ${{ secrets.GITHUB_TOKEN }}
102+
103+
- name: Extract metadata (tags, labels) for Docker
104+
id: meta
105+
uses: docker/metadata-action@v5
106+
with:
107+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
108+
109+
- name: Build and push OCI image
110+
id: push
111+
uses: docker/build-push-action@v6
112+
with:
113+
context: backend/ingest
114+
file: backend/ingest/dockerfile
115+
build-contexts: |
116+
api=backend/api
117+
email=backend/email
118+
billing=backend/billing
119+
push: true
120+
tags: ${{ steps.meta.outputs.tags }}
121+
labels: |
122+
org.opencontainers.image.source=https://github.com/measure-sh/measure/tree/${{ github.ref_name }}/backend/ingest/dockerfile
123+
org.opencontainers.image.description=Measure Ingest Server
124+
org.opencontainers.image.licenses=Apache-2.0
125+
126+
- name: Generate artifact attestation
127+
uses: actions/attest-build-provenance@v1
128+
with:
129+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
130+
subject-digest: ${{ steps.push.outputs.digest }}
131+
push-to-registry: true

‎.github/workflows/publish.yml‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,40 @@ env:
2828
API_IMAGE_NAME: measure-sh/api
2929

3030
jobs:
31+
build-test-ingest:
32+
name: Build & Test Ingest
33+
runs-on: ubuntu-latest
34+
strategy:
35+
matrix:
36+
go-version: ["1.24.x", "1.25.x"]
37+
defaults:
38+
run:
39+
working-directory: backend/ingest
40+
steps:
41+
- name: Checkout repository
42+
uses: actions/checkout@v4
43+
44+
- name: Set Go module cache
45+
run: echo "GOMODCACHE=$RUNNER_TEMP/gomodcache-${{ matrix.go-version }}" >> "$GITHUB_ENV"
46+
47+
- name: Setup Go ${{ matrix.go-version}}
48+
uses: actions/setup-go@v5
49+
with:
50+
go-version: ${{ matrix.go-version }}
51+
cache-dependency-path: backend/ingest/go.sum
52+
53+
- name: Display go version
54+
run: go version
55+
56+
- name: Install dependencies
57+
run: go mod download
58+
59+
- name: Build with ${{ matrix.go-version }}
60+
run: go build -v ./...
61+
62+
- name: Test with ${{ matrix.go-version }}
63+
run: go test ./...
64+
3165
build-test-api:
3266
name: Build & Test API
3367
runs-on: ubuntu-latest
@@ -87,7 +121,7 @@ jobs:
87121
trigger-cloud-deploy:
88122
name: Trigger Cloud Deploy
89123
runs-on: ubuntu-latest
90-
needs: [build-test-api, build-test-dashboard]
124+
needs: [build-test-ingest, build-test-api, build-test-dashboard]
91125
steps:
92126
- name: Dispatch Cloud Deploy
93127
uses: peter-evans/repository-dispatch@v3

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,8 @@ Package.resolved
1919
# flutter
2020
.index-build/
2121

22+
# zed editor
23+
.zed
24+
2225
# email preview output
23-
backend/email/cmd/preview/output/
26+
backend/email/cmd/preview/output/

‎backend/api/.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
.env
21
.DS_Store

‎backend/api/authsession/authsession.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
package authsession
22

33
import (
4-
"backend/api/server"
54
"context"
65
"encoding/json"
76
"errors"
87
"time"
98

9+
"backend/api/server"
10+
1011
"github.com/golang-jwt/jwt/v5"
1112
"github.com/google/uuid"
1213
"github.com/jackc/pgx/v5"

‎backend/api/authsession/authstate.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package authsession
22

33
import (
4-
"backend/api/server"
54
"context"
65
"encoding/json"
76
"fmt"
@@ -10,6 +9,8 @@ import (
109
"net/url"
1110
"strings"
1211

12+
"backend/api/server"
13+
1314
"github.com/google/uuid"
1415
"github.com/leporo/sqlf"
1516
)

‎backend/api/config/ingest.go‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎backend/api/config/query.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import "time"
77
const DefaultQueryCacheTTL = time.Minute * 10
88

99
// AppFiltersTable is the name of the table for event filters.
10-
const AppFiltersTable = "app_filters"
10+
const AppFiltersTable = "app_filters final"
1111

1212
// SpanFiltersTable is the name of the table for span
1313
// filters.
14-
const SpanFiltersTable = "span_filters"
14+
const SpanFiltersTable = "span_filters final"
1515

1616
// AppMetricsTable is the name of the table for app's
1717
// metrics.

‎backend/api/dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@ ARG TARGETTYPE
66
WORKDIR /go/src
77

88
# Copy dependency modules first for caching
9+
COPY --from=libs go.mod go.sum ../libs/
910
COPY --from=email go.mod go.sum ../email/
1011
COPY --from=billing go.mod go.sum ../billing/
1112
COPY ["go.mod", "go.sum", "./"]
1213

1314
RUN go mod download
1415

1516
# Copy source
17+
COPY --from=libs . ../libs/
1618
COPY --from=email . ../email/
1719
COPY --from=billing . ../billing/
1820
COPY . .

‎backend/api/event/attachment.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ import (
1212
"slices"
1313
"time"
1414

15-
"backend/api/objstore"
1615
"backend/api/server"
16+
"backend/libs/objstore"
1717

1818
"cloud.google.com/go/storage"
1919
"github.com/aws/aws-sdk-go-v2/aws"
@@ -22,6 +22,22 @@ import (
2222
"google.golang.org/api/googleapi"
2323
)
2424

25+
type LocationConfig struct {
26+
IsCloud bool
27+
AWSEndpoint string
28+
AttachmentsBucket string
29+
AttachmentsBucketRegion string
30+
}
31+
32+
type UploadConfig struct {
33+
IsCloud bool
34+
AWSEndpoint string
35+
AttachmentsBucket string
36+
AttachmentsBucketRegion string
37+
AttachmentsAccessKey string
38+
AttachmentsSecretAccessKey string
39+
}
40+
2541
// attachmentTypes is a list of all valid attachment types.
2642
var attachmentTypes = []string{"screenshot", "android_method_trace", "layout_snapshot", "layout_snapshot_json"}
2743

@@ -34,10 +50,8 @@ func isNotFound(err error) bool {
3450

3551
// BuildAttachmentLocation builds the location of the attachment
3652
// object based on runtime environment.
37-
func BuildAttachmentLocation(key string) (location string) {
38-
config := server.Server.Config
39-
40-
if config.IsCloud() {
53+
func BuildAttachmentLocation(key string, config LocationConfig) (location string) {
54+
if config.IsCloud {
4155
location = fmt.Sprintf("https://storage.googleapis.com/%s/%s", config.AttachmentsBucket, key)
4256
return
4357
}
@@ -78,9 +92,7 @@ func (a Attachment) Validate() error {
7892
}
7993

8094
// Upload uploads raw file bytes to an S3 compatible storage system.
81-
func (a *Attachment) Upload(ctx context.Context) (err error) {
82-
config := server.Server.Config
83-
95+
func (a *Attachment) Upload(ctx context.Context, config UploadConfig) (err error) {
8496
// set mime type from extension
8597
ext := filepath.Ext(a.Key)
8698
contentType := "application/octet-stream"
@@ -92,7 +104,7 @@ func (a *Attachment) Upload(ctx context.Context) (err error) {
92104
"original_file_name": a.Name,
93105
}
94106

95-
if config.IsCloud() {
107+
if config.IsCloud {
96108
client, errStorage := storage.NewClient(ctx)
97109
if errStorage != nil {
98110
err = errStorage

0 commit comments

Comments
 (0)