Skip to content

Commit af59576

Browse files
kamushadeneskamushadenes
committed
[wip] attempt to sync with upstream
1 parent 45a0c7d commit af59576

File tree

82 files changed

+4269
-476
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+4269
-476
lines changed

‎.github/workflows/build_and_test.yml‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
name: Run Build and Test
2+
3+
on:
4+
push:
5+
branches:
6+
- "main"
7+
pull_request:
8+
branches:
9+
- "*"
10+
11+
jobs:
12+
build:
13+
name: Build
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Checkout
17+
uses: actions/checkout@v4
18+
- name: Setup Go
19+
uses: actions/setup-go@v5
20+
with:
21+
go-version: "1.22.0"
22+
- shell: bash
23+
run: |
24+
make build
25+
26+
e2e-tests:
27+
needs: build
28+
name: e2e / ${{ matrix.suite.name }}
29+
runs-on: ubuntu-latest
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
suite: [
34+
# CLI test suites
35+
{ group: "cmd", name: "cmd", path: "" },
36+
# providers suites, some of the providers are too heavy to run as single test
37+
{
38+
group: "pkg/providers",
39+
name: "container",
40+
path: "container",
41+
container: true,
42+
},
43+
{ group: "pkg/providers", name: "yt", path: "yt", yt: true },
44+
{
45+
group: "pkg/providers",
46+
name: "providers-postgres",
47+
path: "postgres",
48+
},
49+
# e2e test suites
50+
{ group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
51+
{ group: "tests/e2e", name: "pg2pg", path: "pg2pg" },
52+
{ group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
53+
{ group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
54+
{ group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
55+
{ group: "tests/e2e", name: "ch2s3", path: "ch2s3" },
56+
]
57+
steps:
58+
- name: Checkout
59+
uses: actions/checkout@v4
60+
- name: Setup Go
61+
uses: actions/setup-go@v5
62+
with:
63+
go-version: "1.22.0"
64+
- shell: bash
65+
run: |
66+
go install gotest.tools/gotestsum@latest
67+
- shell: bash
68+
run: |
69+
curl https://clickhouse.com/ | sh
70+
sudo ./clickhouse install
71+
- name: Setup PostgreSQL
72+
uses: tj-actions/install-postgresql@v3
73+
with:
74+
postgresql-version: 16
75+
- shell: bash
76+
run: |
77+
pg_dump --version
78+
- uses: engineerd/setup-kind@v0.6.2
79+
if: matrix.suite.container
80+
with:
81+
version: "v0.26.0"
82+
# Handled by the test code
83+
skipClusterCreation: true
84+
skipClusterDeletion: true
85+
skipClusterLogsExport: true
86+
- shell: bash
87+
if: matrix.suite.yt
88+
name: prepare local YT
89+
run: |
90+
go build -o binaries/lightexe ./pkg/providers/yt/lightexe/*.go
91+
docker compose -f "pkg/providers/yt/recipe/docker-compose.yml" up -d --build
92+
export YT_PROXY=localhost:8180
93+
export TEST_DEPS_BINARY_PATH=binaries
94+
- shell: bash
95+
run: |
96+
make run-tests SUITE_GROUP="${{ matrix.suite.group }}" SUITE_PATH="${{ matrix.suite.path }}" SUITE_NAME="${{ matrix.suite.name }}"
97+
env:
98+
TEST_KUBERNETES_INTEGRATION: ${{ matrix.suite.container == true && '1' || '' }}
99+
- name: Upload Test Results
100+
uses: actions/upload-artifact@v4
101+
if: always()
102+
with:
103+
name: test-reports-${{ matrix.suite.name }}
104+
path: reports/*.xml
105+
- name: Fail if tests failed
106+
if: failure()
107+
run: exit 1
108+
109+
generic-tests:
110+
needs: build
111+
name: tests - ${{ matrix.suite.name }}
112+
runs-on: ubuntu-latest
113+
strategy:
114+
fail-fast: false
115+
matrix:
116+
suite: [
117+
# canon test suites
118+
{ group: "tests/canon", name: "canon-parser", path: "parser" },
119+
{ group: "tests/storage", name: "storage-pg", path: "pg" },
120+
# internal test suites
121+
{ group: "internal", name: "internal", path: "..." },
122+
# provider test suites
123+
{ group: "pkg/providers", name: "providers-mongo", path: "mongo" },
124+
{ group: "pkg/providers", name: "providers-mysql", path: "mysql" },
125+
{
126+
group: "pkg/providers",
127+
name: "providers-sample",
128+
path: "sample",
129+
},
130+
{ group: "pkg/providers", name: "providers-kafka", path: "kafka" },
131+
{
132+
group: "pkg/providers",
133+
name: "providers-kinesis",
134+
path: "kinesis",
135+
},
136+
{
137+
group: "pkg/providers",
138+
name: "providers-greenplum",
139+
path: "greenplum",
140+
},
141+
{
142+
group: "pkg/providers",
143+
name: "providers-clickhouse",
144+
path: "clickhouse",
145+
},
146+
{
147+
group: "pkg/providers",
148+
name: "providers-elastic",
149+
path: "elastic",
150+
},
151+
# pkg test suites
152+
{ group: "pkg", name: "abstract", path: "abstract" },
153+
{ group: "pkg", name: "transformer", path: "transformer" },
154+
{ group: "pkg", name: "predicate", path: "predicate" },
155+
{ group: "pkg", name: "dblog", path: "dblog" },
156+
{ group: "pkg", name: "functions", path: "functions" },
157+
{ group: "pkg", name: "maplock", path: "maplock" },
158+
{ group: "pkg", name: "middlewares", path: "middlewares" },
159+
{ group: "pkg", name: "parsequeue", path: "parsequeue" },
160+
{ group: "pkg", name: "util", path: "util" },
161+
{ group: "pkg", name: "stringutil", path: "stringutil" },
162+
{ group: "pkg", name: "serializer", path: "serializer" },
163+
{ group: "pkg", name: "worker", path: "worker" },
164+
{ group: "pkg", name: "schemaregistry", path: "schemaregistry" },
165+
{ group: "pkg", name: "parsers-generic", path: "parsers/generic" },
166+
{ group: "pkg", name: "parsers-tests", path: "parsers/tests" },
167+
{ group: "pkg", name: "parsers-scanner", path: "parsers/scanner" },
168+
]
169+
steps:
170+
- name: Checkout
171+
uses: actions/checkout@v4
172+
- name: Setup Go
173+
uses: actions/setup-go@v5
174+
with:
175+
go-version: "1.22.0"
176+
- shell: bash
177+
run: |
178+
go install gotest.tools/gotestsum@latest
179+
- shell: bash
180+
run: |
181+
curl https://clickhouse.com/ | sh
182+
sudo ./clickhouse install
183+
- name: Setup PostgreSQL
184+
uses: tj-actions/install-postgresql@v3
185+
with:
186+
postgresql-version: 16
187+
- shell: bash
188+
run: |
189+
echo "Running ${{ matrix.suite.group }} suite ${{ matrix.suite.name }}"
190+
export RECIPE_CLICKHOUSE_BIN=clickhouse
191+
export USE_TESTCONTAINERS=1
192+
gotestsum \
193+
--junitfile="reports/${{ matrix.suite.name }}.xml" \
194+
--junitfile-project-name="${{ matrix.suite.group }}" \
195+
--junitfile-testsuite-name="short" \
196+
--rerun-fails \
197+
--format github-actions \
198+
--packages="./${{ matrix.suite.group }}/${{ matrix.suite.path }}/..." \
199+
-- -timeout=15m
200+
- name: Upload Test Results
201+
uses: actions/upload-artifact@v4
202+
if: always()
203+
with:
204+
name: test-reports-${{ matrix.suite.name }}
205+
path: reports/${{ matrix.suite.name }}.xml
206+
- name: Fail if tests failed
207+
if: failure()
208+
run: exit 1
209+
210+
test-report:
211+
needs: [generic-tests, e2e-tests]
212+
name: test-report
213+
if: always() && !contains(needs.*.result, 'skipped')
214+
runs-on: ubuntu-latest
215+
steps:
216+
- name: Checkout
217+
uses: actions/checkout@v4
218+
- name: Download All Test Reports
219+
uses: actions/download-artifact@v4
220+
with:
221+
pattern: test-reports-*
222+
merge-multiple: true
223+
path: reports/
224+
- name: Test Summary
225+
uses: test-summary/action@v2
226+
if: always()
227+
with:
228+
paths: "reports/*.xml"

‎.github/workflows/release-chart.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Release Helm Chart
2+
3+
on:
4+
push:
5+
paths:
6+
- 'helm/transfer/**'
7+
8+
pull_request:
9+
paths:
10+
- 'helm/transfer/**'
11+
12+
jobs:
13+
publish:
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v3
19+
20+
- name: Set up Helm
21+
uses: azure/setup-helm@v3
22+
23+
- name: Package helm
24+
env:
25+
GITHUB_USERNAME: laskoviymishka
26+
GHCR_TOKEN: ${{ secrets.GH_HELM_PAT }}
27+
HELM_EXPERIMENTAL_OCI: 1
28+
run: |
29+
make helm-push
30+
make clean

‎.github/workflows/release-docker-branch.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Auto build and publish of Transfer For PR
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- "*"
7+
8+
jobs:
9+
trigger-build-pr:
10+
permissions:
11+
contents: write
12+
packages: write
13+
uses: ./.github/workflows/release.yml
14+
with:
15+
version: '${{ github.event.pull_request.number }}'
16+
extra_args: '--snapshot'
17+

‎.github/workflows/release-docker-latest.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Auto build and publish of Transfer image
2+
3+
on:
4+
push:
5+
branches:
6+
- 'main'
7+
8+
jobs:
9+
trigger-build-dev:
10+
permissions:
11+
contents: write
12+
packages: write
13+
uses: ./.github/workflows/release.yml
14+
with:
15+
version: dev
16+
extra_args: '--snapshot'
17+
18+
trigger-build-latest:
19+
permissions:
20+
contents: write
21+
packages: write
22+
uses: ./.github/workflows/release.yml
23+
with:
24+
version: latest
25+
extra_args: '--snapshot'
26+

‎.github/workflows/release-docker.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Auto build and publish of Transfer image
2+
3+
on:
4+
push:
5+
branches:
6+
- 'release-*'
7+
tags:
8+
- 'v*.*.*'
9+
10+
jobs:
11+
trigger-build:
12+
permissions:
13+
contents: write
14+
packages: write
15+
uses: ./.github/workflows/release.yml
16+
with:
17+
version: ${{ github.ref_name }}
18+

‎.github/workflows/release-website.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# This is a basic workflow to help you get started with Actions
2+
3+
name: Website
4+
5+
# Controls when the workflow will run
6+
on:
7+
# Triggers the workflow on push or pull request events but only for the "main" branch
8+
push:
9+
paths:
10+
- "docs/**"
11+
branches:
12+
- "main"
13+
pull_request:
14+
branches:
15+
- "main"
16+
paths:
17+
- "docs/**"
18+
19+
# Allows you to run this workflow manually from the Actions tab
20+
workflow_dispatch:
21+
22+
permissions:
23+
contents: read
24+
pages: write
25+
id-token: write
26+
27+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
28+
jobs:
29+
# This workflow contains a single job called "build"
30+
build:
31+
# The type of runner that the job will run on
32+
runs-on: ubuntu-latest
33+
34+
# Steps represent a sequence of tasks that will be executed as part of the job
35+
steps:
36+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
37+
- uses: actions/checkout@v4
38+
- name: Setup Pages
39+
uses: actions/configure-pages@v5
40+
- name: Build docs
41+
uses: diplodoc-platform/docs-build-action@v3
42+
with:
43+
src-root: "./docs"
44+
build-root: "./docs-html"
45+
# Build the landing page
46+
- name: Set up Node.js
47+
uses: actions/setup-node@v3
48+
with:
49+
node-version: '18' # Укажите подходящую версию Node.js
50+
51+
- name: Install dependencies for landing page
52+
run: |
53+
cd docs/website
54+
npm install -g js-yaml
55+
npm install --legacy-peer-deps
56+
57+
- name: Build landing page
58+
run: |
59+
cd docs/website
60+
PUBLIC_URL=/transferia npm run build
61+
cd ../../
62+
ls ./_docs-lint
63+
cp -r ./_docs-lint ./docs/website/build/docs
64+
# Upload the landing page build artifact
65+
- name: Upload landing page artifact
66+
uses: actions/upload-pages-artifact@v3
67+
with:
68+
path: './docs/website/build'
69+
- name: Deploy to GitHub Pages
70+
id: deployment
71+
uses: actions/deploy-pages@v4

0 commit comments

Comments
 (0)