Skip to content

Commit 9ddb0ed

Browse files
vijeyash1vijeyash1
committed
migration added
1 parent 3b0da2a commit 9ddb0ed

File tree

20 files changed

+711
-32
lines changed

20 files changed

+711
-32
lines changed

‎.github/workflows/migration-image.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Migration Docker Image CI
2+
3+
on:
4+
push:
5+
paths-ignore:
6+
- '**.md'
7+
branches:
8+
- 'main'
9+
10+
jobs:
11+
build:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
packages: write
15+
id-token: write
16+
contents: read
17+
actions: read
18+
security-events: write
19+
env:
20+
REGISTRY: ghcr.io
21+
GH_URL: https://github.com
22+
steps:
23+
- name: Checkout GitHub Action
24+
uses: actions/checkout@v3
25+
26+
- name: Set up Docker Buildx
27+
id: buildx
28+
uses: docker/setup-buildx-action@v2
29+
30+
- name: Docker metadata
31+
id: metadata
32+
uses: docker/metadata-action@v4
33+
with:
34+
images: ${{ env.REGISTRY }}/${{ github.repository }}/migration
35+
tags: |
36+
type=semver,pattern={{version}}
37+
type=semver,pattern={{major}}.{{minor}}
38+
type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }}
39+
flavor: |
40+
latest=true
41+
42+
- name: Login to GitHub Container Registry
43+
uses: docker/login-action@v2
44+
with:
45+
registry: ${{ env.REGISTRY }}
46+
username: ${{ github.actor }}
47+
password: ${{ secrets.GITHUB_TOKEN }}
48+
49+
- name: Build image and push to GitHub Container Registry
50+
uses: docker/build-push-action@v4
51+
with:
52+
context: .
53+
file: ./dockerfiles/migration/Dockerfile
54+
tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }}
55+
labels: ${{ steps.metadata.outputs.labels }}
56+
push: true
57+
58+
- name: Install cosign
59+
uses: sigstore/cosign-installer@main
60+
61+
- name: Sign the images
62+
run: |
63+
cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }}
64+
env:
65+
COSIGN_EXPERIMENTAL: 1
66+
67+
- name: Verify the pushed tags
68+
run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/migration.yml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com
69+
env:
70+
COSIGN_EXPERIMENTAL: 1
71+
72+
- name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
73+
uses: aquasecurity/trivy-action@master
74+
with:
75+
scan-type: 'fs'
76+
format: 'github'
77+
output: 'dependency-results.sbom.json'
78+
image-ref: '.'
79+
github-pat: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/migration-pr.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Migration Docker Image CI
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- 'main'
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
env:
12+
REGISTRY: ghcr.io
13+
GH_URL: https://github.com
14+
steps:
15+
-
16+
name: Checkout
17+
uses: actions/checkout@v3
18+
with:
19+
fetch-depth: 0
20+
21+
-
22+
name: Set up QEMU
23+
uses: docker/setup-qemu-action@v2
24+
25+
-
26+
name: Set up Docker Buildx
27+
uses: docker/setup-buildx-action@v1
28+
29+
-
30+
name: Login to ghcr registry
31+
uses: docker/login-action@v2
32+
with:
33+
registry: ${{ env.REGISTRY }}
34+
username: ${{ github.actor }}
35+
password: ${{ secrets.GITHUB_TOKEN }}
36+
37+
-
38+
name: Build and push on PR
39+
uses: docker/build-push-action@v4
40+
if: github.event_name == 'pull_request'
41+
with:
42+
context: .
43+
file: ./dockerfiles/migration/Dockerfile
44+
push: true
45+
tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:pr-${{ github.event.pull_request.number }}
46+
build-args: |
47+
"GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}"

‎.github/workflows/migration-release.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: migration-release
2+
3+
on:
4+
push:
5+
tags:
6+
- "v*.*.*"
7+
8+
jobs:
9+
push_to_registry:
10+
name: Build and push Docker image to GitHub container registry.
11+
runs-on: ubuntu-20.04
12+
permissions:
13+
packages: write
14+
id-token: write
15+
contents: read
16+
actions: read
17+
security-events: write
18+
env:
19+
REGISTRY: ghcr.io
20+
GH_URL: https://github.com
21+
steps:
22+
- name: Set environment variable
23+
run: |
24+
echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV
25+
- name: Test environment variable
26+
run: echo ${{ env.RELEASE_VERSION }}
27+
- name: Check out GitHub repo
28+
uses: actions/checkout@v3
29+
- name: Login to GitHub Container Registry
30+
uses: docker/login-action@v2
31+
with:
32+
registry: ${{ env.REGISTRY }}
33+
username: ${{ github.actor }}
34+
password: ${{ secrets.GITHUB_TOKEN }}
35+
- name: Build image and push to GitHub Container Registry
36+
uses: docker/build-push-action@v4
37+
with:
38+
push: true
39+
context: ./
40+
file: ./dockerfiles/migration/Dockerfile
41+
tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }}
42+
- name: Install cosign
43+
uses: sigstore/cosign-installer@main
44+
- name: Sign the images
45+
run: |
46+
cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }}
47+
env:
48+
COSIGN_EXPERIMENTAL: 1
49+
- name: Verify the pushed tags
50+
run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/migration-release.yml@refs/tags/${{ env.RELEASE_VERSION }} --certificate-oidc-issuer https://token.actions.githubusercontent.com
51+
env:
52+
COSIGN_EXPERIMENTAL: 1
53+
- name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
54+
uses: aquasecurity/trivy-action@master
55+
with:
56+
scan-type: 'fs'
57+
format: 'github'
58+
output: 'dependency-results.sbom.json'
59+
image-ref: '.'
60+
github-pat: ${{ secrets.GITHUB_TOKEN }}

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
civo
2-
migration/
2+
alloc.svg
3+
allocs.pprof
4+
cpu.pprof
5+
steps-to-test.txt

‎charts/client/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ dependencies:
2626
- name: nats
2727
condition: nats.enabled
2828
version: 0.13.4
29-
repository: https://intelops.github.io/kubviz/
29+
repository: https://intelops.github.io/kubviz/
3030
- name: clickhouse
3131
condition: clickhouse.enabled
3232
version: 1.0.2

‎charts/client/templates/deployment.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,22 @@ spec:
2727
serviceAccountName: {{ include "client.serviceAccountName" . }}
2828
securityContext:
2929
{{- toYaml .Values.podSecurityContext | nindent 8 }}
30+
initContainers:
31+
- name: migration-init
32+
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag }}"
33+
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
34+
command:
35+
- /bin/sh
36+
- -c
37+
args:
38+
- "/script/wait-for-clickhouse.sh && /migration sql -e --yes"
39+
env:
40+
- name: SCHEMA_PATH
41+
value : {{ .Values.migration.schema.path }}
42+
- name: DB_ADDRESS
43+
value: {{ include "client.fullname" . }}-clickhouse
44+
- name: DB_PORT
45+
value: "9000"
3046
containers:
3147
- name: {{ .Chart.Name }}
3248
securityContext:

‎charts/client/values.yaml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Default values for client.
1+
# Default values for client.
22
# This is a YAML-formatted file.
33
# Declare variables to be passed into your templates.
44

@@ -106,3 +106,12 @@ datasources:
106106
label: grafana_datasource
107107
labelValue: "1"
108108
uid: vertamedia-clickhouse-datasource
109+
110+
migration:
111+
enabled: true
112+
image:
113+
repository: ghcr.io/intelops/kubviz/migration
114+
pullPolicy: Always
115+
tag: "v1.1.0"
116+
schema:
117+
path: "/sql"

‎client/pkg/clickhouse/db_client.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,12 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
7474
return nil, err
7575
}
7676

77-
tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, quayContainerPushEventTable, jfrogContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
78-
for _, table := range tables {
79-
if err = splconn.Exec(context.Background(), string(table)); err != nil {
80-
return nil, err
81-
}
82-
}
77+
// tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, quayContainerPushEventTable, jfrogContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
78+
// for _, table := range tables {
79+
// if err = splconn.Exec(context.Background(), string(table)); err != nil {
80+
// return nil, err
81+
// }
82+
// }
8383
stdconn := clickhouse.OpenDB(&clickhouse.Options{
8484
Addr: []string{fmt.Sprintf("%s:%d", conf.DBAddress, conf.DbPort)},
8585
})
@@ -126,7 +126,6 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE
126126
tag,
127127
imageName,
128128
string(pushEventJSON),
129-
pushEvent.Timestamp,
130129
size,
131130
shaID,
132131
currentTime,

‎client/pkg/clickhouse/statements.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,6 @@ const azureContainerPushEventTable DBStatement = `
183183
Tag String,
184184
ImageName String,
185185
Event String,
186-
Timestamp String,
187186
Size Int32,
188187
SHAID String,
189188
EventTime DateTime('UTC')
@@ -242,7 +241,7 @@ const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_na
242241
const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)"
243242
const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
244243
const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
245-
const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
244+
const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Size, SHAID, EventTime) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?)"
246245
const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
247246
const InsertQuayContainerPushEvent DBStatement = "INSERT INTO quaycontainerpush (name, repository, nameSpace, dockerURL, homePage, tag, Event, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
248247
const InsertJfrogContainerPushEvent DBStatement = "INSERT INTO jfrogcontainerpush (Domain, EventType, RegistryURL, RepositoryName, SHAID, Size, ImageName, Tag, Event, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"

0 commit comments

Comments
 (0)