Skip to content

Commit 90eda8d

Browse files
juliojimenezjuliojimenez
committed
clickhouse
Signed-off-by: Julio Jimenez <julio@clickhouse.com>
1 parent 6810748 commit 90eda8d

File tree

4 files changed

+205
-18
lines changed

4 files changed

+205
-18
lines changed

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,7 @@ jobs:
3333
s3-bucket: clickhouse-sbom
3434
s3-key: clickbom.json
3535
repository: ${{ github.repository }}
36+
clickhouse-url: ${{ secrets.CLICKHOUSE_URL }}
37+
clickhouse-database: ${{ secrets.CLICKHOUSE_DATABASE }}
38+
clickhouse-username: ${{ secrets.CLICKHOUSE_USERNAME }}
39+
clickhouse-password: ${{ secrets.CLICKHOUSE_PASSWORD }}

‎README.md‎

Lines changed: 61 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,31 @@ Downloads SBOMs from GitHub. Uploads to S3 and ClickHouse.
88
- [Notes](#notes)
99
- [Usage](#usage)
1010
- [Same Repository](#same-repository)
11+
- [Same Repository with ClickHouse](#same-repository-with-clickhouse)
1112

1213
## Inputs
1314

1415
| Name | Description | Default | Required | Sensitive |
1516
| --------------------- | ------------------------------------- | -------------- | -------- | --------- |
16-
| github-token | GitHub token for authentication | | false | true |
17-
| ghapp-token | GitHub App token for authentication | | false | true |
18-
| aws-access-key-id | AWS access key ID for S3 uploads | | true | true |
19-
| aws-secret-access-key | AWS secret access key for S3 uploads | | true | true |
20-
| aws-region | AWS region for S3 uploads | us-east-1 | false | false |
21-
| s3-bucket | S3 bucket name for uploads | | false | false |
22-
| s3-key | S3 key prefix for uploads | sbom/sbom.json | false | false |
17+
| github-token | GitHub Token | | false | true |
18+
| ghapp-token | GitHub App Token | | false | true |
19+
| aws-access-key-id | AWS Access Key ID | | true | true |
20+
| aws-secret-access-key | AWS Secret Access Key | | true | true |
21+
| aws-region | AWS Region | us-east-1 | false | false |
22+
| s3-bucket | S3 Bucket Name | | false | false |
23+
| s3-key | S3 Key Prefix | sbom/sbom.json | false | false |
2324
| repository | Repository to download SBOM from | | true | false |
2425
| sbom-format | Final SBOM format (spdx or cyclonedx) | cyclonedx | false | false |
25-
| clickhouse-url | ClickHouse URL for uploads | | false | true |
26-
| clickhouse-port | ClickHouse port | 8123 | false | false |
27-
| clickhouse-database | ClickHouse database name | default | false | false |
28-
| clickhouse-username | ClickHouse username | default | false | false |
29-
| clickhouse-password | ClickHouse password | (empty) | false | true |
26+
| clickhouse-url | ClickHouse URL | | false | true |
27+
| clickhouse-database | ClickHouse Database Name | default | false | false |
28+
| clickhouse-username | ClickHouse Username | default | false | false |
29+
| clickhouse-password | ClickHouse Password | (empty) | false | true |
3030

3131
### Notes
3232

3333
- Either `github-token` or `ghapp-token` must be provided for authentication to the GitHub API.
3434
- `sbom-format` specifies the format you want the final SBOM to be in. For example, GitHub only supports SPDX, settings this input to `cyclonedx` will convert the SBOM to CycloneDX format.
35+
- At the moment, ClickHouse ingestion is only supported over HTTP.
3536

3637
## Usage
3738

@@ -76,4 +77,51 @@ jobs:
7677
s3-bucket: my-sbom-bucket
7778
s3-key: clickbom.json
7879
repository: ${{ github.repository }}
79-
```
80+
```
81+
82+
### Same Repository with ClickHouse
83+
84+
Downloads the SBOM from the same repository and uploads it to S3. Converts the SBOM to CycloneDX format. Also uploads the SBOM to ClickHouse.
85+
86+
```yaml
87+
name: Upload SBOM
88+
on:
89+
push:
90+
branches:
91+
- main
92+
93+
jobs:
94+
clickbom:
95+
name: ClickBOM
96+
runs-on: ubuntu-latest
97+
98+
permissions:
99+
id-token: write
100+
contents: read
101+
102+
steps:
103+
- name: Checkout repository
104+
uses: actions/checkout@v2
105+
106+
- name: Configure AWS Credentials
107+
id: aws-creds
108+
uses: aws-actions/configure-aws-credentials@v1
109+
with:
110+
role-to-assume: arn:aws:iam::012345678912:role/GitHubOIDCRole
111+
role-session-name: clickbom-session
112+
aws-region: us-east-1
113+
114+
- name: Upload SBOM
115+
uses: ./
116+
with:
117+
github-token: ${{ secrets.GITHUB_TOKEN }}
118+
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
119+
aws-secret-access-key: ${{ steps.aws-creds.outputs.aws-secret-access-key }}
120+
s3-bucket: my-sbom-bucket
121+
s3-key: clickbom.json
122+
repository: ${{ github.repository }}
123+
clickhouse-url: ${{ secrets.CLICKHOUSE_URL }}
124+
clickhouse-database: ${{ secrets.CLICKHOUSE_DATABASE }}
125+
clickhouse-username: ${{ secrets.CLICKHOUSE_USERNAME }}
126+
clickhouse-password: ${{ secrets.CLICKHOUSE_PASSWORD }}
127+
```

‎action.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ inputs:
3535
clickhouse-url:
3636
description: 'ClickHouse URL for storing SBOM components data'
3737
required: false
38-
clickhouse-port:
39-
description: 'ClickHouse port'
40-
required: false
41-
default: '8123'
38+
# clickhouse-port:
39+
# description: 'ClickHouse port'
40+
# required: false
41+
# default: '8123'
4242
clickhouse-database:
4343
description: 'ClickHouse database name'
4444
required: false
@@ -65,7 +65,7 @@ runs:
6565
REPOSITORY: ${{ inputs.repository }}
6666
SBOM_FORMAT: ${{ inputs.sbom-format }}
6767
CLICKHOUSE_URL: ${{ inputs.clickhouse-url }}
68-
CLICKHOUSE_PORT: ${{ inputs.clickhouse-port }}
68+
# CLICKHOUSE_PORT: ${{ inputs.clickhouse-port }}
6969
CLICKHOUSE_DATABASE: ${{ inputs.clickhouse-database }}
7070
CLICKHOUSE_USERNAME: ${{ inputs.clickhouse-username }}
7171
CLICKHOUSE_PASSWORD: ${{ inputs.clickhouse-password }}

‎entrypoint.sh‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,134 @@ upload_to_s3() {
251251
fi
252252
}
253253

254+
# Create ClickHouse table if it doesn't exist, or truncate if it does
255+
setup_clickhouse_table() {
256+
local table_name="$1"
257+
258+
log_info "Setting up ClickHouse table: $table_name"
259+
260+
# Build ClickHouse URL
261+
local clickhouse_url="${CLICKHOUSE_URL}"
262+
local auth_params=""
263+
264+
if [[ -n "${CLICKHOUSE_USERNAME:-}" && "${CLICKHOUSE_USERNAME}" != "default" ]]; then
265+
auth_params="--user ${CLICKHOUSE_USERNAME}"
266+
fi
267+
268+
if [[ -n "${CLICKHOUSE_PASSWORD:-}" ]]; then
269+
auth_params="${auth_params} --password ${CLICKHOUSE_PASSWORD}"
270+
fi
271+
272+
# Check if table exists
273+
local table_exists
274+
if table_exists=$(curl -s ${auth_params} --data "SELECT COUNT(*) FROM system.tables WHERE database='${CLICKHOUSE_DATABASE}' AND name='${table_name}'" "${clickhouse_url}"); then
275+
if [[ "$table_exists" == "1" ]]; then
276+
log_info "Table $table_name exists, truncating..."
277+
if curl -s ${auth_params} --data "TRUNCATE TABLE ${CLICKHOUSE_DATABASE}.${table_name}" "${clickhouse_url}"; then
278+
log_success "Table $table_name truncated"
279+
else
280+
log_error "Failed to truncate table $table_name"
281+
exit 1
282+
fi
283+
else
284+
log_info "Creating new table: $table_name"
285+
local create_table_sql="
286+
CREATE TABLE ${CLICKHOUSE_DATABASE}.${table_name} (
287+
name String,
288+
version String,
289+
license String,
290+
inserted_at DateTime DEFAULT now()
291+
) ENGINE = MergeTree()
292+
ORDER BY (name, version, license);
293+
"
294+
295+
if curl -s ${auth_params} --data "$create_table_sql" "${clickhouse_url}"; then
296+
log_success "Table $table_name created successfully"
297+
else
298+
log_error "Failed to create table $table_name"
299+
exit 1
300+
fi
301+
fi
302+
else
303+
log_error "Failed to check if table exists"
304+
exit 1
305+
fi
306+
}
307+
308+
insert_sbom_data() {
309+
local sbom_file="$1"
310+
local table_name="$2"
311+
local sbom_format="$3"
312+
313+
log_info "Extracting components from $sbom_format SBOM for ClickHouse"
314+
315+
# Build ClickHouse URL
316+
local clickhouse_url="${CLICKHOUSE_URL}"
317+
local auth_params=""
318+
319+
if [[ -n "${CLICKHOUSE_USERNAME:-}" && "${CLICKHOUSE_USERNAME}" != "default" ]]; then
320+
auth_params="--user ${CLICKHOUSE_USERNAME}"
321+
fi
322+
323+
if [[ -n "${CLICKHOUSE_PASSWORD:-}" ]]; then
324+
auth_params="${auth_params} --password ${CLICKHOUSE_PASSWORD}"
325+
fi
326+
327+
# Create temporary file for data
328+
local data_file="$temp_dir/clickhouse_data.tsv"
329+
330+
# Extract data based on SBOM format
331+
case "$sbom_format" in
332+
"cyclonedx")
333+
# Extract from CycloneDX format
334+
jq -r '
335+
.components[]? // empty |
336+
[
337+
.name // "unknown",
338+
.version // "unknown",
339+
(.licenses[]?.license.id // .licenses[]?.license.name // .licenses[]?.expression // "unknown")
340+
] | @tsv
341+
' "$sbom_file" > "$data_file"
342+
;;
343+
"spdx"|"spdxjson")
344+
# Extract from SPDX format
345+
jq -r '
346+
.packages[]? // empty |
347+
select(.name != null) |
348+
[
349+
.name // "unknown",
350+
.versionInfo // "unknown",
351+
(.licenseDeclared // .licenseConcluded // "unknown")
352+
] | @tsv
353+
' "$sbom_file" > "$data_file"
354+
;;
355+
*)
356+
log_error "Unsupported SBOM format for ClickHouse: $sbom_format"
357+
exit 1
358+
;;
359+
esac
360+
361+
# Check if we have data to insert
362+
if [[ ! -s "$data_file" ]]; then
363+
log_warning "No component data found in SBOM"
364+
return
365+
fi
366+
367+
local component_count=$(wc -l < "$data_file")
368+
log_info "Found $component_count components to insert"
369+
370+
# Insert data into ClickHouse
371+
if curl -s ${auth_params} \
372+
-H "Content-Type: text/tab-separated-values" \
373+
--data-binary "@$data_file" \
374+
"${clickhouse_url}/?query=INSERT%20INTO%20${CLICKHOUSE_DATABASE}.${table_name}%20(name,%20version,%20license)%20FORMAT%20TSV"; then
375+
log_success "Inserted $component_count components into ClickHouse table $table_name"
376+
else
377+
log_error "Failed to insert data into ClickHouse"
378+
exit 1
379+
fi
380+
}
381+
254382
# Global variable for temp directory (so cleanup can access it)
255383
temp_dir=""
256384

@@ -317,6 +445,13 @@ main() {
317445

318446
log_success "SBOM processing completed successfully!"
319447
log_info "SBOM available at: s3://$S3_BUCKET/$s3_key"
448+
449+
if [[ -n "${CLICKHOUSE_URL:-}" ]]; then
450+
local table_name=$(echo "$REPOSITORY" | sed 's|[^a-zA-Z0-9]|_|g' | tr '[:upper:]' '[:lower:]')
451+
setup_clickhouse_table "$table_name"
452+
insert_sbom_data "$processed_sbom" "$table_name" "$desired_format"
453+
log_info "Component data available in ClickHouse table: ${CLICKHOUSE_DATABASE}.${table_name}"
454+
fi
320455
}
321456

322457
# Run main function

0 commit comments

Comments
 (0)