Skip to content

Commit 1d3468c

Browse files
dvasilasdvasilas
committed
WKBCH-15: Update Clickhouse schema
1 parent 2f07e8d commit 1d3468c

File tree

4 files changed

+63
-78
lines changed

4 files changed

+63
-78
lines changed

‎templates/clickhouse/init.d/02-create-ingest-table.sql‎

Lines changed: 30 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,43 @@
11
CREATE TABLE IF NOT EXISTS logs.access_logs_ingest
22
(
33
-- Common
4-
timestamp DateTime,
54
insertedAt DateTime DEFAULT now(),
6-
hostname LowCardinality(String),
7-
8-
-- Analytics
9-
action LowCardinality(String),
10-
accountName String,
11-
userName String,
12-
clientPort UInt32,
13-
httpMethod LowCardinality(String),
14-
bytesDeleted UInt64,
15-
bytesReceived UInt64,
16-
bodyLength UInt64,
17-
contentLength UInt64,
18-
elapsed_ms Float32,
5+
hostname LowCardinality(Nullable(String)),
196

207
-- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
218
startTime DateTime64(3), -- AWS "Time" field
22-
requester String,
23-
operation String,
24-
requestURI String,
25-
errorCode String,
26-
objectSize UInt64,
27-
totalTime Float32,
28-
turnAroundTime Float32,
29-
referer String,
30-
userAgent String,
31-
versionId String,
32-
signatureVersion LowCardinality(String),
33-
cipherSuite LowCardinality(String),
34-
authenticationType LowCardinality(String),
35-
hostHeader String,
36-
tlsVersion LowCardinality(String),
37-
aclRequired LowCardinality(String),
9+
requester Nullable(String),
10+
operation Nullable(String),
11+
requestURI Nullable(String),
12+
errorCode Nullable(String),
13+
objectSize Nullable(UInt64),
14+
totalTime Nullable(Float32),
15+
turnAroundTime Nullable(Float32),
16+
referer Nullable(String),
17+
userAgent Nullable(String),
18+
versionId Nullable(String),
19+
signatureVersion LowCardinality(Nullable(String)),
20+
cipherSuite LowCardinality(Nullable(String)),
21+
authenticationType LowCardinality(Nullable(String)),
22+
hostHeader Nullable(String),
23+
tlsVersion LowCardinality(Nullable(String)),
24+
aclRequired LowCardinality(Nullable(String)),
3825

3926
-- Shared between AWS access server logs and Analytics logs
40-
bucketOwner String, -- AWS "Bucket Owner" field
41-
bucketName String, -- AWS "Bucket" field
42-
req_id String, -- AWS "Request ID" field
43-
bytesSent UInt64, -- AWS "Bytes Sent" field
44-
clientIP String, -- AWS "Remote IP" field
45-
httpCode UInt16, -- AWS "HTTP Status" field
46-
objectKey String, -- AWS "Key" field
27+
bucketOwner Nullable(String), -- AWS "Bucket Owner" field
28+
bucketName String DEFAULT '', -- AWS "Bucket" field
29+
req_id String DEFAULT '', -- AWS "Request ID" field
30+
bytesSent Nullable(UInt64), -- AWS "Bytes Sent" field
31+
clientIP Nullable(String), -- AWS "Remote IP" field
32+
httpCode Nullable(UInt16), -- AWS "HTTP Status" field
33+
objectKey Nullable(String), -- AWS "Key" field
4734

4835
-- Scality server access logs extra fields.
49-
logFormatVersion LowCardinality(String),
50-
loggingEnabled Bool,
51-
loggingTargetBucket String,
52-
loggingTargetPrefix String,
53-
awsAccessKeyID String,
54-
raftSessionID UInt16
36+
logFormatVersion LowCardinality(Nullable(String)),
37+
loggingEnabled Bool DEFAULT false,
38+
loggingTargetBucket String DEFAULT '',
39+
loggingTargetPrefix String DEFAULT '',
40+
awsAccessKeyID Nullable(String),
41+
raftSessionID UInt16 DEFAULT 0
5542
)
5643
Engine = Null();

‎templates/clickhouse/init.d/03-create-storage-table.sql‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,45 @@
11
CREATE TABLE IF NOT EXISTS logs.access_logs
22
(
33
-- Common
4-
timestamp DateTime,
54
insertedAt DateTime DEFAULT now(),
6-
hostname LowCardinality(String),
5+
hostname LowCardinality(Nullable(String)),
76

87
-- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
98
startTime DateTime64(3), -- AWS "Time" field
10-
requester String,
11-
operation String,
12-
requestURI String,
13-
errorCode String,
14-
objectSize UInt64,
15-
totalTime Float32,
16-
turnAroundTime Float32,
17-
referer String,
18-
userAgent String,
19-
versionId String,
20-
signatureVersion LowCardinality(String),
21-
cipherSuite LowCardinality(String),
22-
authenticationType LowCardinality(String),
23-
hostHeader String,
24-
tlsVersion LowCardinality(String),
25-
aclRequired LowCardinality(String),
9+
requester Nullable(String),
10+
operation Nullable(String),
11+
requestURI Nullable(String),
12+
errorCode Nullable(String),
13+
objectSize Nullable(UInt64),
14+
totalTime Nullable(Float32),
15+
turnAroundTime Nullable(Float32),
16+
referer Nullable(String),
17+
userAgent Nullable(String),
18+
versionId Nullable(String),
19+
signatureVersion LowCardinality(Nullable(String)),
20+
cipherSuite LowCardinality(Nullable(String)),
21+
authenticationType LowCardinality(Nullable(String)),
22+
hostHeader Nullable(String),
23+
tlsVersion LowCardinality(Nullable(String)),
24+
aclRequired LowCardinality(Nullable(String)),
2625

2726
-- Shared between AWS access server logs and Analytics logs
28-
bucketOwner String, -- AWS "Bucket Owner" field
29-
bucketName String, -- AWS "Bucket" field
30-
req_id String, -- AWS "Request ID" field
31-
bytesSent UInt64, -- AWS "Bytes Sent" field
32-
clientIP String, -- AWS "Remote IP" field
33-
httpCode UInt16, -- AWS "HTTP Status" field
34-
objectKey String, -- AWS "Key" field
27+
bucketOwner Nullable(String), -- AWS "Bucket Owner" field
28+
bucketName String DEFAULT '', -- AWS "Bucket" field
29+
req_id String DEFAULT '', -- AWS "Request ID" field
30+
bytesSent Nullable(UInt64), -- AWS "Bytes Sent" field
31+
clientIP Nullable(String), -- AWS "Remote IP" field
32+
httpCode Nullable(UInt16), -- AWS "HTTP Status" field
33+
objectKey Nullable(String), -- AWS "Key" field
3534

3635
-- Scality server access logs extra fields.
37-
logFormatVersion LowCardinality(String),
38-
loggingEnabled Bool,
39-
loggingTargetBucket String,
40-
loggingTargetPrefix String,
41-
awsAccessKeyID String,
42-
raftSessionID UInt16
36+
logFormatVersion LowCardinality(Nullable(String)),
37+
loggingEnabled Bool DEFAULT false,
38+
loggingTargetBucket String DEFAULT '',
39+
loggingTargetPrefix String DEFAULT '',
40+
awsAccessKeyID Nullable(String),
41+
raftSessionID UInt16 DEFAULT 0
4342
)
4443
Engine = MergeTree()
45-
PARTITION BY toStartOfInterval(insertedAt, INTERVAL 24 HOUR)
46-
ORDER BY (raftSessionID, bucketName, insertedAt, timestamp, req_id);
44+
PARTITION BY toStartOfInterval(insertedAt, INTERVAL 1 HOUR)
45+
ORDER BY (raftSessionID, bucketName, insertedAt, startTime, req_id);

‎templates/clickhouse/init.d/04-create-offsets-table.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS logs.offsets
33
bucketName String,
44
raftSessionID UInt16,
55
lastProcessedInsertedAt DateTime,
6-
lastProcessedTimestamp DateTime64(3),
6+
lastProcessedStartTime DateTime64(3),
77
lastProcessedReqId String
88
)
99
ENGINE = ReplacingMergeTree(lastProcessedInsertedAt)

‎templates/clickhouse/init.d/06-create-materialized-view.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS logs.access_logs_ingest_mv
22
TO logs.access_logs_federated
33
AS
44
SELECT
5-
timestamp,
65
insertedAt,
76
hostname,
87

0 commit comments

Comments
 (0)