cleanup unnecessary data

Author: losfair

cmd/ctmon-ingest/main.go

@@ -411,25 +411,24 @@ func ingestBatch(db *sql.DB, batch []*CertificateDetails) error {
 
 	query := `
 		INSERT INTO ct_log_entries (
-			log_id, log_index, retrieval_timestamp, leaf_input, extra_data,
+			log_id, log_index, retrieval_timestamp, leaf_input,
 			entry_timestamp, entry_type, certificate_sha256, tbs_certificate_sha256,
 			not_before, not_after, subject_common_name, subject_organization, 
 			subject_alternative_names, issuer_common_name, issuer_organization,
-			serial_number, is_ca, precert_issuer_key_hash, raw_leaf_certificate_der
+			serial_number, is_ca, precert_issuer_key_hash 
 		) VALUES
 	`
 
 	var values []string
 	var args []interface{}
 
 	for _, details := range batch {
-		values = append(values, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+		values = append(values, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 		args = append(args,
 			details.LogID,
 			details.LogIndex,
 			details.RetrievalTimestamp,
 			details.LeafInputBase64,
-			details.ExtraDataBase64,
 			details.EntryTimestamp,
 			details.EntryType,
 			details.CertificateSHA256,
@@ -444,7 +443,6 @@ func ingestBatch(db *sql.DB, batch []*CertificateDetails) error {
 			details.SerialNumber,
 			boolToUint8(details.IsCA),
 			nullableString(details.PrecertIssuerKeyHash),
-			details.RawLeafCertificateDERBase64,
 		)
 	}
 

schema.sql

@@ -6,8 +6,8 @@ CREATE TABLE ct_log_entries
     retrieval_timestamp DateTime DEFAULT now() COMMENT 'Timestamp when the entry was fetched and ingested into ClickHouse',
 
     -- Raw CT Log Data (as returned by the get-entries endpoint)
-    leaf_input String COMMENT 'Base64 encoded MerkleTreeLeaf structure from the log entry',
-    extra_data String COMMENT 'Base64 encoded extra data from the log entry (e.g., certificate chain)',
+    leaf_input String COMMENT 'Base64 encoded MerkleTreeLeaf structure from the log entry' CODEC(ZSTD(1)),
+    extra_data String DEFAULT '' COMMENT 'Base64 encoded extra data from the log entry (e.g., certificate chain)',
 
     -- Parsed from MerkleTreeLeaf -> TimestampedEntry
     entry_timestamp DateTime COMMENT 'Timestamp from the TimestampedEntry (milliseconds since epoch, converted to DateTime)',
@@ -64,7 +64,7 @@ CREATE TABLE ct_log_entries
     precert_poison_extension_present UInt8 DEFAULT 0 COMMENT 'Boolean (0 or 1) indicating if the X.509v3 Precertificate Poison extension is present',
 
     -- Stored Raw Certificate (optional, for full reprocessing capability)
-    raw_leaf_certificate_der String COMMENT 'Base64 encoded DER of the leaf certificate itself (extracted from leaf_input)',
+    raw_leaf_certificate_der String DEFAULT '' COMMENT 'Base64 encoded DER of the leaf certificate itself (extracted from leaf_input)',
 
     -- PROJECTION for SAN lookups (optional, for performance)
     -- PROJECTION san_projection (

ui/app/api/certificate/[sha256]/route.ts

@@ -61,6 +61,7 @@ export async function GET(
       WHERE certificate_sha256 = {sha256:String}
       AND entry_type = 'x509_entry'
       ORDER BY not_after DESC
+      SETTINGS max_execution_time = 30, max_threads = 1, max_memory_usage = 134217728
     `;
 
     const resultSet = await client.query({

ui/app/search/[domain]/page.tsx

@@ -75,7 +75,7 @@ async function getSearchResults(
       AND entry_type = 'x509_entry'
       ORDER BY not_after DESC 
       LIMIT {limit:UInt32}
-      SETTINGS max_execution_time = 30, max_memory_usage = 134217728
+      SETTINGS max_execution_time = 30, max_threads = 1, max_memory_usage = 134217728
     `;
 
     const resultSet = await client.query({