Skip to content

Commit 0dc6208

Browse files
authored
Implement flexible event ingestion and querying (#268)
This PR adds the capability to ingest events with varying schemas. During ingestion, events with the same schema are stored in separate files which are then merged into a single Parquet file. This allows for schema to dynamically change over time without having any impact on performance. Fixes #195
1 parent 89807da commit 0dc6208

23 files changed

+1247
-767
lines changed

server/Cargo.toml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ aws-sdk-s3 = "0.19"
1717
aws-smithy-async = { version = "0.49.0", features = ["rt-tokio"] }
1818
base64 = "0.20.0"
1919
bytes = "1"
20-
chrono = "0.4.19"
20+
chrono = "0.4.23"
2121
chrono-humanize = "0.2.2"
2222
clap = { version = "4.0.32", default-features = false, features = [
2323
"std",
@@ -49,7 +49,7 @@ rustls = "0.20.6"
4949
rustls-pemfile = "1.0.1"
5050
rust-flatten-json = "0.2.0"
5151
semver = "1.0.14"
52-
serde = "^1.0.8"
52+
serde = { version = "1.0", features = ["rc"] }
5353
serde_derive = "^1.0.8"
5454
serde_json = "^1.0.8"
5555
thiserror = "1"
@@ -65,6 +65,9 @@ actix-web-static-files = "4.0"
6565
static-files = "0.2.1"
6666
ulid = { version = "1.0", features = ["serde"] }
6767
ureq = { version = "2.5.0", features = ["json"] }
68+
hex = "0.4.3"
69+
itertools = "0.10.5"
70+
xxhash-rust = { version = "0.8.6", features = ["xxh3"] }
6871

6972
[build-dependencies]
7073
static-files = "0.2.1"
@@ -77,7 +80,6 @@ zip = { version = "0.6.3", default_features = false, features = ["deflate"] }
7780
[dev-dependencies]
7881
maplit = "1.0.2"
7982
rstest = "0.15.0"
80-
serial_test = { version = "0.9.0", default-features = false }
8183

8284
[package.metadata.parseable_ui]
8385
assets-url = "https://github.com/parseablehq/console/releases/download/v0.0.8/build.zip"

0 commit comments

Comments
 (0)