Skip to content

Commit 033edc5

Browse files
committed
Update to latest Timely
Point Materialize at latest Timely. We need to incorporate some changes around reachability logging, which is now typed, and event iterators that return cow'ed data. Some of the complexity stems from the fact that event links are single-writer, so we need separate event links for each reachability log variant. Signed-off-by: Moritz Hoffmann <mh@materialize.com>
1 parent e362cc9 commit 033edc5

File tree

41 files changed

+203
-163
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+203
-163
lines changed

Cargo.lock

Lines changed: 6 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/adapter-types/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ mz-ore = { path = "../ore" }
1515
mz-repr = { path = "../repr" }
1616
mz-storage-types = { path = "../storage-types" }
1717
serde = "1.0.152"
18-
timely = "0.16.0"
18+
timely = "0.17.0"
1919
workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }
2020

2121
[package.metadata.cargo-udeps.ignore]

src/adapter/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ bytesize = "1.1.0"
1818
chrono = { version = "0.4.35", default-features = false, features = ["std"] }
1919
dec = "0.4.8"
2020
derivative = "2.2.0"
21-
differential-dataflow = "0.13.3"
21+
differential-dataflow = "0.13.4"
2222
enum-kinds = "0.5.1"
2323
fail = { version = "0.5.1", features = ["failpoints"] }
2424
futures = "0.3.25"
@@ -78,7 +78,7 @@ serde_plain = "1.0.1"
7878
sha2 = "0.10.6"
7979
smallvec = { version = "1.10.0", features = ["union"] }
8080
static_assertions = "1.1"
81-
timely = "0.16.0"
81+
timely = "0.17.0"
8282
tokio = { version = "1.38.0", features = ["rt", "time"] }
8383
tokio-postgres = { version = "0.7.8" }
8484
tracing = "0.1.37"

src/catalog/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ bytesize = "1.1.0"
1818
chrono = { version = "0.4.35", default-features = false, features = ["std"] }
1919
clap = { version = "4.5.23", features = ["derive"] }
2020
derivative = "2.2.0"
21-
differential-dataflow = "0.13.3"
21+
differential-dataflow = "0.13.4"
2222
futures = "0.3.25"
2323
ipnet = "2.5.0"
2424
itertools = "0.12.1"
@@ -60,7 +60,7 @@ serde_plain = "1.0.1"
6060
static_assertions = "1.1"
6161
sha2 = "0.10.6"
6262
thiserror = "1.0.37"
63-
timely = "0.16.0"
63+
timely = "0.17.0"
6464
tokio = { version = "1.38.0" }
6565
tracing = "0.1.37"
6666
uuid = "1.2.2"

src/catalog/src/builtin.rs

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5567,15 +5567,22 @@ pub static MZ_DATAFLOW_OPERATOR_REACHABILITY_PER_WORKER: LazyLock<BuiltinView> =
55675567
oid: oid::VIEW_MZ_DATAFLOW_OPERATOR_REACHABILITY_PER_WORKER_OID,
55685568
column_defs: None,
55695569
sql: "SELECT
5570-
address,
5570+
addr2.id,
5571+
reachability.worker_id,
55715572
port,
5572-
worker_id,
55735573
update_type,
55745574
time,
55755575
pg_catalog.count(*) as count
55765576
FROM
5577-
mz_introspection.mz_dataflow_operator_reachability_raw
5578-
GROUP BY address, port, worker_id, update_type, time",
5577+
mz_introspection.mz_dataflow_operator_reachability_raw reachability,
5578+
mz_introspection.mz_dataflow_addresses_per_worker addr1,
5579+
mz_introspection.mz_dataflow_addresses_per_worker addr2
5580+
WHERE
5581+
addr2.address = addr1.address || reachability.source
5582+
AND addr1.id = reachability.id
5583+
AND addr1.worker_id = reachability.worker_id
5584+
AND addr2.worker_id = reachability.worker_id
5585+
GROUP BY addr2.id, reachability.worker_id, port, update_type, time",
55795586
access: vec![PUBLIC_SELECT],
55805587
});
55815588

@@ -5587,13 +5594,13 @@ pub static MZ_DATAFLOW_OPERATOR_REACHABILITY: LazyLock<BuiltinView> =
55875594
column_defs: None,
55885595
sql: "
55895596
SELECT
5590-
address,
5597+
id,
55915598
port,
55925599
update_type,
55935600
time,
55945601
pg_catalog.sum(count) as count
55955602
FROM mz_introspection.mz_dataflow_operator_reachability_per_worker
5596-
GROUP BY address, port, update_type, time",
5603+
GROUP BY id, port, update_type, time",
55975604
access: vec![PUBLIC_SELECT],
55985605
});
55995606

src/cluster/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ workspace = true
1313
anyhow = "1.0.66"
1414
async-trait = "0.1.83"
1515
crossbeam-channel = "0.5.8"
16-
differential-dataflow = "0.13.3"
16+
differential-dataflow = "0.13.4"
1717
futures = "0.3.25"
1818
mz-cluster-client = { path = "../cluster-client" }
1919
mz-ore = { path = "../ore", features = ["async", "process", "tracing_"] }
2020
mz-persist-client = { path = "../persist-client" }
2121
mz-service = { path = "../service" }
2222
mz-txn-wal = { path = "../txn-wal" }
2323
regex = "1.7.0"
24-
timely = "0.16.0"
24+
timely = "0.17.0"
2525
tokio = { version = "1.38.0", features = ["fs", "rt", "sync", "net"] }
2626
tracing = "0.1.37"
2727
workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }

src/compute-client/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ async-trait = "0.1.83"
1515
bytesize = "1.1.0"
1616
crossbeam-channel = "0.5.8"
1717
derivative = "2.2.0"
18-
differential-dataflow = "0.13.3"
18+
differential-dataflow = "0.13.4"
1919
futures = "0.3.25"
2020
http = "1.1.0"
2121
mz-build-info = { path = "../build-info" }
@@ -43,7 +43,7 @@ prost = { version = "0.13.2", features = ["no-recursion-limit"] }
4343
serde = { version = "1.0.152", features = ["derive"] }
4444
serde_json = "1.0.125"
4545
thiserror = "1.0.37"
46-
timely = "0.16.0"
46+
timely = "0.17.0"
4747
tokio = "1.38.0"
4848
tokio-stream = "0.1.11"
4949
tonic = "0.12.1"

src/compute-client/src/logging.rs

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -448,16 +448,10 @@ impl LogVariant {
448448
.finish(),
449449

450450
LogVariant::Timely(TimelyLog::Reachability) => RelationDesc::builder()
451-
.with_column(
452-
"address",
453-
ScalarType::List {
454-
element_type: Box::new(ScalarType::UInt64),
455-
custom_id: None,
456-
}
457-
.nullable(false),
458-
)
459-
.with_column("port", ScalarType::UInt64.nullable(false))
451+
.with_column("id", ScalarType::UInt64.nullable(false))
460452
.with_column("worker_id", ScalarType::UInt64.nullable(false))
453+
.with_column("source", ScalarType::UInt64.nullable(false))
454+
.with_column("port", ScalarType::UInt64.nullable(false))
461455
.with_column("update_type", ScalarType::String.nullable(false))
462456
.with_column("time", ScalarType::MzTimestamp.nullable(true))
463457
.finish(),

src/compute-types/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ workspace = true
1212
[dependencies]
1313
columnar = "0.2.2"
1414
columnation = "0.1.0"
15-
differential-dataflow = "0.13.3"
15+
differential-dataflow = "0.13.4"
1616
itertools = "0.12.1"
1717
mz-dyncfg = { path = "../dyncfg" }
1818
mz-expr = { path = "../expr" }
@@ -24,7 +24,7 @@ proptest = { version = "1.6.0", default-features = false, features = ["std"] }
2424
proptest-derive = { version = "0.5.1", features = ["boxed_union"] }
2525
prost = { version = "0.13.2", features = ["no-recursion-limit"] }
2626
serde = { version = "1.0.152", features = ["derive"] }
27-
timely = "0.16.0"
27+
timely = "0.17.0"
2828
tracing = "0.1.37"
2929
workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }
3030

src/compute/Cargo.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ bytesize = "1.1.0"
1616
columnar = "0.2.2"
1717
crossbeam-channel = "0.5.8"
1818
dec = { version = "0.4.8", features = ["serde"] }
19-
differential-dataflow = "0.13.3"
20-
differential-dogs3 = "0.1.3"
19+
differential-dataflow = "0.13.4"
20+
differential-dogs3 = "0.1.4"
2121
futures = "0.3.25"
2222
itertools = "0.12.1"
2323
lgalloc = "0.4"
@@ -39,7 +39,7 @@ prometheus = { version = "0.13.3", default-features = false }
3939
scopeguard = "1.1.0"
4040
serde = { version = "1.0.152", features = ["derive"] }
4141
smallvec = { version = "1.10.0", features = ["serde", "union"] }
42-
timely = "0.16.0"
42+
timely = "0.17.0"
4343
tokio = { version = "1.38.0", features = ["fs", "rt", "sync", "net"] }
4444
tracing = "0.1.37"
4545
uuid = { version = "1.7.0", features = ["serde", "v4"] }

0 commit comments

Comments
 (0)