diff --git a/Cargo.lock b/Cargo.lock index b5761f4f33..a413d367ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -335,6 +335,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.21" @@ -434,10 +440,10 @@ dependencies = [ ] [[package]] -name = "arcshift" -version = "0.4.2" +name = "archery" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809ce7e2c99d17c161c0ca1673665bab3a3e1aa277b0f793abffc66b57c32d9f" +checksum = "70e0a5f99dfebb87bb342d0f53bb92c81842e100bbb915223e38349580e5441d" [[package]] name = "argon2" @@ -1184,6 +1190,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitmaps" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" + [[package]] name = "bitvec" version = "1.0.1" @@ -1531,6 +1543,12 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.51" @@ -1603,6 +1621,33 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2122,6 +2167,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -2979,18 +3060,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enum_dispatch" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "env_filter" version = "0.1.4" @@ -4895,6 +4964,29 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "imbl" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fade8ae6828627ad1fa094a891eccfb25150b383047190a3648d66d06186501" +dependencies = [ + "archery", + "bitmaps", + "imbl-sized-chunks", + "rand_core 0.9.3", + "rand_xoshiro", + "version_check", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4241005618a62f8d57b2febd02510fb96e0137304728543dfc5fd6f052c22d" +dependencies = [ + "bitmaps", +] + [[package]] name = "impl-more" version = "0.1.9" @@ -5078,12 +5170,32 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -6153,6 +6265,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -6740,6 +6858,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polling" version = "3.11.0" @@ -8256,7 +8402,7 @@ version = "0.6.1-edge.3" dependencies = [ "ahash 0.8.12", "anyhow", - "arcshift", + "arc-swap", "argon2", "async-channel", "async_zip", @@ -8270,13 +8416,13 @@ dependencies = [ "compio-quic", "compio-tls", "compio-ws", + "criterion", "ctrlc", "cyper", "cyper-axum", "dashmap", "derive_more", "dotenvy", - "enum_dispatch", "err_trail", "error_set", "figlet-rs", @@ -8287,6 +8433,7 @@ dependencies = [ "human-repr", "hwlocality", "iggy_common", + "imbl", "jsonwebtoken", "lending-iterator", "mimalloc", @@ -8310,7 +8457,6 @@ dependencies = [ "send_wrapper", "serde", "serde_with", - "slab", "socket2 0.6.1", "static-toml", "strum 0.27.2", @@ -9225,6 +9371,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index 3b68471ce5..89a1195552 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -121,7 +121,6 @@ derive_more = { version = "2.1.1", features = ["full"] } dirs = "6.0.0" dlopen2 = "0.8.2" dotenvy = "0.15.7" -enum_dispatch = "0.3.13" env_logger = "0.11.8" err_trail = { version = "0.11.0", features = ["tracing"] } error_set = "0.9.1" diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 8423e83d3e..145f83c85f 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -24,6 +24,7 @@ alloc-no-stdlib: 2.0.4, "BSD-3-Clause", alloc-stdlib: 0.2.2, "BSD-3-Clause", allocator-api2: 0.2.21, "Apache-2.0 OR MIT", android_system_properties: 0.1.5, "Apache-2.0 OR MIT", +anes: 0.1.6, "Apache-2.0 OR MIT", anstream: 0.6.21, "Apache-2.0 OR MIT", anstyle: 1.0.13, "Apache-2.0 OR MIT", anstyle-parse: 0.2.7, "Apache-2.0 OR MIT", @@ -33,7 +34,7 @@ anyhow: 1.0.100, "Apache-2.0 OR MIT", apache-avro: 0.17.0, "Apache-2.0", arbitrary: 1.4.2, "Apache-2.0 OR MIT", arc-swap: 1.8.0, "Apache-2.0 OR MIT", -arcshift: 0.4.2, "Apache-2.0 OR MIT", +archery: 1.2.2, "MIT", argon2: 0.5.3, "Apache-2.0 OR MIT", array-init: 2.1.0, "Apache-2.0 OR MIT", arrayref: 0.3.9, "BSD-2-Clause", @@ -98,6 +99,7 @@ bit-set: 0.8.0, "Apache-2.0 OR MIT", bit-vec: 0.8.0, "Apache-2.0 OR MIT", bitflags: 1.3.2, "Apache-2.0 OR MIT", bitflags: 2.10.0, "Apache-2.0 OR MIT", +bitmaps: 3.2.1, "MPL-2.0+", bitvec: 1.0.1, "MIT", blake2: 0.10.6, "Apache-2.0 OR MIT", blake3: 1.8.2, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0", @@ -128,6 +130,7 @@ cargo-platform: 0.1.9, "Apache-2.0 OR MIT", cargo-platform: 0.3.2, "Apache-2.0 OR MIT", cargo_metadata: 0.19.2, "MIT", cargo_metadata: 0.23.1, "MIT", +cast: 0.3.0, "Apache-2.0 OR MIT", cc: 1.2.51, "Apache-2.0 OR MIT", cesu8: 1.1.0, "Apache-2.0 OR MIT", cfg-if: 1.0.4, "Apache-2.0 OR MIT", @@ -135,6 +138,9 @@ cfg_aliases: 0.2.1, "MIT", charming: 0.6.0, "Apache-2.0 OR MIT", charming_macros: 0.1.0, "Apache-2.0 OR MIT", chrono: 0.4.42, "Apache-2.0 OR MIT", +ciborium: 0.2.2, "Apache-2.0", +ciborium-io: 0.2.2, "Apache-2.0", +ciborium-ll: 0.2.2, "Apache-2.0", cipher: 0.4.4, "Apache-2.0 OR MIT", clap: 4.5.53, "Apache-2.0 OR MIT", clap_builder: 4.5.53, "Apache-2.0 OR MIT", @@ -182,6 +188,8 @@ crc: 3.3.0, "Apache-2.0 OR MIT", crc-catalog: 2.4.0, "Apache-2.0 OR MIT", crc32c: 0.6.8, "Apache-2.0 OR MIT", crc32fast: 1.5.0, "Apache-2.0 OR MIT", +criterion: 0.5.1, "Apache-2.0 OR MIT", +criterion-plot: 0.5.0, "Apache-2.0 OR MIT", critical-section: 1.2.0, "Apache-2.0 OR MIT", crossbeam: 0.8.4, "Apache-2.0 OR MIT", crossbeam-channel: 0.5.15, "Apache-2.0 OR MIT", @@ -261,7 +269,6 @@ embedded-io: 0.4.0, "Apache-2.0 OR MIT", embedded-io: 0.6.1, "Apache-2.0 OR MIT", encode_unicode: 1.0.0, "Apache-2.0 OR MIT", encoding_rs: 0.8.35, "(Apache-2.0 OR MIT) AND BSD-3-Clause", -enum_dispatch: 0.3.13, "Apache-2.0 OR MIT", env_filter: 0.1.4, "Apache-2.0 OR MIT", env_logger: 0.11.8, "Apache-2.0 OR MIT", equivalent: 1.0.2, "Apache-2.0 OR MIT", @@ -410,6 +417,8 @@ iggy_connector_sdk: 0.1.1-edge.1, "Apache-2.0", iggy_connector_stdout_sink: 0.1.0, "Apache-2.0", iggy_examples: 0.0.5, "Apache-2.0", ignore: 0.4.25, "MIT OR Unlicense", +imbl: 6.1.0, "MPL-2.0+", +imbl-sized-chunks: 0.1.3, "MPL-2.0+", impl-more: 0.1.9, "Apache-2.0 OR MIT", implicit-clone: 0.6.0, "Apache-2.0 OR MIT", implicit-clone-derive: 0.1.2, "Apache-2.0 OR MIT", @@ -427,7 +436,9 @@ io-uring: 0.7.11, "Apache-2.0 OR MIT", io_uring_buf_ring: 0.2.3, "MIT", ipnet: 2.11.0, "Apache-2.0 OR MIT", iri-string: 0.7.10, "Apache-2.0 OR MIT", +is-terminal: 0.4.17, "MIT", is_terminal_polyfill: 1.70.2, "Apache-2.0 OR MIT", +itertools: 0.10.5, "Apache-2.0 OR MIT", itertools: 0.13.0, "Apache-2.0 OR MIT", itertools: 0.14.0, "Apache-2.0 OR MIT", itoa: 1.0.17, "Apache-2.0 OR MIT", @@ -537,6 +548,7 @@ octocrab: 0.49.4, "Apache-2.0 OR MIT", oid-registry: 0.8.1, "Apache-2.0 OR MIT", once_cell: 1.21.3, "Apache-2.0 OR MIT", once_cell_polyfill: 1.70.2, "Apache-2.0 OR MIT", +oorandom: 11.1.5, "MIT", opaque-debug: 0.3.1, "Apache-2.0 OR MIT", opendal: 0.54.1, "Apache-2.0", openssl: 0.10.75, "Apache-2.0", @@ -591,6 +603,9 @@ pinned: 0.1.0, "Apache-2.0 OR MIT", pkcs1: 0.7.5, "Apache-2.0 OR MIT", pkcs8: 0.10.2, "Apache-2.0 OR MIT", pkg-config: 0.3.32, "Apache-2.0 OR MIT", +plotters: 0.3.7, "MIT", +plotters-backend: 0.3.7, "MIT", +plotters-svg: 0.3.7, "MIT", polling: 3.11.0, "Apache-2.0 OR MIT", polonius-the-crab: 0.2.1, "Apache-2.0 OR MIT OR Zlib", polyval: 0.6.2, "Apache-2.0 OR MIT", @@ -803,6 +818,7 @@ time-core: 0.1.6, "Apache-2.0 OR MIT", time-macros: 0.2.24, "Apache-2.0 OR MIT", tiny-keccak: 2.0.2, "CC0-1.0", tinystr: 0.8.2, "Unicode-3.0", +tinytemplate: 1.2.1, "Apache-2.0 OR MIT", tinyvec: 1.10.0, "Apache-2.0 OR MIT OR Zlib", tinyvec_macros: 0.1.1, "Apache-2.0 OR MIT OR Zlib", tokio: 1.48.0, "MIT", diff --git a/bdd/go/tests/tcp_test/offset_feature_deserialize.go b/bdd/go/tests/tcp_test/offset_feature_deserialize.go index abec1494ea..d37808eb96 100644 --- a/bdd/go/tests/tcp_test/offset_feature_deserialize.go +++ b/bdd/go/tests/tcp_test/offset_feature_deserialize.go @@ -183,15 +183,14 @@ var _ = ginkgo.Describe("GET CONSUMER OFFSET:", func() { consumer := iggcon.NewGroupConsumer(randomU32Identifier()) partitionId := uint32(1) - offset, err := client.GetConsumerOffset( + _, err := client.GetConsumerOffset( consumer, randomU32Identifier(), randomU32Identifier(), &partitionId, ) - itShouldNotReturnError(err) - itShouldReturnNilOffsetForNewConsumerGroup(offset) + itShouldReturnUnauthenticatedError(err) }) }) }) diff --git a/core/common/src/collections/segmented_slab.rs b/core/common/src/collections/segmented_slab.rs index fc75239a4f..4bbac82810 100644 --- a/core/common/src/collections/segmented_slab.rs +++ b/core/common/src/collections/segmented_slab.rs @@ -247,6 +247,14 @@ impl SegmentedSlab (self, true) } + /// Set value at key, returning new slab. Ignores success/failure. + /// + /// Convenience method for RCU patterns where the key is known to exist. + #[inline] + pub fn set(self, key: usize, value: T) -> Self { + self.update(key, value).0 + } + /// Remove entry at key, returning (new_slab, removed_value). /// /// Freed slot will be reused by future inserts. @@ -334,6 +342,16 @@ mod tests { assert!(!success); } + #[test] + fn test_set() { + let slab = TestSlab::new(); + let (slab, key) = slab.insert("original"); + + let slab = slab.set(key, "updated"); + assert_eq!(slab.get(key), Some(&"updated")); + assert_eq!(slab.len(), 1); + } + #[test] fn test_remove_and_reuse() { let slab = TestSlab::new(); diff --git a/core/common/src/types/personal_access_tokens/mod.rs b/core/common/src/types/personal_access_tokens/mod.rs index e28a585de5..0e1e333919 100644 --- a/core/common/src/types/personal_access_tokens/mod.rs +++ b/core/common/src/types/personal_access_tokens/mod.rs @@ -28,8 +28,8 @@ const SIZE: usize = 50; #[derive(Clone, Debug)] pub struct PersonalAccessToken { pub user_id: UserId, - pub name: Arc, - pub token: Arc, + pub name: Arc, + pub token: Arc, pub expiry_at: Option, } @@ -49,8 +49,8 @@ impl PersonalAccessToken { ( Self { user_id, - name: Arc::new(name.to_string()), - token: Arc::new(token_hash), + name: Arc::from(name), + token: Arc::from(token_hash), expiry_at: Self::calculate_expiry_at(now, expiry), }, token, @@ -65,8 +65,8 @@ impl PersonalAccessToken { ) -> Self { Self { user_id, - name: Arc::new(name.into()), - token: Arc::new(token_hash.into()), + name: Arc::from(name), + token: Arc::from(token_hash), expiry_at, } } @@ -106,12 +106,12 @@ mod tests { let name = "test_token"; let (personal_access_token, raw_token) = PersonalAccessToken::new(user_id, name, now, IggyExpiry::NeverExpire); - assert_eq!(personal_access_token.name.as_str(), name); + assert_eq!(&*personal_access_token.name, name); assert!(!personal_access_token.token.is_empty()); assert!(!raw_token.is_empty()); - assert_ne!(personal_access_token.token.as_str(), raw_token); + assert_ne!(&*personal_access_token.token, raw_token); assert_eq!( - personal_access_token.token.as_str(), + &*personal_access_token.token, PersonalAccessToken::hash_token(&raw_token) ); } diff --git a/core/integration/tests/server/scenarios/concurrent_scenario.rs b/core/integration/tests/server/scenarios/concurrent_scenario.rs index 535d6ac768..2c8767270a 100644 --- a/core/integration/tests/server/scenarios/concurrent_scenario.rs +++ b/core/integration/tests/server/scenarios/concurrent_scenario.rs @@ -29,15 +29,17 @@ const MULTIPLE_CLIENT_COUNT: usize = 10; const OPERATIONS_PER_CLIENT: usize = OPERATIONS_COUNT / MULTIPLE_CLIENT_COUNT; const USER_PASSWORD: &str = "secret"; const TEST_STREAM_NAME: &str = "race-test-stream"; +const TEST_TOPIC_NAME: &str = "race-test-topic"; const PARTITIONS_COUNT: u32 = 1; +const PARTITIONS_TO_ADD: u32 = 1; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ResourceType { User, Stream, Topic, + Partition, // TODO(hubcio): add ConsumerGroup - // TODO(hubcio): add Partition } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -70,6 +72,24 @@ pub async fn run( root_client.create_stream(TEST_STREAM_NAME).await.unwrap(); } + // For partition tests, create parent stream and topic first + if resource_type == ResourceType::Partition { + root_client.create_stream(TEST_STREAM_NAME).await.unwrap(); + let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap(); + root_client + .create_topic( + &stream_id, + TEST_TOPIC_NAME, + PARTITIONS_COUNT, + CompressionAlgorithm::default(), + None, + IggyExpiry::NeverExpire, + MaxTopicSize::ServerDefault, + ) + .await + .unwrap(); + } + let results = match (resource_type, scenario_type) { (ResourceType::User, ScenarioType::Cold) => { execute_multiple_clients_users_cold(client_factory, use_barrier).await @@ -86,6 +106,11 @@ pub async fn run( (ResourceType::Topic, ScenarioType::Cold) => { execute_multiple_clients_topics_cold(client_factory, use_barrier).await } + (ResourceType::Partition, ScenarioType::Hot) => { + execute_multiple_clients_partitions_hot(client_factory, use_barrier).await + } + // Partitions don't have names, so Cold scenario doesn't apply + (ResourceType::Partition, ScenarioType::Cold) => vec![], _ => vec![], // TODO: Figure out why those tests timeout in CI. /* @@ -374,6 +399,51 @@ async fn execute_multiple_clients_topics_cold( .collect() } +async fn execute_multiple_clients_partitions_hot( + client_factory: &dyn ClientFactory, + use_barrier: bool, +) -> Vec { + let mut handles = Vec::with_capacity(MULTIPLE_CLIENT_COUNT); + let barrier = if use_barrier { + Some(Arc::new(Barrier::new(MULTIPLE_CLIENT_COUNT))) + } else { + None + }; + + for _ in 0..MULTIPLE_CLIENT_COUNT { + let client = create_client(client_factory).await; + login_root(&client).await; + + let barrier_clone = barrier.clone(); + let handle = tokio::spawn(async move { + if let Some(b) = barrier_clone { + b.wait().await; + } + + let mut results = Vec::with_capacity(OPERATIONS_PER_CLIENT); + let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap(); + let topic_id = Identifier::named(TEST_TOPIC_NAME).unwrap(); + + for _ in 0..OPERATIONS_PER_CLIENT { + let result = client + .create_partitions(&stream_id, &topic_id, PARTITIONS_TO_ADD) + .await + .map(|_| ()); + results.push(result); + } + results + }); + + handles.push(handle); + } + + let all_results = join_all(handles).await; + all_results + .into_iter() + .flat_map(|r| r.expect("Tokio task panicked")) + .collect() +} + fn validate_results(results: &[OperationResult], scenario_type: ScenarioType) { let success_count = results.iter().filter(|r| r.is_ok()).count(); let error_count = results.iter().filter(|r| r.is_err()).count(); @@ -542,6 +612,22 @@ async fn validate_server_state( ); } } + ResourceType::Partition => { + let mut partition_counts = Vec::new(); + for client in &clients { + let count = validate_partitions_state(client).await; + partition_counts.push(count); + } + + let first_count = partition_counts[0]; + for (i, count) in partition_counts.iter().enumerate().skip(1) { + assert_eq!( + *count, first_count, + "Client {} sees different partition count ({}) than client 0 ({})", + i, count, first_count + ); + } + } } } @@ -735,6 +821,26 @@ async fn validate_topics_state(client: &IggyClient, scenario_type: ScenarioType) test_topics } +async fn validate_partitions_state(client: &IggyClient) -> u32 { + let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap(); + let topic_id = Identifier::named(TEST_TOPIC_NAME).unwrap(); + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("Failed to get test topic") + .expect("Test topic not found"); + + // Expected: initial partition + OPERATIONS_COUNT added partitions + let expected_partitions = PARTITIONS_COUNT + (OPERATIONS_COUNT as u32 * PARTITIONS_TO_ADD); + assert_eq!( + topic.partitions_count, expected_partitions, + "Hot path: Expected {} partitions (1 initial + {} added), but found {}", + expected_partitions, OPERATIONS_COUNT, topic.partitions_count + ); + + topic.partitions_count +} + async fn cleanup_resources(client: &IggyClient, resource_type: ResourceType) { match resource_type { ResourceType::User => { @@ -755,8 +861,8 @@ async fn cleanup_resources(client: &IggyClient, resource_type: ResourceType) { .await; } } - // Just delete test stream - ResourceType::Topic => { + // Just delete test stream (also cleans up topics and partitions) + ResourceType::Topic | ResourceType::Partition => { let _ = client .delete_stream(&Identifier::named(TEST_STREAM_NAME).unwrap()) .await; diff --git a/core/integration/tests/streaming/mod.rs b/core/integration/tests/streaming/mod.rs index 6ad7736478..5c321226ec 100644 --- a/core/integration/tests/streaming/mod.rs +++ b/core/integration/tests/streaming/mod.rs @@ -16,31 +16,380 @@ * under the License. */ -use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize}; +use iggy_common::sharding::IggyNamespace; +use iggy_common::{ + CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, +}; use server::{ configs::system::SystemConfig, - shard::{task_registry::TaskRegistry, transmission::connector::ShardConnector}, - slab::{streams::Streams, traits_ext::EntityMarker}, + metadata::Metadata, + shard::{ + namespace::IggyFullNamespace, + shard_local_partitions::{PartitionData, ShardLocalPartitions}, + system::messages::PollingArgs, + task_registry::TaskRegistry, + transmission::connector::ShardConnector, + }, streaming::{ - self, - partitions::{partition, storage::create_partition_file_hierarchy}, - segments::{Segment, storage::create_segment_storage}, - streams::{storage::create_stream_file_hierarchy, stream}, - topics::{storage::create_topic_file_hierarchy, topic}, + partitions::{ + helpers::create_message_deduplicator, + journal::Journal, + partition::{ConsumerGroupOffsets, ConsumerOffsets}, + storage::create_partition_file_hierarchy, + }, + polling_consumer::PollingConsumer, + segments::{ + IggyMessagesBatchMut, IggyMessagesBatchSet, Segment, storage::create_segment_storage, + }, + streams::storage::create_stream_file_hierarchy, + topics::storage::create_topic_file_hierarchy, + traits::MainOps, }, }; +use std::cell::RefCell; use std::rc::Rc; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; mod common; mod get_by_offset; mod get_by_timestamp; mod snapshot; +/// Test harness that uses SharedMetadata + PartitionDataStore +pub struct TestStreams { + pub shared_metadata: &'static Metadata, + pub partition_store: RefCell, +} + +impl TestStreams { + pub fn new(shared_metadata: &'static Metadata) -> Self { + Self { + shared_metadata, + partition_store: RefCell::new(ShardLocalPartitions::new()), + } + } + + /// Access partition data by Identifier-based stream/topic IDs. + /// Converts Identifiers to numeric IDs using SharedMetadata, then accesses partition_store. + pub fn with_partition_by_id( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: usize, + f: F, + ) -> R + where + F: FnOnce( + ( + &PartitionData, + usize, + usize, + &Arc, + &Arc, + &Arc, + ), + ) -> R, + { + let numeric_stream_id = self + .shared_metadata + .get_stream_id(stream_id) + .expect("Stream must exist"); + let numeric_topic_id = self + .shared_metadata + .get_topic_id(numeric_stream_id, topic_id) + .expect("Topic must exist"); + + let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + let store = self.partition_store.borrow(); + let partition_data = store.get(&namespace).expect("Partition must exist"); + + f(( + partition_data, + numeric_stream_id, + numeric_topic_id, + &partition_data.offset, + &partition_data.consumer_offsets, + &partition_data.consumer_group_offsets, + )) + } +} + +impl MainOps for TestStreams { + type Namespace = IggyFullNamespace; + type PollingArgs = PollingArgs; + type Consumer = PollingConsumer; + type In = IggyMessagesBatchMut; + type Out = ( + server::binary::handlers::messages::poll_messages_handler::IggyPollMetadata, + IggyMessagesBatchSet, + ); + type Error = IggyError; + + async fn append_messages( + &self, + config: &SystemConfig, + _registry: &Rc, + ns: &Self::Namespace, + mut batch: Self::In, + ) -> Result<(), Self::Error> { + // Resolve Identifier to numeric IDs + let numeric_stream_id = self + .shared_metadata + .get_stream_id(ns.stream_id()) + .expect("Stream must exist"); + let numeric_topic_id = self + .shared_metadata + .get_topic_id(numeric_stream_id, ns.topic_id()) + .expect("Topic must exist"); + let partition_id = ns.partition_id(); + + let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + + if batch.count() == 0 { + return Ok(()); + } + + // Get necessary data from partition_store + let (current_offset, current_position, segment_start_offset, message_deduplicator) = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(&namespace) + .expect("partition_store: partition must exist"); + + let current_offset = if partition_data.should_increment_offset { + partition_data.offset.load(Ordering::Relaxed) + 1 + } else { + 0 + }; + + let segment = partition_data.log.active_segment(); + let current_position = segment.current_position; + let segment_start_offset = segment.start_offset; + let message_deduplicator = partition_data.message_deduplicator.clone(); + + ( + current_offset, + current_position, + segment_start_offset, + message_deduplicator, + ) + }; + + // Prepare batch for persistence (outside the borrow) + batch + .prepare_for_persistence( + segment_start_offset, + current_offset, + current_position, + message_deduplicator.as_ref(), + ) + .await; + + // Append to journal + let (journal_messages_count, journal_size) = { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(&namespace) + .expect("partition_store: partition must exist"); + + let segment = partition_data.log.active_segment_mut(); + + if segment.end_offset == 0 { + segment.start_timestamp = batch.first_timestamp().unwrap(); + } + + let batch_messages_size = batch.size(); + let batch_messages_count = batch.count(); + + partition_data + .stats + .increment_size_bytes(batch_messages_size as u64); + partition_data + .stats + .increment_messages_count(batch_messages_count as u64); + + segment.end_timestamp = batch.last_timestamp().unwrap(); + segment.end_offset = batch.last_offset().unwrap(); + + let (journal_messages_count, journal_size) = + partition_data.log.journal_mut().append(batch)?; + + let last_offset = if batch_messages_count == 0 { + current_offset + } else { + current_offset + batch_messages_count as u64 - 1 + }; + + if partition_data.should_increment_offset { + partition_data.offset.store(last_offset, Ordering::Relaxed); + } else { + partition_data.should_increment_offset = true; + partition_data.offset.store(last_offset, Ordering::Relaxed); + } + partition_data.log.active_segment_mut().current_position += batch_messages_size; + + (journal_messages_count, journal_size) + }; + + // Check if journal should be persisted + let unsaved_messages_count_exceeded = + journal_messages_count >= config.partition.messages_required_to_save; + let unsaved_messages_size_exceeded = journal_size + >= config + .partition + .size_of_messages_required_to_save + .as_bytes_u64() as u32; + + let is_full = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(&namespace) + .expect("partition_store: partition must exist"); + partition_data.log.active_segment().is_full() + }; + + // Persist if needed + if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded { + // Commit journal and persist to disk + let batches = { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(&namespace) + .expect("partition_store: partition must exist"); + let batches = partition_data.log.journal_mut().commit(); + partition_data.log.ensure_indexes(); + batches.append_indexes_to(partition_data.log.active_indexes_mut().unwrap()); + batches + }; + + self.persist_messages_to_disk(&namespace, batches).await?; + } + + Ok(()) + } + + async fn poll_messages( + &self, + ns: &Self::Namespace, + consumer: Self::Consumer, + args: Self::PollingArgs, + ) -> Result { + // Resolve Identifier to numeric IDs + let numeric_stream_id = self + .shared_metadata + .get_stream_id(ns.stream_id()) + .expect("Stream must exist"); + let numeric_topic_id = self + .shared_metadata + .get_topic_id(numeric_stream_id, ns.topic_id()) + .expect("Topic must exist"); + let partition_id = ns.partition_id(); + + let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + + // Delegate to the shared partition_ops module + server::streaming::partition_ops::poll_messages( + &self.partition_store, + &namespace, + consumer, + args, + ) + .await + } +} + +impl TestStreams { + /// Persist messages to disk using partition_store's storage. + async fn persist_messages_to_disk( + &self, + namespace: &IggyNamespace, + batches: IggyMessagesBatchSet, + ) -> Result { + let batch_count = batches.count(); + + if batch_count == 0 { + return Ok(0); + } + + // Extract storage from partition_store + let (messages_writer, index_writer, has_segments) = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + if !partition_data.log.has_segments() { + return Ok(0); + } + + let messages_writer = partition_data + .log + .active_storage() + .messages_writer + .as_ref() + .expect("Messages writer not initialized") + .clone(); + let index_writer = partition_data + .log + .active_storage() + .index_writer + .as_ref() + .expect("Index writer not initialized") + .clone(); + (messages_writer, index_writer, true) + }; + + if !has_segments { + return Ok(0); + } + + // Lock and save messages + let guard = messages_writer.lock.lock().await; + let saved = messages_writer.as_ref().save_batch_set(batches).await?; + + // Get unsaved indexes from partition_store + let unsaved_indexes_slice = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + partition_data.log.active_indexes().unwrap().unsaved_slice() + }; + + // Save indexes + index_writer + .as_ref() + .save_indexes(unsaved_indexes_slice) + .await?; + + // Update index and increment segment stats + { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(namespace) + .expect("partition_store: partition must exist"); + + let indexes = partition_data.log.active_indexes_mut().unwrap(); + indexes.mark_saved(); + + let segment = partition_data.log.active_segment_mut(); + segment.size = + iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64()); + } + + drop(guard); + Ok(batch_count) + } +} + struct BootstrapResult { - streams: Streams, + streams: TestStreams, stream_id: Identifier, topic_id: Identifier, partition_id: usize, + #[allow(dead_code)] + namespace: IggyNamespace, task_registry: Rc, } @@ -54,79 +403,108 @@ async fn bootstrap_test_environment( let topic_size = MaxTopicSize::Unlimited; let partitions_count = 1; - let streams = Streams::default(); - // Create stream together with its dirs - let stream = stream::create_and_insert_stream_mem(&streams, stream_name); - create_stream_file_hierarchy(stream.id(), config).await?; - // Create topic together with its dirs - let stream_id = Identifier::numeric(stream.id() as u32).unwrap(); - let parent_stats = streams.with_stream_by_id(&stream_id, |(_, stats)| stats.clone()); + // Create SharedMetadata (leaked for 'static lifetime in tests) + let shared_metadata: &'static Metadata = Box::leak(Box::new(Metadata::default())); + + // Create stream in SharedMetadata + let stream_id_num: usize = 1; + let _stream_stats = shared_metadata.register_stream( + stream_id_num, + Arc::from(stream_name.as_str()), + IggyTimestamp::now(), + ); + create_stream_file_hierarchy(stream_id_num, config).await?; + + // Create topic in SharedMetadata + let topic_id_num: usize = 1; let message_expiry = config.resolve_message_expiry(topic_expiry); let max_topic_size = config.resolve_max_topic_size(topic_size)?; - - let topic = topic::create_and_insert_topics_mem( - &streams, - &stream_id, - topic_name, - 1, + let _topic_stats = shared_metadata.register_topic( + stream_id_num, + topic_id_num, + Arc::from(topic_name.as_str()), + IggyTimestamp::now(), message_expiry, CompressionAlgorithm::default(), max_topic_size, - parent_stats, + 1, // replication_factor + partitions_count, ); - create_topic_file_hierarchy(stream.id(), topic.id(), config).await?; - // Create partition together with its dirs - let topic_id = Identifier::numeric(topic.id() as u32).unwrap(); - let parent_stats = streams.with_topic_by_id( - &stream_id, - &topic_id, - streaming::topics::helpers::get_stats(), + create_topic_file_hierarchy(stream_id_num, topic_id_num, config).await?; + + // Create partition in SharedMetadata + let partition_id: usize = 0; + let namespace = IggyNamespace::new(stream_id_num, topic_id_num, partition_id); + let partition_stats = shared_metadata.register_partition( + stream_id_num, + topic_id_num, + partition_id, + IggyTimestamp::now(), ); - let partitions = partition::create_and_insert_partitions_mem( - &streams, - &stream_id, - &topic_id, - parent_stats, - partitions_count, + create_partition_file_hierarchy(stream_id_num, topic_id_num, partition_id, config).await?; + + // Create TestStreams with partition data + let streams = TestStreams::new(shared_metadata); + + // Initialize partition data in partition_store + let start_offset = 0u64; + let segment = Segment::new( + start_offset, + config.segment.size, + config.segment.message_expiry, + ); + let messages_size = 0; + let indexes_size = 0; + let storage = create_segment_storage( config, + stream_id_num, + topic_id_num, + partition_id, + messages_size, + indexes_size, + start_offset, + ) + .await?; + + // Create partition data with the log + let consumer_offsets = Arc::new(ConsumerOffsets::with_capacity(0)); + let consumer_group_offsets = Arc::new(ConsumerGroupOffsets::with_capacity(0)); + let message_deduplicator = create_message_deduplicator(config).map(Arc::new); + let current_offset = Arc::new(AtomicU64::new(0)); + + let mut partition_data = PartitionData::new( + partition_stats, + current_offset, + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + IggyTimestamp::now(), + 0, // revision_id + false, // should_increment_offset ); - for partition in partitions { - create_partition_file_hierarchy(stream.id(), topic.id(), partition.id(), config).await?; - - // Open the log - let start_offset = 0; - let segment = Segment::new( - start_offset, - config.segment.size, - config.segment.message_expiry, - ); - let messages_size = 0; - let indexes_size = 0; - let storage = create_segment_storage( - config, - stream.id(), - topic.id(), - partition.id(), - messages_size, - indexes_size, - start_offset, - ) - .await?; - streams.with_partition_by_id_mut(&stream_id, &topic_id, partition.id(), |(.., log)| { - log.add_persisted_segment(segment, storage); - }); - } + // Add the segment to the log + partition_data.log.add_persisted_segment(segment, storage); + + // Insert into partition store + streams + .partition_store + .borrow_mut() + .insert(namespace, partition_data); // Create a test task registry with dummy stop sender from ShardConnector let connector: ShardConnector<()> = ShardConnector::new(shard_id); let task_registry = Rc::new(TaskRegistry::new(shard_id, vec![connector.stop_sender])); + let stream_id = Identifier::numeric(stream_id_num as u32).unwrap(); + let topic_id = Identifier::numeric(topic_id_num as u32).unwrap(); + Ok(BootstrapResult { streams, stream_id, topic_id, - partition_id: 0, + partition_id, + namespace, task_registry, }) } diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index ff38a114bf..3493ef42b6 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml @@ -40,7 +40,7 @@ iggy-web = ["dep:rust-embed", "dep:mime_guess"] [dependencies] ahash = { workspace = true } anyhow = { workspace = true } -arcshift = "0.4.2" +arc-swap = "1.8.0" argon2 = { workspace = true } async-channel = { workspace = true } async_zip = { workspace = true } @@ -60,7 +60,6 @@ cyper-axum = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true } dotenvy = { workspace = true } -enum_dispatch = { workspace = true } err_trail = { workspace = true } error_set = { workspace = true } figlet-rs = { workspace = true } @@ -69,8 +68,8 @@ flume = { workspace = true } futures = { workspace = true } hash32 = "1.0.0" human-repr = { workspace = true } - iggy_common = { workspace = true } +imbl = "6.1.0" jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] } lending-iterator = "0.1.7" mimalloc = { workspace = true, optional = true } @@ -107,7 +106,6 @@ rustls-pemfile = "2.2.0" send_wrapper = "0.6.0" serde = { workspace = true } serde_with = { workspace = true } -slab = "0.4.11" socket2 = "0.6.1" static-toml = "1.3.0" strum = { workspace = true } @@ -139,3 +137,10 @@ vergen-git2 = { version = "1.0.7", features = [ "rustc", "si", ] } + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } + +[[bench]] +name = "segmented_slab" +harness = false diff --git a/core/server/benches/segmented_slab.rs b/core/server/benches/segmented_slab.rs new file mode 100644 index 0000000000..db19012b46 --- /dev/null +++ b/core/server/benches/segmented_slab.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use iggy_common::collections::SegmentedSlab; +use imbl::HashMap as ImHashMap; + +const ENTRIES: usize = 10_000; +const SEGMENT_SIZE: usize = 1024; + +type BenchSlab = SegmentedSlab; + +fn setup_imhashmap() -> ImHashMap { + let mut map = ImHashMap::new(); + for i in 0..ENTRIES { + map = map.update(i, i * 2); + } + map +} + +fn setup_segmented_slab() -> BenchSlab { + let mut slab: BenchSlab = SegmentedSlab::new(); + for i in 0..ENTRIES { + let (new_slab, _) = slab.insert(i * 2); + slab = new_slab; + } + slab +} + +fn bench_get(c: &mut Criterion) { + let mut group = c.benchmark_group("get"); + group.throughput(Throughput::Elements(100)); + + let im_map = setup_imhashmap(); + let slab = setup_segmented_slab(); + + group.bench_function(BenchmarkId::new("ImHashMap", ENTRIES), |b| { + b.iter(|| { + for i in 0..100 { + std::hint::black_box(im_map.get(&(i * 100))); + } + }); + }); + + group.bench_function(BenchmarkId::new("SegmentedSlab", ENTRIES), |b| { + b.iter(|| { + for i in 0..100 { + std::hint::black_box(slab.get(i * 100)); + } + }); + }); + + group.finish(); +} + +fn bench_update(c: &mut Criterion) { + let mut group = c.benchmark_group("update"); + group.throughput(Throughput::Elements(1)); + + group.bench_function(BenchmarkId::new("ImHashMap", ENTRIES), |b| { + b.iter_batched( + setup_imhashmap, + |mut map| { + for i in 0..100 { + map = map.update(i % ENTRIES, i); + } + map + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function(BenchmarkId::new("SegmentedSlab", ENTRIES), |b| { + b.iter_batched( + setup_segmented_slab, + |mut slab| { + for i in 0..100 { + let (new_slab, _) = slab.update(i % ENTRIES, i); + slab = new_slab; + } + slab + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn bench_clone(c: &mut Criterion) { + let mut group = c.benchmark_group("clone"); + group.throughput(Throughput::Elements(1)); + + let im_map = setup_imhashmap(); + let slab = setup_segmented_slab(); + + group.bench_function(BenchmarkId::new("ImHashMap", ENTRIES), |b| { + b.iter(|| std::hint::black_box(im_map.clone())); + }); + + group.bench_function(BenchmarkId::new("SegmentedSlab", ENTRIES), |b| { + b.iter(|| std::hint::black_box(slab.clone())); + }); + + group.finish(); +} + +fn bench_insert(c: &mut Criterion) { + let mut group = c.benchmark_group("insert"); + group.throughput(Throughput::Elements(1)); + + group.bench_function("ImHashMap", |b| { + b.iter_batched( + ImHashMap::::new, + |mut map| { + for i in 0..100 { + map = map.update(i, i); + } + map + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function("SegmentedSlab", |b| { + b.iter_batched( + BenchSlab::::new, + |mut slab| { + for _ in 0..100 { + let (new_slab, _) = slab.insert(0usize); + slab = new_slab; + } + slab + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn bench_remove(c: &mut Criterion) { + let mut group = c.benchmark_group("remove"); + group.throughput(Throughput::Elements(1)); + + group.bench_function(BenchmarkId::new("ImHashMap", ENTRIES), |b| { + b.iter_batched( + setup_imhashmap, + |mut map| { + for i in 0..100 { + map = map.without(&(i % ENTRIES)); + } + map + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function(BenchmarkId::new("SegmentedSlab", ENTRIES), |b| { + b.iter_batched( + setup_segmented_slab, + |mut slab| { + for i in 0..100 { + let (new_slab, _) = slab.remove(i % ENTRIES); + slab = new_slab; + } + slab + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +criterion_group!( + benches, + bench_get, + bench_update, + bench_clone, + bench_insert, + bench_remove +); +criterion_main!(benches); diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs index e10fe6ab86..d090e3a0f6 100644 --- a/core/server/src/binary/command.rs +++ b/core/server/src/binary/command.rs @@ -18,9 +18,9 @@ use crate::define_server_command_enum; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use bytes::{BufMut, Bytes, BytesMut}; -use enum_dispatch::enum_dispatch; use iggy_common::SenderKind; use iggy_common::change_password::ChangePassword; use iggy_common::create_consumer_group::CreateConsumerGroup; @@ -72,53 +72,57 @@ use strum::EnumString; use tracing::error; define_server_command_enum! { - Ping(Ping), PING_CODE, PING, false; - GetStats(GetStats), GET_STATS_CODE, GET_STATS, false; - GetMe(GetMe), GET_ME_CODE, GET_ME, false; - GetClient(GetClient), GET_CLIENT_CODE, GET_CLIENT, true; - GetClients(GetClients), GET_CLIENTS_CODE, GET_CLIENTS, false; - GetSnapshot(GetSnapshot), GET_SNAPSHOT_FILE_CODE, GET_SNAPSHOT_FILE, false; - GetClusterMetadata(GetClusterMetadata), GET_CLUSTER_METADATA_CODE, GET_CLUSTER_METADATA, false; - PollMessages(PollMessages), POLL_MESSAGES_CODE, POLL_MESSAGES, true; - FlushUnsavedBuffer(FlushUnsavedBuffer), FLUSH_UNSAVED_BUFFER_CODE, FLUSH_UNSAVED_BUFFER, true; - GetUser(GetUser), GET_USER_CODE, GET_USER, true; - GetUsers(GetUsers), GET_USERS_CODE, GET_USERS, false; - CreateUser(CreateUser), CREATE_USER_CODE, CREATE_USER, true; - DeleteUser(DeleteUser), DELETE_USER_CODE, DELETE_USER, true; - UpdateUser(UpdateUser), UPDATE_USER_CODE, UPDATE_USER, true; - UpdatePermissions(UpdatePermissions), UPDATE_PERMISSIONS_CODE, UPDATE_PERMISSIONS, true; - ChangePassword(ChangePassword), CHANGE_PASSWORD_CODE, CHANGE_PASSWORD, true; - LoginUser(LoginUser), LOGIN_USER_CODE, LOGIN_USER, true; - LogoutUser(LogoutUser), LOGOUT_USER_CODE, LOGOUT_USER, false; - GetPersonalAccessTokens(GetPersonalAccessTokens), GET_PERSONAL_ACCESS_TOKENS_CODE, GET_PERSONAL_ACCESS_TOKENS, false; - CreatePersonalAccessToken(CreatePersonalAccessToken), CREATE_PERSONAL_ACCESS_TOKEN_CODE, CREATE_PERSONAL_ACCESS_TOKEN, true; - DeletePersonalAccessToken(DeletePersonalAccessToken), DELETE_PERSONAL_ACCESS_TOKEN_CODE, DELETE_PERSONAL_ACCESS_TOKEN, false; - LoginWithPersonalAccessToken(LoginWithPersonalAccessToken), LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN, true; - SendMessages(SendMessages), SEND_MESSAGES_CODE, SEND_MESSAGES, false; - GetConsumerOffset(GetConsumerOffset), GET_CONSUMER_OFFSET_CODE, GET_CONSUMER_OFFSET, true; - StoreConsumerOffset(StoreConsumerOffset), STORE_CONSUMER_OFFSET_CODE, STORE_CONSUMER_OFFSET, true; - DeleteConsumerOffset(DeleteConsumerOffset), DELETE_CONSUMER_OFFSET_CODE, DELETE_CONSUMER_OFFSET, true; - GetStream(GetStream), GET_STREAM_CODE, GET_STREAM, true; - GetStreams(GetStreams), GET_STREAMS_CODE, GET_STREAMS, false; - CreateStream(CreateStream), CREATE_STREAM_CODE, CREATE_STREAM, true; - DeleteStream(DeleteStream), DELETE_STREAM_CODE, DELETE_STREAM, true; - UpdateStream(UpdateStream), UPDATE_STREAM_CODE, UPDATE_STREAM, true; - PurgeStream(PurgeStream), PURGE_STREAM_CODE, PURGE_STREAM, true; - GetTopic(GetTopic), GET_TOPIC_CODE, GET_TOPIC, true; - GetTopics(GetTopics), GET_TOPICS_CODE, GET_TOPICS, false; - CreateTopic(CreateTopic), CREATE_TOPIC_CODE, CREATE_TOPIC, true; - DeleteTopic(DeleteTopic), DELETE_TOPIC_CODE, DELETE_TOPIC, true; - UpdateTopic(UpdateTopic), UPDATE_TOPIC_CODE, UPDATE_TOPIC, true; - PurgeTopic(PurgeTopic), PURGE_TOPIC_CODE, PURGE_TOPIC, true; - CreatePartitions(CreatePartitions), CREATE_PARTITIONS_CODE, CREATE_PARTITIONS, true; - DeletePartitions(DeletePartitions), DELETE_PARTITIONS_CODE, DELETE_PARTITIONS, true; - DeleteSegments(DeleteSegments), DELETE_SEGMENTS_CODE, DELETE_SEGMENTS, true; - GetConsumerGroup(GetConsumerGroup), GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUP, true; - GetConsumerGroups(GetConsumerGroups), GET_CONSUMER_GROUPS_CODE, GET_CONSUMER_GROUPS, false; - CreateConsumerGroup(CreateConsumerGroup), CREATE_CONSUMER_GROUP_CODE, CREATE_CONSUMER_GROUP, true; - DeleteConsumerGroup(DeleteConsumerGroup), DELETE_CONSUMER_GROUP_CODE, DELETE_CONSUMER_GROUP, true; - JoinConsumerGroup(JoinConsumerGroup), JOIN_CONSUMER_GROUP_CODE, JOIN_CONSUMER_GROUP, true; - LeaveConsumerGroup(LeaveConsumerGroup), LEAVE_CONSUMER_GROUP_CODE, LEAVE_CONSUMER_GROUP, true; + @unauth { + Ping(Ping), PING_CODE, PING, false; + LoginUser(LoginUser), LOGIN_USER_CODE, LOGIN_USER, true; + LoginWithPersonalAccessToken(LoginWithPersonalAccessToken), LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN, true; + } + @auth { + GetStats(GetStats), GET_STATS_CODE, GET_STATS, false; + GetMe(GetMe), GET_ME_CODE, GET_ME, false; + GetClient(GetClient), GET_CLIENT_CODE, GET_CLIENT, true; + GetClients(GetClients), GET_CLIENTS_CODE, GET_CLIENTS, false; + GetSnapshot(GetSnapshot), GET_SNAPSHOT_FILE_CODE, GET_SNAPSHOT_FILE, false; + GetClusterMetadata(GetClusterMetadata), GET_CLUSTER_METADATA_CODE, GET_CLUSTER_METADATA, false; + PollMessages(PollMessages), POLL_MESSAGES_CODE, POLL_MESSAGES, true; + FlushUnsavedBuffer(FlushUnsavedBuffer), FLUSH_UNSAVED_BUFFER_CODE, FLUSH_UNSAVED_BUFFER, true; + GetUser(GetUser), GET_USER_CODE, GET_USER, true; + GetUsers(GetUsers), GET_USERS_CODE, GET_USERS, false; + CreateUser(CreateUser), CREATE_USER_CODE, CREATE_USER, true; + DeleteUser(DeleteUser), DELETE_USER_CODE, DELETE_USER, true; + UpdateUser(UpdateUser), UPDATE_USER_CODE, UPDATE_USER, true; + UpdatePermissions(UpdatePermissions), UPDATE_PERMISSIONS_CODE, UPDATE_PERMISSIONS, true; + ChangePassword(ChangePassword), CHANGE_PASSWORD_CODE, CHANGE_PASSWORD, true; + LogoutUser(LogoutUser), LOGOUT_USER_CODE, LOGOUT_USER, false; + GetPersonalAccessTokens(GetPersonalAccessTokens), GET_PERSONAL_ACCESS_TOKENS_CODE, GET_PERSONAL_ACCESS_TOKENS, false; + CreatePersonalAccessToken(CreatePersonalAccessToken), CREATE_PERSONAL_ACCESS_TOKEN_CODE, CREATE_PERSONAL_ACCESS_TOKEN, true; + DeletePersonalAccessToken(DeletePersonalAccessToken), DELETE_PERSONAL_ACCESS_TOKEN_CODE, DELETE_PERSONAL_ACCESS_TOKEN, false; + SendMessages(SendMessages), SEND_MESSAGES_CODE, SEND_MESSAGES, false; + GetConsumerOffset(GetConsumerOffset), GET_CONSUMER_OFFSET_CODE, GET_CONSUMER_OFFSET, true; + StoreConsumerOffset(StoreConsumerOffset), STORE_CONSUMER_OFFSET_CODE, STORE_CONSUMER_OFFSET, true; + DeleteConsumerOffset(DeleteConsumerOffset), DELETE_CONSUMER_OFFSET_CODE, DELETE_CONSUMER_OFFSET, true; + GetStream(GetStream), GET_STREAM_CODE, GET_STREAM, true; + GetStreams(GetStreams), GET_STREAMS_CODE, GET_STREAMS, false; + CreateStream(CreateStream), CREATE_STREAM_CODE, CREATE_STREAM, true; + DeleteStream(DeleteStream), DELETE_STREAM_CODE, DELETE_STREAM, true; + UpdateStream(UpdateStream), UPDATE_STREAM_CODE, UPDATE_STREAM, true; + PurgeStream(PurgeStream), PURGE_STREAM_CODE, PURGE_STREAM, true; + GetTopic(GetTopic), GET_TOPIC_CODE, GET_TOPIC, true; + GetTopics(GetTopics), GET_TOPICS_CODE, GET_TOPICS, false; + CreateTopic(CreateTopic), CREATE_TOPIC_CODE, CREATE_TOPIC, true; + DeleteTopic(DeleteTopic), DELETE_TOPIC_CODE, DELETE_TOPIC, true; + UpdateTopic(UpdateTopic), UPDATE_TOPIC_CODE, UPDATE_TOPIC, true; + PurgeTopic(PurgeTopic), PURGE_TOPIC_CODE, PURGE_TOPIC, true; + CreatePartitions(CreatePartitions), CREATE_PARTITIONS_CODE, CREATE_PARTITIONS, true; + DeletePartitions(DeletePartitions), DELETE_PARTITIONS_CODE, DELETE_PARTITIONS, true; + DeleteSegments(DeleteSegments), DELETE_SEGMENTS_CODE, DELETE_SEGMENTS, true; + GetConsumerGroup(GetConsumerGroup), GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUP, true; + GetConsumerGroups(GetConsumerGroups), GET_CONSUMER_GROUPS_CODE, GET_CONSUMER_GROUPS, false; + CreateConsumerGroup(CreateConsumerGroup), CREATE_CONSUMER_GROUP_CODE, CREATE_CONSUMER_GROUP, true; + DeleteConsumerGroup(DeleteConsumerGroup), DELETE_CONSUMER_GROUP_CODE, DELETE_CONSUMER_GROUP, true; + JoinConsumerGroup(JoinConsumerGroup), JOIN_CONSUMER_GROUP_CODE, JOIN_CONSUMER_GROUP, true; + LeaveConsumerGroup(LeaveConsumerGroup), LEAVE_CONSUMER_GROUP_CODE, LEAVE_CONSUMER_GROUP, true; + } } /// Indicates whether a command handler completed normally or migrated the connection. @@ -130,12 +134,44 @@ pub enum HandlerResult { Migrated { to_shard: u16 }, } -#[enum_dispatch] -pub trait ServerCommandHandler { +/// Handler trait for commands that require authentication. +/// +/// Commands implementing this trait receive an [`Auth`] proof token that +/// guarantees the user was authenticated before the handler was called. +/// This token can be passed to shard methods that require authentication. +/// +/// # Compile-time Guarantees +/// - Handlers receive `Auth` only if dispatch performed authentication +/// - Shard methods requiring auth can only be called with a valid `Auth` token +pub trait AuthenticatedHandler { + /// Return the command code + fn code(&self) -> u32; + + /// Handle the command execution with authentication proof + #[allow(async_fn_in_trait)] + async fn handle( + self, + sender: &mut SenderKind, + length: u32, + auth: Auth, + session: &Session, + shard: &Rc, + ) -> Result; +} + +/// Handler trait for commands that do NOT require authentication. +/// +/// Commands implementing this trait can be executed without a valid user session. +/// This includes: `Ping`, `LoginUser`, `LoginWithPersonalAccessToken`. +/// +/// # Compile-time Guarantees +/// - These handlers do NOT receive an `Auth` token +/// - They cannot call shard methods that require authentication +pub trait UnauthenticatedHandler { /// Return the command code fn code(&self) -> u32; - /// Handle the command execution + /// Handle the command execution without authentication #[allow(async_fn_in_trait)] async fn handle( self, diff --git a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs index af6c7a9a0b..2f3d915e14 100644 --- a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs +++ b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs @@ -19,31 +19,32 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -//use crate::streaming::systems::system::SharedSystem; -use anyhow::Result; use iggy_common::get_cluster_metadata::GetClusterMetadata; use iggy_common::{BytesSerializable, IggyError, SenderKind}; use tracing::{debug, instrument}; -impl ServerCommandHandler for GetClusterMetadata { +impl AuthenticatedHandler for GetClusterMetadata { fn code(&self) -> u32 { iggy_common::GET_CLUSTER_METADATA_CODE } - #[instrument(skip_all, name = "trace_get_cluster_metadata", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_get_cluster_metadata", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); + let _ = auth; let cluster_metadata = shard.get_cluster_metadata(session)?; diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs index 192d867f53..d1ed0970d7 100644 --- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs @@ -17,76 +17,74 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::consumer_groups::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; -use crate::slab::traits_ext::EntityMarker; use crate::state::command::EntryCommand; use crate::state::models::CreateConsumerGroupWithId; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_consumer_group::CreateConsumerGroup; use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for CreateConsumerGroup { +impl AuthenticatedHandler for CreateConsumerGroup { fn code(&self) -> u32 { iggy_common::CREATE_CONSUMER_GROUP_CODE } - #[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - let cg = shard.create_consumer_group( + let cg_id = shard.create_consumer_group( session, &self.stream_id, &self.topic_id, self.name.clone(), )?; - let cg_id = cg.id(); - - let event = ShardEvent::CreatedConsumerGroup { - stream_id: self.stream_id.clone(), - topic_id: self.topic_id.clone(), - cg, - }; - shard.broadcast_event_to_all_shards(event).await?; let stream_id = self.stream_id.clone(); let topic_id = self.topic_id.clone(); shard .state - .apply( - session.get_user_id(), - &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { - group_id: cg_id as u32, - command: self - }), - ) + .apply( + auth.user_id(), + &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { + group_id: cg_id as u32, + command: self, + }), + ) .await .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - failed to apply create consumer group for stream_id: {stream_id}, topic_id: {topic_id}, group_id: {cg_id}, session: {session}" ) })?; - let response = shard.streams.with_consumer_group_by_id( - &stream_id, - &topic_id, - &Identifier::numeric(cg_id as u32).unwrap(), - |(root, members)| mapper::map_consumer_group(root, members), - ); + + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&stream_id, &topic_id)?; + + let cg_identifier = Identifier::numeric(cg_id as u32).unwrap(); + let metadata = shard.metadata.load(); + let response = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .and_then(|t| t.consumer_groups.get(cg_id)) + .map(mapper::map_consumer_group_from_meta) + .ok_or_else(|| IggyError::ConsumerGroupIdNotFound(cg_identifier, topic_id.clone()))?; sender.send_ok_response(&response).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs index 05c3cd9dea..efc917c35d 100644 --- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs @@ -17,65 +17,52 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::consumer_groups::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; -use crate::slab::traits_ext::EntityMarker; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::polling_consumer::ConsumerGroupId; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_consumer_group::DeleteConsumerGroup; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeleteConsumerGroup { +impl AuthenticatedHandler for DeleteConsumerGroup { fn code(&self) -> u32 { iggy_common::DELETE_CONSUMER_GROUP_CODE } - #[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - let cg = shard.delete_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id).error(|e: &IggyError| { + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&self.stream_id, &self.topic_id)?; + + let cg_meta = shard.delete_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id).error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {} for topic with ID: {} in stream with ID: {} for session: {}", self.group_id, self.topic_id, self.stream_id, session ) })?; - let cg_id = cg.id(); - let partition_ids = cg.partitions(); - - // Remove all consumer group members from ClientManager using helper functions to resolve identifiers - let stream_id_usize = shard.streams.with_stream_by_id( - &self.stream_id, - crate::streaming::streams::helpers::get_stream_id(), - ); - let topic_id_usize = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - crate::streaming::topics::helpers::get_topic_id(), - ); + let cg_id = cg_meta.id; - // Get members from the deleted consumer group and make them leave - let slab = cg.members().inner().shared_get(); - for (_, member) in slab.iter() { + for (_, member) in cg_meta.members.iter() { if let Err(err) = shard.client_manager.leave_consumer_group( member.client_id, - stream_id_usize, - topic_id_usize, + numeric_stream_id, + numeric_topic_id, cg_id, ) { tracing::warn!( @@ -87,12 +74,11 @@ impl ServerCommandHandler for DeleteConsumerGroup { } let cg_id_spez = ConsumerGroupId(cg_id); - // Delete all consumer group offsets for this group using the specialized method shard.delete_consumer_group_offsets( cg_id_spez, &self.stream_id, &self.topic_id, - partition_ids, + &cg_meta.partitions, ).await.error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}", @@ -102,19 +88,12 @@ impl ServerCommandHandler for DeleteConsumerGroup { ) })?; - let event = ShardEvent::DeletedConsumerGroup { - id: cg_id, - stream_id: self.stream_id.clone(), - topic_id: self.topic_id.clone(), - group_id: self.group_id.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; let stream_id = self.stream_id.clone(); let topic_id = self.topic_id.clone(); shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::DeleteConsumerGroup(self), ) .await diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs index 7156a4d53f..31153e8aeb 100644 --- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs @@ -17,21 +17,20 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_consumer_group::GetConsumerGroup; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetConsumerGroup { +impl AuthenticatedHandler for GetConsumerGroup { fn code(&self) -> u32 { iggy_common::GET_CONSUMER_GROUP_CODE } @@ -40,11 +39,11 @@ impl ServerCommandHandler for GetConsumerGroup { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; let exists = shard .ensure_consumer_group_exists(&self.stream_id, &self.topic_id, &self.group_id) .is_ok(); @@ -52,30 +51,35 @@ impl ServerCommandHandler for GetConsumerGroup { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); } - let numeric_topic_id = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - topics::helpers::get_topic_id(), - ); - let numeric_stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); + + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&self.stream_id, &self.topic_id)?; let has_permission = shard .permissioner - .borrow() - .get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id) + .get_consumer_group(auth.user_id(), numeric_stream_id, numeric_topic_id) .is_ok(); if !has_permission { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); } - let consumer_group = shard.streams.with_consumer_group_by_id( - &self.stream_id, - &self.topic_id, - &self.group_id, - |(root, members)| mapper::map_consumer_group(root, members), - ); + let numeric_group_id = shard + .metadata + .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &self.group_id) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound(self.group_id.clone(), self.topic_id.clone()) + })?; + + let metadata = shard.metadata.load(); + let consumer_group = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .and_then(|t| t.consumer_groups.get(numeric_group_id)) + .map(mapper::map_consumer_group_from_meta) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound(self.group_id.clone(), self.topic_id.clone()) + })?; sender.send_ok_response(&consumer_group).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs index d59f33c977..42a7b572f4 100644 --- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs @@ -17,22 +17,19 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; use crate::shard::IggyShard; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_consumer_groups::GetConsumerGroups; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetConsumerGroups { +impl AuthenticatedHandler for GetConsumerGroups { fn code(&self) -> u32 { iggy_common::GET_CONSUMER_GROUPS_CODE } @@ -41,35 +38,38 @@ impl ServerCommandHandler for GetConsumerGroups { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; - shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?; - let numeric_topic_id = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - topics::helpers::get_topic_id(), - ); - let numeric_stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); - shard.permissioner.borrow().get_consumer_groups( - session.get_user_id(), + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&self.stream_id, &self.topic_id)?; + shard.permissioner.get_consumer_groups( + auth.user_id(), numeric_stream_id, numeric_topic_id, )?; - let consumer_groups = - shard - .streams - .with_consumer_groups(&self.stream_id, &self.topic_id, |cgs| { - cgs.with_components(|cgs| { - let (roots, members) = cgs.into_components(); - mapper::map_consumer_groups(roots, members) - }) - }); + let metadata = shard.metadata.load(); + + let consumer_groups = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .map(|topic| { + use bytes::{BufMut, BytesMut}; + let mut bytes = BytesMut::new(); + for (_, cg_meta) in topic.consumer_groups.iter() { + bytes.put_u32_le(cg_meta.id as u32); + bytes.put_u32_le(cg_meta.partitions.len() as u32); + bytes.put_u32_le(cg_meta.members.len() as u32); + bytes.put_u8(cg_meta.name.len() as u8); + bytes.put_slice(cg_meta.name.as_bytes()); + } + bytes.freeze() + }) + .unwrap_or_default(); sender.send_ok_response(&consumer_groups).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs index cc9a8e27b3..251c7996e8 100644 --- a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs @@ -17,30 +17,30 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::consumer_groups::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::join_consumer_group::JoinConsumerGroup; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for JoinConsumerGroup { +impl AuthenticatedHandler for JoinConsumerGroup { fn code(&self) -> u32 { iggy_common::JOIN_CONSUMER_GROUP_CODE } - #[instrument(skip_all, name = "trace_join_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_group_id = self.group_id.as_string()))] + #[instrument(skip_all, name = "trace_join_consumer_group", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_group_id = self.group_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs index b56d94a48a..9d368d6613 100644 --- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs +++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs @@ -18,35 +18,34 @@ use super::COMPONENT; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; -use iggy_common::SenderKind; - use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::IggyError; +use iggy_common::SenderKind; use iggy_common::leave_consumer_group::LeaveConsumerGroup; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for LeaveConsumerGroup { +impl AuthenticatedHandler for LeaveConsumerGroup { fn code(&self) -> u32 { iggy_common::LEAVE_CONSUMER_GROUP_CODE } - #[instrument(skip_all, name = "trace_leave_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_group_id = self.group_id.as_string()))] + #[instrument(skip_all, name = "trace_leave_consumer_group", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_group_id = self.group_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard .leave_consumer_group( session, diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs index 45f22349d0..e0414c85a1 100644 --- a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs +++ b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs @@ -17,20 +17,20 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::consumer_offsets::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_consumer_offset::DeleteConsumerOffset; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for DeleteConsumerOffset { +impl AuthenticatedHandler for DeleteConsumerOffset { fn code(&self) -> u32 { iggy_common::DELETE_CONSUMER_OFFSET_CODE } @@ -39,6 +39,7 @@ impl ServerCommandHandler for DeleteConsumerOffset { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs index 29ee6f3101..868c2a2371 100644 --- a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs +++ b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs @@ -17,20 +17,20 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_consumer_offset::GetConsumerOffset; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetConsumerOffset { +impl AuthenticatedHandler for GetConsumerOffset { fn code(&self) -> u32 { iggy_common::GET_CONSUMER_OFFSET_CODE } @@ -39,6 +39,7 @@ impl ServerCommandHandler for GetConsumerOffset { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs index dcbb79db3c..4661f524a2 100644 --- a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs +++ b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs @@ -19,20 +19,20 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::consumer_offsets::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::store_consumer_offset::StoreConsumerOffset; use tracing::debug; -impl ServerCommandHandler for StoreConsumerOffset { +impl AuthenticatedHandler for StoreConsumerOffset { fn code(&self) -> u32 { iggy_common::STORE_CONSUMER_OFFSET_CODE } @@ -41,6 +41,7 @@ impl ServerCommandHandler for StoreConsumerOffset { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs index 937a03cb35..8fe71f96c4 100644 --- a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs +++ b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs @@ -17,34 +17,35 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::messages::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::{FlushUnsavedBuffer, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for FlushUnsavedBuffer { +impl AuthenticatedHandler for FlushUnsavedBuffer { fn code(&self) -> u32 { iggy_common::FLUSH_UNSAVED_BUFFER_CODE } - #[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_partition_id = self.partition_id, iggy_fsync = self.fsync))] + #[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), iggy_partition_id = self.partition_id, iggy_fsync = self.fsync))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - let user_id = session.get_user_id(); + let user_id = auth.user_id(); let stream_id = self.stream_id.clone(); let topic_id = self.topic_id.clone(); let partition_id = self.partition_id; diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs index 0839e3a55e..e508ae6c92 100644 --- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs +++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs @@ -17,13 +17,13 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; use crate::shard::system::messages::PollingArgs; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use iggy_common::SenderKind; use iggy_common::{IggyError, PollMessages, PooledBuffer}; use std::rc::Rc; @@ -44,7 +44,7 @@ impl IggyPollMetadata { } } -impl ServerCommandHandler for PollMessages { +impl AuthenticatedHandler for PollMessages { fn code(&self) -> u32 { iggy_common::POLL_MESSAGES_CODE } @@ -53,6 +53,7 @@ impl ServerCommandHandler for PollMessages { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -68,7 +69,7 @@ impl ServerCommandHandler for PollMessages { } = self; let args = PollingArgs::new(strategy, count, auto_commit); - let user_id = session.get_user_id(); + let user_id = auth.user_id(); let client_id = session.client_id; let (metadata, mut batch) = shard .poll_messages( diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs index 3758487995..c005532d7d 100644 --- a/core/server/src/binary/handlers/messages/send_messages_handler.rs +++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs @@ -16,13 +16,13 @@ * under the License. */ -use crate::binary::command::{BinaryServerCommand, HandlerResult, ServerCommandHandler}; +use crate::binary::command::{AuthenticatedHandler, BinaryServerCommand, HandlerResult}; use crate::shard::IggyShard; use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload}; +use crate::streaming::auth::Auth; use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut}; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; -use anyhow::Result; +use crate::streaming::topics; use compio::buf::{IntoInner as _, IoBuf}; use iggy_common::Identifier; use iggy_common::PooledBuffer; @@ -34,13 +34,13 @@ use iggy_common::{IggyError, Partitioning, SendMessages, Validatable}; use std::rc::Rc; use tracing::{debug, error, info, instrument}; -impl ServerCommandHandler for SendMessages { +impl AuthenticatedHandler for SendMessages { fn code(&self) -> u32 { iggy_common::SEND_MESSAGES_CODE } #[instrument(skip_all, name = "trace_send_messages", fields( - iggy_user_id = session.get_user_id(), + iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string(), @@ -50,6 +50,7 @@ impl ServerCommandHandler for SendMessages { mut self, sender: &mut SenderKind, length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -110,47 +111,35 @@ impl ServerCommandHandler for SendMessages { ); batch.validate()?; - shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?; + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&self.stream_id, &self.topic_id)?; - let numeric_stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); - - let numeric_topic_id = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - topics::helpers::get_topic_id(), - ); - - // TODO(tungtose): dry this code && get partition_id below have a side effect - let partition_id = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - |(root, auxilary, ..)| match self.partitioning.kind { - PartitioningKind::Balanced => { - let upperbound = root.partitions().len(); - let pid = auxilary.get_next_partition_id(upperbound); - Ok(pid) - } - PartitioningKind::PartitionId => Ok(u32::from_le_bytes( - self.partitioning.value[..self.partitioning.length as usize] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize), - PartitioningKind::MessagesKey => { - let upperbound = root.partitions().len(); - Ok( - topics::helpers::calculate_partition_id_by_messages_key_hash( - upperbound, - &self.partitioning.value, - ), - ) - } - }, - )?; + let partition_id = match self.partitioning.kind { + PartitioningKind::Balanced => shard + .metadata + .get_next_partition_id(numeric_stream_id, numeric_topic_id) + .ok_or(IggyError::TopicIdNotFound( + self.stream_id.clone(), + self.topic_id.clone(), + ))?, + PartitioningKind::PartitionId => u32::from_le_bytes( + self.partitioning.value[..self.partitioning.length as usize] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize, + PartitioningKind::MessagesKey => { + let partitions_count = shard + .metadata + .partitions_count(numeric_stream_id, numeric_topic_id); + topics::helpers::calculate_partition_id_by_messages_key_hash( + partitions_count, + &self.partitioning.value, + ) + } + }; let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); - let user_id = session.get_user_id(); + let user_id = auth.user_id(); let unsupport_socket_transfer = matches!( self.partitioning.kind, PartitioningKind::Balanced | PartitioningKind::MessagesKey diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs index 5c1930a8ac..2afb17ab8f 100644 --- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs +++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs @@ -17,85 +17,126 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::partitions::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; -use crate::slab::traits_ext::EntityMarker; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, +}; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_partitions::CreatePartitions; -use iggy_common::{IggyError, SenderKind}; +use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for CreatePartitions { +impl AuthenticatedHandler for CreatePartitions { fn code(&self) -> u32 { iggy_common::CREATE_PARTITIONS_CODE } - #[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - // Acquire partition lock to serialize filesystem operations - let _partition_guard = shard.fs_locks.partition_lock.lock().await; - - let partitions = shard - .create_partitions( - session, - &self.stream_id, - &self.topic_id, - self.partitions_count, - ) - .await?; - let partition_ids = partitions.iter().map(|p| p.id()).collect::>(); - let event = ShardEvent::CreatedPartitions { - stream_id: self.stream_id.clone(), - topic_id: self.topic_id.clone(), - partitions, + let request = ShardRequest { + stream_id: Identifier::default(), + topic_id: Identifier::default(), + partition_id: 0, + payload: ShardRequestPayload::CreatePartitions { + user_id: auth.user_id(), + stream_id: self.stream_id.clone(), + topic_id: self.topic_id.clone(), + partitions_count: self.partitions_count, + }, }; - shard.broadcast_event_to_all_shards(event).await?; - shard.streams.with_topic_by_id_mut( - &self.stream_id, - &self.topic_id, - topics::helpers::rebalance_consumer_groups(&partition_ids), - ); + let message = ShardMessage::Request(request); + match shard.send_request_to_shard_or_recoil(None, message).await? { + ShardSendRequestResult::Recoil(message) => { + if let ShardMessage::Request(ShardRequest { payload, .. }) = message + && let ShardRequestPayload::CreatePartitions { + stream_id, + topic_id, + partitions_count, + .. + } = payload + { + let _partition_guard = shard.fs_locks.partition_lock.lock().await; + + // Get numeric IDs BEFORE create (for rebalance operation) + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&stream_id, &topic_id)?; + + let partition_infos = shard + .create_partitions(session, &stream_id, &topic_id, partitions_count) + .await?; + let partition_ids = partition_infos.iter().map(|p| p.id).collect::>(); + + let event = crate::shard::transmission::event::ShardEvent::CreatedPartitions { + stream_id: stream_id.clone(), + topic_id: topic_id.clone(), + partitions: partition_infos, + }; + shard.broadcast_event_to_all_shards(event).await?; + + // Rebalance consumer groups using SharedMetadata + shard.metadata.rebalance_consumer_groups_for_topic( + numeric_stream_id, + numeric_topic_id, + &partition_ids, + ); + + shard + .state + .apply(auth.user_id(), &EntryCommand::CreatePartitions(self)) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {numeric_stream_id}, topic_id: {numeric_topic_id}, session: {session}" + ) + })?; + + sender.send_empty_ok_response().await?; + } else { + unreachable!( + "Expected a CreatePartitions request inside of CreatePartitions handler, impossible state" + ); + } + } + ShardSendRequestResult::Response(response) => match response { + ShardResponse::CreatePartitionsResponse(_partitions) => { + // Consumer group rebalancing is handled via the CreatedPartitions event + // broadcast in the shard-0 handler, so we don't need to do it here. + // Apply state for WAL replay. + shard + .state + .apply(auth.user_id(), &EntryCommand::CreatePartitions(self)) + .await?; + + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => { + return Err(err); + } + _ => unreachable!( + "Expected a CreatePartitionsResponse inside of CreatePartitions handler, impossible state" + ), + }, + } - let stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); - let topic_id = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - topics::helpers::get_topic_id(), - ); - shard - .state - .apply( - session.get_user_id(), - &EntryCommand::CreatePartitions(self), - ) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}" - ) - })?; - sender.send_empty_ok_response().await?; Ok(HandlerResult::Finished) } } diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs index 9090ec2c86..d11d3d122d 100644 --- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs +++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs @@ -17,82 +17,135 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::partitions::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, +}; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_partitions::DeletePartitions; -use iggy_common::{IggyError, SenderKind}; +use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeletePartitions { +impl AuthenticatedHandler for DeletePartitions { fn code(&self) -> u32 { iggy_common::DELETE_PARTITIONS_CODE } - #[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - let stream_id = self.stream_id.clone(); - let topic_id = self.topic_id.clone(); - - // Acquire partition lock to serialize filesystem operations - let _partition_guard = shard.fs_locks.partition_lock.lock().await; - let deleted_partition_ids = shard - .delete_partitions( - session, - &self.stream_id, - &self.topic_id, - self.partitions_count, - ) - .await?; - let event = ShardEvent::DeletedPartitions { - stream_id: self.stream_id.clone(), - topic_id: self.topic_id.clone(), - partitions_count: self.partitions_count, - partition_ids: deleted_partition_ids, + let request = ShardRequest { + stream_id: Identifier::default(), + topic_id: Identifier::default(), + partition_id: 0, + payload: ShardRequestPayload::DeletePartitions { + user_id: auth.user_id(), + stream_id: self.stream_id.clone(), + topic_id: self.topic_id.clone(), + partitions_count: self.partitions_count, + }, }; - shard.broadcast_event_to_all_shards(event).await?; - let remaining_partition_ids = shard.streams.with_topic_by_id( - &self.stream_id, - &self.topic_id, - crate::streaming::topics::helpers::get_partition_ids(), - ); - shard.streams.with_topic_by_id_mut( - &self.stream_id, - &self.topic_id, - crate::streaming::topics::helpers::rebalance_consumer_groups(&remaining_partition_ids), - ); + let message = ShardMessage::Request(request); + match shard.send_request_to_shard_or_recoil(None, message).await? { + ShardSendRequestResult::Recoil(message) => { + if let ShardMessage::Request(ShardRequest { payload, .. }) = message + && let ShardRequestPayload::DeletePartitions { + stream_id, + topic_id, + partitions_count, + .. + } = payload + { + let _partition_guard = shard.fs_locks.partition_lock.lock().await; + + // Get numeric IDs BEFORE delete + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&stream_id, &topic_id)?; + + let deleted_partition_ids = shard + .delete_partitions(session, &stream_id, &topic_id, partitions_count) + .await?; + + let event = crate::shard::transmission::event::ShardEvent::DeletedPartitions { + stream_id: stream_id.clone(), + topic_id: topic_id.clone(), + partitions_count, + partition_ids: deleted_partition_ids, + }; + shard.broadcast_event_to_all_shards(event).await?; + + // Rebalance consumer groups using SharedMetadata + let remaining_partition_ids: Vec<_> = shard + .metadata + .load() + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .map(|t| t.partitions.iter().map(|(id, _)| id).collect()) + .unwrap_or_default(); + shard.metadata.rebalance_consumer_groups_for_topic( + numeric_stream_id, + numeric_topic_id, + &remaining_partition_ids, + ); + + shard + .state + .apply(auth.user_id(), &EntryCommand::DeletePartitions(self)) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {}, topic_id: {}, session: {session}", + stream_id, topic_id + ) + })?; + + sender.send_empty_ok_response().await?; + } else { + unreachable!( + "Expected a DeletePartitions request inside of DeletePartitions handler, impossible state" + ); + } + } + ShardSendRequestResult::Response(response) => match response { + ShardResponse::DeletePartitionsResponse(_partition_ids) => { + // Consumer group rebalancing is handled via the DeletedPartitions event + // broadcast in the shard-0 handler, so we don't need to do it here. + // Apply state for WAL replay. + shard + .state + .apply(auth.user_id(), &EntryCommand::DeletePartitions(self)) + .await?; + + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => { + return Err(err); + } + _ => unreachable!( + "Expected a DeletePartitionsResponse inside of DeletePartitions handler, impossible state" + ), + }, + } - shard - .state - .apply( - session.get_user_id(), - &EntryCommand::DeletePartitions(self), - ) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}" - ) - })?; - sender.send_empty_ok_response().await?; Ok(HandlerResult::Finished) } } diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs index 50b896fb01..fd393a32d3 100644 --- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs +++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs @@ -17,34 +17,34 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::personal_access_tokens::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::state::command::EntryCommand; use crate::state::models::CreatePersonalAccessTokenWithHash; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_personal_access_token::CreatePersonalAccessToken; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for CreatePersonalAccessToken { +impl AuthenticatedHandler for CreatePersonalAccessToken { fn code(&self) -> u32 { iggy_common::CREATE_PERSONAL_ACCESS_TOKEN_CODE } - #[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -60,15 +60,11 @@ impl ServerCommandHandler for CreatePersonalAccessToken { })?; let bytes = mapper::map_raw_pat(&token); let hash = personal_access_token.token.to_string(); - let event = ShardEvent::CreatedPersonalAccessToken { - personal_access_token: personal_access_token.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash { command: CreatePersonalAccessToken { name: self.name.to_owned(), diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs index bf89c70cab..c50b6af872 100644 --- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs +++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs @@ -17,30 +17,31 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::personal_access_tokens::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_personal_access_token::DeletePersonalAccessToken; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeletePersonalAccessToken { +impl AuthenticatedHandler for DeletePersonalAccessToken { fn code(&self) -> u32 { iggy_common::DELETE_PERSONAL_ACCESS_TOKEN_CODE } - #[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -53,17 +54,10 @@ impl ServerCommandHandler for DeletePersonalAccessToken { "{COMPONENT} (error: {e}) - failed to delete personal access token with name: {token_name}, session: {session}" )})?; - // Broadcast the event to other shards - let event = crate::shard::transmission::event::ShardEvent::DeletedPersonalAccessToken { - user_id: session.get_user_id(), - name: self.name.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::DeletePersonalAccessToken(DeletePersonalAccessToken { name: self.name, }), diff --git a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs index 9e2923868c..35885c1a8f 100644 --- a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs +++ b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs @@ -17,12 +17,13 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::personal_access_tokens::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use err_trail::ErrContext; use iggy_common::IggyError; @@ -31,7 +32,7 @@ use iggy_common::get_personal_access_tokens::GetPersonalAccessTokens; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetPersonalAccessTokens { +impl AuthenticatedHandler for GetPersonalAccessTokens { fn code(&self) -> u32 { iggy_common::GET_PERSONAL_ACCESS_TOKENS_CODE } @@ -40,6 +41,7 @@ impl ServerCommandHandler for GetPersonalAccessTokens { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs index 1603aaa6a1..73914ffcd7 100644 --- a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs +++ b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs @@ -19,19 +19,18 @@ use crate::shard::IggyShard; use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + BinaryServerCommand, HandlerResult, ServerCommand, UnauthenticatedHandler, }; use crate::binary::handlers::personal_access_tokens::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken; use iggy_common::{IggyError, SenderKind}; use tracing::{debug, instrument}; -impl ServerCommandHandler for LoginWithPersonalAccessToken { +impl UnauthenticatedHandler for LoginWithPersonalAccessToken { fn code(&self) -> u32 { iggy_common::LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE } diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs index a4eb206543..e24b8a3dfa 100644 --- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs +++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs @@ -17,7 +17,7 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::partitions::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; @@ -27,9 +27,8 @@ use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; use crate::state::command::EntryCommand; -use crate::streaming; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_segments::DeleteSegments; use iggy_common::sharding::IggyNamespace; @@ -37,16 +36,17 @@ use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeleteSegments { +impl AuthenticatedHandler for DeleteSegments { fn code(&self) -> u32 { iggy_common::DELETE_SEGMENTS_CODE } - #[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -56,24 +56,11 @@ impl ServerCommandHandler for DeleteSegments { let topic_id = self.topic_id.clone(); let partition_id = self.partition_id as usize; let segments_count = self.segments_count; - - // Ensure authentication and topic exists - shard.ensure_authenticated(session)?; - shard.ensure_topic_exists(&stream_id, &topic_id)?; shard.ensure_partition_exists(&stream_id, &topic_id, partition_id)?; - // Get numeric IDs for namespace - let numeric_stream_id = shard - .streams - .with_stream_by_id(&stream_id, streaming::streams::helpers::get_stream_id()); - - let numeric_topic_id = shard.streams.with_topic_by_id( - &stream_id, - &topic_id, - streaming::topics::helpers::get_topic_id(), - ); + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&stream_id, &topic_id)?; - // Route request to the correct shard let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); let payload = ShardRequestPayload::DeleteSegments { segments_count }; let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload); @@ -92,8 +79,9 @@ impl ServerCommandHandler for DeleteSegments { }) = message && let ShardRequestPayload::DeleteSegments { segments_count } = payload { + let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?; shard - .delete_segments_base(&stream_id, &topic_id, partition_id, segments_count) + .delete_segments_base(stream, topic, partition_id, segments_count) .await .error(|e: &IggyError| { format!( @@ -114,7 +102,7 @@ impl ServerCommandHandler for DeleteSegments { shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::DeleteSegments(self), ) .await diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs index c2935ef6a3..e48271a25a 100644 --- a/core/server/src/binary/handlers/streams/create_stream_handler.rs +++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs @@ -17,39 +17,36 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::streams::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; -use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker}; use crate::state::command::EntryCommand; use crate::state::models::CreateStreamWithId; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_stream::CreateStream; use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for CreateStream { +impl AuthenticatedHandler for CreateStream { fn code(&self) -> u32 { iggy_common::CREATE_STREAM_CODE } - #[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -60,7 +57,7 @@ impl ServerCommandHandler for CreateStream { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::CreateStream { - user_id: session.get_user_id(), + user_id: auth.user_id(), name: self.name.clone(), }, }; @@ -71,27 +68,19 @@ impl ServerCommandHandler for CreateStream { if let ShardMessage::Request(ShardRequest { payload, .. }) = message && let ShardRequestPayload::CreateStream { name, .. } = payload { + // TODO(hubcio): investigate if we can process commands on the shard that the + // command arrived on, without sending it to shard 0 (use fs locks) + // Acquire stream lock to serialize filesystem operations let _stream_guard = shard.fs_locks.stream_lock.lock().await; - let stream = shard.create_stream(session, name).await?; - let created_stream_id = stream.id(); - - let event = ShardEvent::CreatedStream { - id: created_stream_id, - stream: stream.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; + let created_stream_id = shard.create_stream(session, name).await?; - let response = shard - .streams - .with_components_by_id(created_stream_id, |(root, stats)| { - mapper::map_stream(&root, &stats) - }); + let response = shard.get_stream_from_shared_metadata(created_stream_id); shard .state - .apply(session.get_user_id(), &EntryCommand::CreateStream(CreateStreamWithId { + .apply(auth.user_id(), &EntryCommand::CreateStream(CreateStreamWithId { stream_id: created_stream_id as u32, command: self })) @@ -110,13 +99,12 @@ impl ServerCommandHandler for CreateStream { } } ShardSendRequestResult::Response(response) => match response { - ShardResponse::CreateStreamResponse(stream) => { - let created_stream_id = stream.id(); - let response = mapper::map_stream(stream.root(), stream.stats()); + ShardResponse::CreateStreamResponse(created_stream_id) => { + let response = shard.get_stream_from_shared_metadata(created_stream_id); shard .state - .apply(session.get_user_id(), &EntryCommand::CreateStream(CreateStreamWithId { + .apply(auth.user_id(), &EntryCommand::CreateStream(CreateStreamWithId { stream_id: created_stream_id as u32, command: self })) diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs index d1e23cb04a..63eebe9da3 100644 --- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs +++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs @@ -17,20 +17,19 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::streams::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; -use crate::slab::traits_ext::EntityMarker; + use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_stream::DeleteStream; use iggy_common::{Identifier, IggyError, SenderKind}; @@ -38,16 +37,17 @@ use std::rc::Rc; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeleteStream { +impl AuthenticatedHandler for DeleteStream { fn code(&self) -> u32 { iggy_common::DELETE_STREAM_CODE } - #[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] + #[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -58,7 +58,7 @@ impl ServerCommandHandler for DeleteStream { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::DeleteStream { - user_id: session.get_user_id(), + user_id: auth.user_id(), stream_id: self.stream_id, }, }; @@ -71,7 +71,7 @@ impl ServerCommandHandler for DeleteStream { { // Acquire stream lock to serialize filesystem operations let _stream_guard = shard.fs_locks.stream_lock.lock().await; - let stream = shard + let stream_info = shard .delete_stream(session, &stream_id) .await .error(|e: &IggyError| { @@ -79,19 +79,12 @@ impl ServerCommandHandler for DeleteStream { })?; info!( "Deleted stream with name: {}, ID: {}", - stream.root().name(), - stream.id() + stream_info.name, stream_info.id ); - let event = ShardEvent::DeletedStream { - id: stream.id(), - stream_id: stream_id.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - shard .state - .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: stream_id.clone() })) + .apply(auth.user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: stream_id.clone() })) .await .error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}, session: {session}") @@ -100,10 +93,10 @@ impl ServerCommandHandler for DeleteStream { } } ShardSendRequestResult::Response(response) => match response { - ShardResponse::DeleteStreamResponse(stream) => { + ShardResponse::DeleteStreamResponse(stream_id_num) => { shard .state - .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream.id() as u32).try_into().unwrap() })) + .apply(auth.user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream_id_num as u32).try_into().unwrap() })) .await .error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}, session: {session}") diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs index fe9cd1f83b..b0ccad2011 100644 --- a/core/server/src/binary/handlers/streams/get_stream_handler.rs +++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs @@ -17,15 +17,12 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; use crate::shard::IggyShard; -use crate::slab::traits_ext::EntityComponentSystem; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::streams; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::IggyError; use iggy_common::SenderKind; @@ -33,7 +30,7 @@ use iggy_common::get_stream::GetStream; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetStream { +impl AuthenticatedHandler for GetStream { fn code(&self) -> u32 { iggy_common::GET_STREAM_CODE } @@ -42,28 +39,25 @@ impl ServerCommandHandler for GetStream { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; - let exists = shard.ensure_stream_exists(&self.stream_id).is_ok(); - if !exists { + + let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); - } - let stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); + }; + let has_permission = shard .permissioner - .borrow() - .get_stream(session.get_user_id(), stream_id) + .get_stream(auth.user_id(), numeric_stream_id) .error(|e: &IggyError| { format!( "permission denied to get stream with ID: {} for user with ID: {}, error: {e}", self.stream_id, - session.get_user_id(), + auth.user_id(), ) }) .is_ok(); @@ -71,9 +65,8 @@ impl ServerCommandHandler for GetStream { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); } - let response = shard - .streams - .with_components_by_id(stream_id, |(root, stats)| mapper::map_stream(&root, &stats)); + + let response = shard.get_stream_from_shared_metadata(numeric_stream_id); sender.send_ok_response(&response).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs index f7bf06f2de..b9b42cdab3 100644 --- a/core/server/src/binary/handlers/streams/get_streams_handler.rs +++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs @@ -17,15 +17,13 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::streams::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; use crate::shard::IggyShard; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::IggyError; use iggy_common::SenderKind; @@ -33,7 +31,7 @@ use iggy_common::get_streams::GetStreams; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetStreams { +impl AuthenticatedHandler for GetStreams { fn code(&self) -> u32 { iggy_common::GET_STREAMS_CODE } @@ -42,26 +40,22 @@ impl ServerCommandHandler for GetStreams { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; shard .permissioner - .borrow() - .get_streams(session.get_user_id()) + .get_streams(auth.user_id()) .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to get streams for user {}", - session.get_user_id() + auth.user_id() ) })?; - let response = shard.streams.with_components(|stream_ref| { - let (roots, stats) = stream_ref.into_components(); - mapper::map_streams(&roots, &stats) - }); + let response = shard.get_streams_from_shared_metadata(); sender.send_ok_response(&response).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs index bfa1896139..dd8c00ffb2 100644 --- a/core/server/src/binary/handlers/streams/purge_stream_handler.rs +++ b/core/server/src/binary/handlers/streams/purge_stream_handler.rs @@ -17,32 +17,32 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::streams::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; use crate::shard::transmission::event::ShardEvent; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::purge_stream::PurgeStream; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for PurgeStream { +impl AuthenticatedHandler for PurgeStream { fn code(&self) -> u32 { iggy_common::PURGE_STREAM_CODE } - #[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] + #[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -60,7 +60,7 @@ impl ServerCommandHandler for PurgeStream { shard.broadcast_event_to_all_shards(event).await?; shard .state - .apply(session.get_user_id(), &EntryCommand::PurgeStream(self)) + .apply(auth.user_id(), &EntryCommand::PurgeStream(self)) .await .error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to apply purge stream with id: {stream_id}, session: {session}") diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs index 8e4de65744..745a29fc09 100644 --- a/core/server/src/binary/handlers/streams/update_stream_handler.rs +++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs @@ -17,56 +17,96 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::streams::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, +}; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::update_stream::UpdateStream; -use iggy_common::{IggyError, SenderKind}; +use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for UpdateStream { +impl AuthenticatedHandler for UpdateStream { fn code(&self) -> u32 { iggy_common::UPDATE_STREAM_CODE } - #[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] + #[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - let stream_id = self.stream_id.clone(); - shard - .update_stream(session, &self.stream_id, self.name.clone()) - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to update stream with id: {stream_id}, session: {session}") - })?; - let event = ShardEvent::UpdatedStream { - stream_id: self.stream_id.clone(), - name: self.name.clone(), + let request = ShardRequest { + stream_id: Identifier::default(), + topic_id: Identifier::default(), + partition_id: 0, + payload: ShardRequestPayload::UpdateStream { + user_id: auth.user_id(), + stream_id: self.stream_id.clone(), + name: self.name.clone(), + }, }; - shard.broadcast_event_to_all_shards(event).await?; - shard - .state - .apply(session.get_user_id(), &EntryCommand::UpdateStream(self)) - .await - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}") - })?; - sender.send_empty_ok_response().await?; + + let message = ShardMessage::Request(request); + match shard.send_request_to_shard_or_recoil(None, message).await? { + ShardSendRequestResult::Recoil(message) => { + if let ShardMessage::Request(ShardRequest { payload, .. }) = message + && let ShardRequestPayload::UpdateStream { + stream_id, name, .. + } = payload + { + shard.update_stream(session, &stream_id, name.clone())?; + + shard + .state + .apply(auth.user_id(), &EntryCommand::UpdateStream(self)) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}" + ) + })?; + + sender.send_empty_ok_response().await?; + } else { + unreachable!( + "Expected an UpdateStream request inside of UpdateStream handler, impossible state" + ); + } + } + ShardSendRequestResult::Response(response) => match response { + ShardResponse::UpdateStreamResponse => { + shard + .state + .apply(auth.user_id(), &EntryCommand::UpdateStream(self)) + .await?; + + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => { + return Err(err); + } + _ => unreachable!( + "Expected an UpdateStreamResponse inside of UpdateStream handler, impossible state" + ), + }, + } + Ok(HandlerResult::Finished) } } diff --git a/core/server/src/binary/handlers/system/get_client_handler.rs b/core/server/src/binary/handlers/system/get_client_handler.rs index b9e16aef34..da71329bd8 100644 --- a/core/server/src/binary/handlers/system/get_client_handler.rs +++ b/core/server/src/binary/handlers/system/get_client_handler.rs @@ -17,11 +17,12 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use iggy_common::IggyError; use iggy_common::SenderKind; @@ -29,7 +30,7 @@ use iggy_common::get_client::GetClient; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetClient { +impl AuthenticatedHandler for GetClient { fn code(&self) -> u32 { iggy_common::GET_CLIENT_CODE } @@ -38,6 +39,7 @@ impl ServerCommandHandler for GetClient { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/system/get_clients_handler.rs b/core/server/src/binary/handlers/system/get_clients_handler.rs index 81ac21b5f3..6288cf969b 100644 --- a/core/server/src/binary/handlers/system/get_clients_handler.rs +++ b/core/server/src/binary/handlers/system/get_clients_handler.rs @@ -17,12 +17,13 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::system::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use err_trail::ErrContext; use iggy_common::IggyError; @@ -31,7 +32,7 @@ use iggy_common::get_clients::GetClients; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetClients { +impl AuthenticatedHandler for GetClients { fn code(&self) -> u32 { iggy_common::GET_CLIENTS_CODE } @@ -40,6 +41,7 @@ impl ServerCommandHandler for GetClients { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/system/get_me_handler.rs b/core/server/src/binary/handlers/system/get_me_handler.rs index 339063ab30..ba22c9660f 100644 --- a/core/server/src/binary/handlers/system/get_me_handler.rs +++ b/core/server/src/binary/handlers/system/get_me_handler.rs @@ -17,12 +17,13 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::system::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use err_trail::ErrContext; use iggy_common::IggyError; @@ -30,7 +31,7 @@ use iggy_common::SenderKind; use iggy_common::get_me::GetMe; use std::rc::Rc; -impl ServerCommandHandler for GetMe { +impl AuthenticatedHandler for GetMe { fn code(&self) -> u32 { iggy_common::GET_ME_CODE } @@ -39,6 +40,7 @@ impl ServerCommandHandler for GetMe { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/system/get_snapshot.rs b/core/server/src/binary/handlers/system/get_snapshot.rs index cc2163b91c..3e6f76ba90 100644 --- a/core/server/src/binary/handlers/system/get_snapshot.rs +++ b/core/server/src/binary/handlers/system/get_snapshot.rs @@ -17,10 +17,11 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use bytes::Bytes; use iggy_common::IggyError; @@ -29,7 +30,7 @@ use iggy_common::get_snapshot::GetSnapshot; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetSnapshot { +impl AuthenticatedHandler for GetSnapshot { fn code(&self) -> u32 { iggy_common::GET_SNAPSHOT_FILE_CODE } @@ -38,6 +39,7 @@ impl ServerCommandHandler for GetSnapshot { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/system/get_stats_handler.rs b/core/server/src/binary/handlers/system/get_stats_handler.rs index 2fe368ce14..2284295c17 100644 --- a/core/server/src/binary/handlers/system/get_stats_handler.rs +++ b/core/server/src/binary/handlers/system/get_stats_handler.rs @@ -17,7 +17,7 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::system::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; @@ -27,6 +27,7 @@ use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use err_trail::ErrContext; use iggy_common::SenderKind; @@ -35,7 +36,7 @@ use iggy_common::{Identifier, IggyError}; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetStats { +impl AuthenticatedHandler for GetStats { fn code(&self) -> u32 { iggy_common::GET_STATS_CODE } @@ -44,6 +45,7 @@ impl ServerCommandHandler for GetStats { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -55,7 +57,7 @@ impl ServerCommandHandler for GetStats { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::GetStats { - user_id: session.get_user_id(), + user_id: auth.user_id(), }, }; diff --git a/core/server/src/binary/handlers/system/ping_handler.rs b/core/server/src/binary/handlers/system/ping_handler.rs index e22fa7ba8d..a7d777c283 100644 --- a/core/server/src/binary/handlers/system/ping_handler.rs +++ b/core/server/src/binary/handlers/system/ping_handler.rs @@ -16,10 +16,9 @@ * under the License. */ -use crate::binary::command::{BinaryServerCommand, HandlerResult, ServerCommandHandler}; +use crate::binary::command::{BinaryServerCommand, HandlerResult, UnauthenticatedHandler}; use crate::shard::IggyShard; use crate::streaming::session::Session; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::IggyTimestamp; use iggy_common::SenderKind; @@ -27,7 +26,7 @@ use iggy_common::ping::Ping; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for Ping { +impl UnauthenticatedHandler for Ping { fn code(&self) -> u32 { iggy_common::PING_CODE } diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs index fa5c0270a6..fcf7771fe2 100644 --- a/core/server/src/binary/handlers/topics/create_topic_handler.rs +++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs @@ -17,40 +17,38 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::topics::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; - use crate::shard::IggyShard; use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; -use crate::slab::traits_ext::EntityMarker; + use crate::state::command::EntryCommand; use crate::state::models::CreateTopicWithId; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::streams; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_topic::CreateTopic; use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for CreateTopic { +impl AuthenticatedHandler for CreateTopic { fn code(&self) -> u32 { iggy_common::CREATE_TOPIC_CODE } - #[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] + #[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))] async fn handle( mut self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -61,7 +59,7 @@ impl ServerCommandHandler for CreateTopic { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::CreateTopic { - user_id: session.get_user_id(), + user_id: auth.user_id(), stream_id: self.stream_id.clone(), name: self.name.clone(), partitions_count: self.partitions_count, @@ -90,7 +88,7 @@ impl ServerCommandHandler for CreateTopic { // Acquire topic lock to serialize filesystem operations let _topic_guard = shard.fs_locks.topic_lock.lock().await; - let topic = shard + let topic_id = shard .create_topic( session, &stream_id, @@ -101,21 +99,24 @@ impl ServerCommandHandler for CreateTopic { replication_factor, ) .await?; - self.message_expiry = topic.root().message_expiry(); - self.max_topic_size = topic.root().max_topic_size(); let stream_id_num = shard - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - let topic_id = topic.id(); - - let event = ShardEvent::CreatedTopic { - stream_id: stream_id.clone(), - topic, - }; + .metadata + .get_stream_id(&stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - shard.broadcast_event_to_all_shards(event).await?; - let partitions = shard + // Get topic metadata to update command fields + let metadata = shard.metadata.load(); + if let Some(topic_meta) = metadata + .streams + .get(stream_id_num) + .and_then(|s| s.topics.get(topic_id)) + { + self.message_expiry = topic_meta.message_expiry; + self.max_topic_size = topic_meta.max_topic_size; + } + + let partition_infos = shard .create_partitions( session, &stream_id, @@ -126,18 +127,14 @@ impl ServerCommandHandler for CreateTopic { let event = ShardEvent::CreatedPartitions { stream_id: stream_id.clone(), topic_id: Identifier::numeric(topic_id as u32).unwrap(), - partitions, + partitions: partition_infos.clone(), }; shard.broadcast_event_to_all_shards(event).await?; - let response = shard.streams.with_topic_by_id( - &stream_id, - &Identifier::numeric(topic_id as u32).unwrap(), - |(root, _, stats)| mapper::map_topic(&root, &stats), - ); + let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id); shard .state - .apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId { + .apply(auth.user_id(), &EntryCommand::CreateTopic(CreateTopicWithId { topic_id: topic_id as u32, command: self })) @@ -155,31 +152,35 @@ impl ServerCommandHandler for CreateTopic { } } ShardSendRequestResult::Response(response) => match response { - ShardResponse::CreateTopicResponse(topic) => { - let topic_id = topic.id(); - self.message_expiry = topic.root().message_expiry(); - self.max_topic_size = topic.root().max_topic_size(); + ShardResponse::CreateTopicResponse(topic_id) => { + let stream_id_num = shard + .metadata + .get_stream_id(&self.stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(self.stream_id.clone()))?; - let stream_id = shard + // Get topic metadata to update command fields + let metadata = shard.metadata.load(); + if let Some(topic_meta) = metadata .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); + .get(stream_id_num) + .and_then(|s| s.topics.get(topic_id)) + { + self.message_expiry = topic_meta.message_expiry; + self.max_topic_size = topic_meta.max_topic_size; + } - let response = shard.streams.with_topic_by_id( - &self.stream_id, - &Identifier::numeric(topic_id as u32).unwrap(), - |(root, _, stats)| mapper::map_topic(&root, &stats), - ); + let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id); shard .state - .apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId { + .apply(auth.user_id(), &EntryCommand::CreateTopic(CreateTopicWithId { topic_id: topic_id as u32, command: self })) .await .error(|e: &IggyError| { format!( - "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id}, topic_id: {topic_id:?}" + "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id_num}, topic_id: {topic_id:?}" ) })?; diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs index 668dbd7075..508b042e63 100644 --- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs +++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs @@ -17,22 +17,20 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::topics::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; -use crate::slab::traits_ext::EntityMarker; + use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::streams; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_topic::DeleteTopic; use iggy_common::{Identifier, IggyError, SenderKind}; @@ -40,16 +38,17 @@ use std::rc::Rc; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeleteTopic { +impl AuthenticatedHandler for DeleteTopic { fn code(&self) -> u32 { iggy_common::DELETE_TOPIC_CODE } - #[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -60,7 +59,7 @@ impl ServerCommandHandler for DeleteTopic { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::DeleteTopic { - user_id: session.get_user_id(), + user_id: auth.user_id(), stream_id: self.stream_id.clone(), topic_id: self.topic_id.clone(), }, @@ -79,28 +78,17 @@ impl ServerCommandHandler for DeleteTopic { // Acquire topic lock to serialize filesystem operations let _topic_guard = shard.fs_locks.topic_lock.lock().await; - let topic = shard.delete_topic(session, &stream_id, &topic_id).await?; - let stream_id_num = shard - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - let topic_id_num = topic.root().id(); + let topic_info = shard.delete_topic(session, &stream_id, &topic_id).await?; + let topic_id_num = topic_info.id; + let stream_id_num = topic_info.stream_id; info!( "Deleted topic with name: {}, ID: {} in stream with ID: {}", - topic.root().name(), - topic_id_num, - stream_id_num + topic_info.name, topic_id_num, stream_id_num ); - let event = ShardEvent::DeletedTopic { - id: topic_id_num, - stream_id: stream_id.clone(), - topic_id: topic_id.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - shard .state - .apply(session.get_user_id(), &EntryCommand::DeleteTopic(self)) + .apply(auth.user_id(), &EntryCommand::DeleteTopic(self)) .await .error(|e: &IggyError| format!( "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {topic_id_num} in stream with ID: {stream_id_num}, session: {session}", @@ -113,14 +101,13 @@ impl ServerCommandHandler for DeleteTopic { } } ShardSendRequestResult::Response(response) => match response { - ShardResponse::DeleteTopicResponse(topic) => { + ShardResponse::DeleteTopicResponse(topic_id_num) => { shard .state - .apply(session.get_user_id(), &EntryCommand::DeleteTopic(self)) + .apply(auth.user_id(), &EntryCommand::DeleteTopic(self)) .await .error(|e: &IggyError| format!( - "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {}, session: {session}", - topic.id() + "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {topic_id_num}, session: {session}" ))?; sender.send_empty_ok_response().await?; diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs index fcc42ac0b3..b3af68ae4c 100644 --- a/core/server/src/binary/handlers/topics/get_topic_handler.rs +++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs @@ -17,21 +17,19 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::streams; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_topic::GetTopic; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetTopic { +impl AuthenticatedHandler for GetTopic { fn code(&self) -> u32 { iggy_common::GET_TOPIC_CODE } @@ -40,46 +38,35 @@ impl ServerCommandHandler for GetTopic { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; - let exists = shard - .ensure_topic_exists(&self.stream_id, &self.topic_id) - .is_ok(); - if !exists { + + let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); - } + }; + + let Some(numeric_topic_id) = shard + .metadata + .get_topic_id(numeric_stream_id, &self.topic_id) + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; - let numeric_stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); let has_permission = shard .permissioner - .borrow() - .get_topic( - session.get_user_id(), - numeric_stream_id, - self.topic_id - .get_u32_value() - .unwrap_or(0) - .try_into() - .unwrap(), - ) + .get_topic(auth.user_id(), numeric_stream_id, numeric_topic_id) .is_ok(); if !has_permission { sender.send_empty_ok_response().await?; return Ok(HandlerResult::Finished); } - let response = - shard - .streams - .with_topic_by_id(&self.stream_id, &self.topic_id, |(root, _, stats)| { - mapper::map_topic(&root, &stats) - }); + let response = shard.get_topic_from_shared_metadata(numeric_stream_id, numeric_topic_id); sender.send_ok_response(&response).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs index 96c3f93250..f171ec5245 100644 --- a/core/server/src/binary/handlers/topics/get_topics_handler.rs +++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs @@ -17,22 +17,19 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; -use crate::binary::mapper; use crate::shard::IggyShard; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::streams; -use anyhow::Result; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_topics::GetTopics; use std::rc::Rc; use tracing::debug; -impl ServerCommandHandler for GetTopics { +impl AuthenticatedHandler for GetTopics { fn code(&self) -> u32 { iggy_common::GET_TOPICS_CODE } @@ -41,26 +38,23 @@ impl ServerCommandHandler for GetTopics { self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard.ensure_authenticated(session)?; shard.ensure_stream_exists(&self.stream_id)?; - let numeric_stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); + + let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + shard .permissioner - .borrow() - .get_topics(session.get_user_id(), numeric_stream_id)?; + .get_topics(auth.user_id(), numeric_stream_id)?; - let response = shard.streams.with_topics(&self.stream_id, |topics| { - topics.with_components(|topics| { - let (roots, _, stats) = topics.into_components(); - mapper::map_topics(&roots, &stats) - }) - }); + let response = shard.get_topics_from_shared_metadata(numeric_stream_id); sender.send_ok_response(&response).await?; Ok(HandlerResult::Finished) } diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs index 6e86293aeb..950c8d4211 100644 --- a/core/server/src/binary/handlers/topics/purge_topic_handler.rs +++ b/core/server/src/binary/handlers/topics/purge_topic_handler.rs @@ -17,30 +17,31 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::topics::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::purge_topic::PurgeTopic; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for PurgeTopic { +impl AuthenticatedHandler for PurgeTopic { fn code(&self) -> u32 { iggy_common::PURGE_TOPIC_CODE } - #[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -66,7 +67,7 @@ impl ServerCommandHandler for PurgeTopic { shard .state - .apply(session.get_user_id(), &EntryCommand::PurgeTopic(self)) + .apply(auth.user_id(), &EntryCommand::PurgeTopic(self)) .await .error(|e: &IggyError| { format!( diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs index 90f5e11ec2..e2a65bbdd6 100644 --- a/core/server/src/binary/handlers/topics/update_topic_handler.rs +++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs @@ -17,37 +17,36 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::topics::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::update_topic::UpdateTopic; use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, instrument}; -impl ServerCommandHandler for UpdateTopic { +impl AuthenticatedHandler for UpdateTopic { fn code(&self) -> u32 { iggy_common::UPDATE_TOPIC_CODE } - #[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] + #[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))] async fn handle( mut self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -58,7 +57,7 @@ impl ServerCommandHandler for UpdateTopic { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::UpdateTopic { - user_id: session.get_user_id(), + user_id: auth.user_id(), stream_id: self.stream_id.clone(), topic_id: self.topic_id.clone(), name: self.name.clone(), @@ -84,6 +83,10 @@ impl ServerCommandHandler for UpdateTopic { .. } = payload { + // Get numeric IDs BEFORE update (name might change during update) + let (stream_id_num, topic_id_num) = + shard.resolve_topic_id(&stream_id, &topic_id)?; + shard.update_topic( session, &stream_id, @@ -95,42 +98,20 @@ impl ServerCommandHandler for UpdateTopic { replication_factor, )?; - let name_changed = !name.is_empty(); - let lookup_topic_id = if name_changed { - Identifier::named(&name).unwrap() - } else { - topic_id.clone() - }; - - self.message_expiry = shard.streams.with_topic_by_id( - &stream_id, - &lookup_topic_id, - topics::helpers::get_message_expiry(), - ); - self.max_topic_size = shard.streams.with_topic_by_id( - &stream_id, - &lookup_topic_id, - topics::helpers::get_max_topic_size(), - ); - - let stream_id_num = shard + let metadata = shard.metadata.load(); + let topic_meta = metadata .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - - let event = ShardEvent::UpdatedTopic { - stream_id: stream_id.clone(), - topic_id: topic_id.clone(), - name: name.clone(), - message_expiry: self.message_expiry, - compression_algorithm: self.compression_algorithm, - max_topic_size: self.max_topic_size, - replication_factor: self.replication_factor, - }; - shard.broadcast_event_to_all_shards(event).await?; + .get(stream_id_num) + .and_then(|s| s.topics.get(topic_id_num)) + .ok_or_else(|| { + IggyError::TopicIdNotFound(topic_id.clone(), stream_id.clone()) + })?; + self.message_expiry = topic_meta.message_expiry; + self.max_topic_size = topic_meta.max_topic_size; shard .state - .apply(session.get_user_id(), &EntryCommand::UpdateTopic(self)) + .apply(auth.user_id(), &EntryCommand::UpdateTopic(self)) .await .error(|e: &IggyError| format!( "{COMPONENT} (error: {e}) - failed to apply update topic with id: {topic_id} in stream with ID: {stream_id_num}, session: {session}" @@ -145,12 +126,13 @@ impl ServerCommandHandler for UpdateTopic { ShardSendRequestResult::Response(response) => match response { ShardResponse::UpdateTopicResponse => { let stream_id = shard - .streams - .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id()); + .metadata + .get_stream_id(&self.stream_id) + .unwrap_or_default(); shard .state - .apply(session.get_user_id(), &EntryCommand::UpdateTopic(self)) + .apply(auth.user_id(), &EntryCommand::UpdateTopic(self)) .await .error(|e: &IggyError| format!( "{COMPONENT} (error: {e}) - failed to apply update topic in stream with ID: {stream_id}, session: {session}" diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs index 0dadd98bbd..501c5c661e 100644 --- a/core/server/src/binary/handlers/users/change_password_handler.rs +++ b/core/server/src/binary/handlers/users/change_password_handler.rs @@ -17,82 +17,139 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, +}; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use crate::streaming::utils::crypto; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::change_password::ChangePassword; -use iggy_common::{IggyError, SenderKind}; +use iggy_common::{Identifier, IggyError, SenderKind}; use std::rc::Rc; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for ChangePassword { +impl AuthenticatedHandler for ChangePassword { fn code(&self) -> u32 { iggy_common::CHANGE_PASSWORD_CODE } - #[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - info!("Changing password for user with ID: {}...", self.user_id); - shard - .change_password( - session, - &self.user_id, - &self.current_password, - &self.new_password, - ) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to change password for user_id: {}, session: {session}", - self.user_id - ) - })?; + let user_id_for_log = self.user_id.clone(); + let new_password_hash = crypto::hash_password(&self.new_password); + let request = ShardRequest { + stream_id: Identifier::default(), + topic_id: Identifier::default(), + partition_id: 0, + payload: ShardRequestPayload::ChangePassword { + session_user_id: auth.user_id(), + user_id: self.user_id.clone(), + current_password: self.current_password.clone(), + new_password: self.new_password.clone(), + }, + }; - info!("Changed password for user with ID: {}.", self.user_id); + let message = ShardMessage::Request(request); + match shard.send_request_to_shard_or_recoil(None, message).await? { + ShardSendRequestResult::Recoil(message) => { + if let ShardMessage::Request(ShardRequest { payload, .. }) = message + && let ShardRequestPayload::ChangePassword { + user_id, + current_password, + new_password, + .. + } = payload + { + info!("Changing password for user with ID: {}...", user_id); - let event = ShardEvent::ChangedPassword { - user_id: self.user_id.clone(), - current_password: self.current_password.clone(), - new_password: self.new_password.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; + let _user_guard = shard.fs_locks.user_lock.lock().await; + shard + .change_password(session, &user_id, ¤t_password, &new_password) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to change password for user_id: {}, session: {session}", + user_id + ) + })?; + + info!("Changed password for user with ID: {}.", user_id_for_log); + + shard + .state + .apply( + auth.user_id(), + &EntryCommand::ChangePassword(ChangePassword { + user_id: user_id_for_log.clone(), + current_password: "".into(), + new_password: new_password_hash.clone(), + }), + ) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}", + user_id_for_log + ) + })?; + + sender.send_empty_ok_response().await?; + } else { + unreachable!( + "Expected a ChangePassword request inside of ChangePassword handler, impossible state" + ); + } + } + ShardSendRequestResult::Response(response) => match response { + ShardResponse::ChangePasswordResponse => { + info!("Changed password for user with ID: {}.", user_id_for_log); + + shard + .state + .apply( + auth.user_id(), + &EntryCommand::ChangePassword(ChangePassword { + user_id: user_id_for_log.clone(), + current_password: "".into(), + new_password: new_password_hash.clone(), + }), + ) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}", + user_id_for_log + ) + })?; + + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => { + return Err(err); + } + _ => unreachable!( + "Expected a ChangePasswordResponse inside of ChangePassword handler, impossible state" + ), + }, + } - // For the security of the system, we hash the password before storing it in metadata. - shard - .state - .apply( - session.get_user_id(), - &EntryCommand::ChangePassword(ChangePassword { - user_id: self.user_id.to_owned(), - current_password: "".into(), - new_password: crypto::hash_password(&self.new_password), - }), - ) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}", - self.user_id - ) - })?; - sender.send_empty_ok_response().await?; Ok(HandlerResult::Finished) } } diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs index 3cace71b3f..467c3a8a17 100644 --- a/core/server/src/binary/handlers/users/create_user_handler.rs +++ b/core/server/src/binary/handlers/users/create_user_handler.rs @@ -17,22 +17,21 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; use crate::state::command::EntryCommand; use crate::state::models::CreateUserWithId; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use crate::streaming::utils::crypto; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::create_user::CreateUser; use iggy_common::{Identifier, IggyError, SenderKind}; @@ -40,16 +39,17 @@ use std::rc::Rc; use tracing::debug; use tracing::instrument; -impl ServerCommandHandler for CreateUser { +impl AuthenticatedHandler for CreateUser { fn code(&self) -> u32 { iggy_common::CREATE_USER_CODE } - #[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -60,7 +60,7 @@ impl ServerCommandHandler for CreateUser { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::CreateUser { - user_id: session.get_user_id(), + user_id: auth.user_id(), username: self.username.clone(), password: self.password.clone(), status: self.status, @@ -91,22 +91,12 @@ impl ServerCommandHandler for CreateUser { })?; let user_id = user.id; - - let event = ShardEvent::CreatedUser { - user_id, - username: username.clone(), - password: password.clone(), - status, - permissions: permissions.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - let response = mapper::map_user(&user); shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::CreateUser(CreateUserWithId { user_id, command: CreateUser { @@ -140,7 +130,7 @@ impl ServerCommandHandler for CreateUser { shard .state .apply( - session.get_user_id(), + auth.user_id(), &EntryCommand::CreateUser(CreateUserWithId { user_id, command: CreateUser { diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs index a7fa3ec1b4..d1a501c2f7 100644 --- a/core/server/src/binary/handlers/users/delete_user_handler.rs +++ b/core/server/src/binary/handlers/users/delete_user_handler.rs @@ -16,39 +16,37 @@ * under the License. */ -use std::rc::Rc; - use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::delete_user::DeleteUser; use iggy_common::{Identifier, IggyError, SenderKind}; +use std::rc::Rc; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for DeleteUser { +impl AuthenticatedHandler for DeleteUser { fn code(&self) -> u32 { iggy_common::DELETE_USER_CODE } - #[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -59,7 +57,7 @@ impl ServerCommandHandler for DeleteUser { topic_id: Identifier::default(), partition_id: 0, payload: ShardRequestPayload::DeleteUser { - session_user_id: session.get_user_id(), + session_user_id: auth.user_id(), user_id: self.user_id, }, }; @@ -81,8 +79,6 @@ impl ServerCommandHandler for DeleteUser { })?; info!("Deleted user: {} with ID: {}.", user.username, user.id); - let event = ShardEvent::DeletedUser { user_id }; - shard.broadcast_event_to_all_shards(event).await?; shard .state diff --git a/core/server/src/binary/handlers/users/get_user_handler.rs b/core/server/src/binary/handlers/users/get_user_handler.rs index 97322a3406..d34792b4e1 100644 --- a/core/server/src/binary/handlers/users/get_user_handler.rs +++ b/core/server/src/binary/handlers/users/get_user_handler.rs @@ -19,18 +19,19 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use iggy_common::IggyError; use iggy_common::SenderKind; use iggy_common::get_user::GetUser; use tracing::debug; -impl ServerCommandHandler for GetUser { +impl AuthenticatedHandler for GetUser { fn code(&self) -> u32 { iggy_common::GET_USER_CODE } @@ -39,6 +40,7 @@ impl ServerCommandHandler for GetUser { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/users/get_users_handler.rs b/core/server/src/binary/handlers/users/get_users_handler.rs index bca683a0ac..350d9de6da 100644 --- a/core/server/src/binary/handlers/users/get_users_handler.rs +++ b/core/server/src/binary/handlers/users/get_users_handler.rs @@ -19,12 +19,13 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; use err_trail::ErrContext; use iggy_common::IggyError; @@ -32,7 +33,7 @@ use iggy_common::SenderKind; use iggy_common::get_users::GetUsers; use tracing::debug; -impl ServerCommandHandler for GetUsers { +impl AuthenticatedHandler for GetUsers { fn code(&self) -> u32 { iggy_common::GET_USERS_CODE } @@ -41,6 +42,7 @@ impl ServerCommandHandler for GetUsers { self, sender: &mut SenderKind, _length: u32, + _auth: Auth, session: &Session, shard: &Rc, ) -> Result { diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs index 550be02b70..ed7fa3252c 100644 --- a/core/server/src/binary/handlers/users/login_user_handler.rs +++ b/core/server/src/binary/handlers/users/login_user_handler.rs @@ -17,21 +17,20 @@ */ use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + BinaryServerCommand, HandlerResult, ServerCommand, UnauthenticatedHandler, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::binary::mapper; use crate::shard::IggyShard; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::login_user::LoginUser; use iggy_common::{IggyError, SenderKind}; use std::rc::Rc; use tracing::{debug, info, instrument, warn}; -impl ServerCommandHandler for LoginUser { +impl UnauthenticatedHandler for LoginUser { fn code(&self) -> u32 { iggy_common::LOGIN_USER_CODE } diff --git a/core/server/src/binary/handlers/users/logout_user_handler.rs b/core/server/src/binary/handlers/users/logout_user_handler.rs index 714815173c..ff771c1307 100644 --- a/core/server/src/binary/handlers/users/logout_user_handler.rs +++ b/core/server/src/binary/handlers/users/logout_user_handler.rs @@ -19,39 +19,40 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::logout_user::LogoutUser; use iggy_common::{IggyError, SenderKind}; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for LogoutUser { +impl AuthenticatedHandler for LogoutUser { fn code(&self) -> u32 { iggy_common::LOGOUT_USER_CODE } - #[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - info!("Logging out user with ID: {}...", session.get_user_id()); + info!("Logging out user with ID: {}...", auth.user_id()); shard.logout_user(session).error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to logout user, session: {session}") })?; - info!("Logged out user with ID: {}.", session.get_user_id()); + info!("Logged out user with ID: {}.", auth.user_id()); session.clear_user_id(); sender.send_empty_ok_response().await?; Ok(HandlerResult::Finished) diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs index 30829a8401..e612153e0f 100644 --- a/core/server/src/binary/handlers/users/update_permissions_handler.rs +++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs @@ -19,57 +19,118 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; - use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, +}; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::update_permissions::UpdatePermissions; -use iggy_common::{IggyError, SenderKind}; +use iggy_common::{Identifier, IggyError, SenderKind}; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for UpdatePermissions { +impl AuthenticatedHandler for UpdatePermissions { fn code(&self) -> u32 { iggy_common::UPDATE_PERMISSIONS_CODE } - #[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { debug!("session: {session}, command: {self}"); - shard - .update_permissions(session, &self.user_id, self.permissions.clone()) - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to update permissions for user_id: {}, session: {session}", - self.user_id - ))?; - info!("Updated permissions for user with ID: {}.", self.user_id); - let event = ShardEvent::UpdatedPermissions { - user_id: self.user_id.clone(), - permissions: self.permissions.clone(), + let user_id_for_log = self.user_id.clone(); + let request = ShardRequest { + stream_id: Identifier::default(), + topic_id: Identifier::default(), + partition_id: 0, + payload: ShardRequestPayload::UpdatePermissions { + session_user_id: auth.user_id(), + user_id: self.user_id.clone(), + permissions: self.permissions.clone(), + }, }; - shard.broadcast_event_to_all_shards(event).await?; - shard - .state - .apply( - session.get_user_id(), - &EntryCommand::UpdatePermissions(self), - ) - .await?; - sender.send_empty_ok_response().await?; + let message = ShardMessage::Request(request); + match shard.send_request_to_shard_or_recoil(None, message).await? { + ShardSendRequestResult::Recoil(message) => { + if let ShardMessage::Request(ShardRequest { payload, .. }) = message + && let ShardRequestPayload::UpdatePermissions { + user_id, + permissions, + .. + } = payload + { + let _user_guard = shard.fs_locks.user_lock.lock().await; + shard + .update_permissions(session, &user_id, permissions) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to update permissions for user_id: {}, session: {session}", + user_id + ) + })?; + + info!("Updated permissions for user with ID: {}.", user_id_for_log); + + shard + .state + .apply(auth.user_id(), &EntryCommand::UpdatePermissions(self)) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply update permissions for user_id: {}, session: {session}", + user_id_for_log + ) + })?; + + sender.send_empty_ok_response().await?; + } else { + unreachable!( + "Expected an UpdatePermissions request inside of UpdatePermissions handler, impossible state" + ); + } + } + ShardSendRequestResult::Response(response) => match response { + ShardResponse::UpdatePermissionsResponse => { + info!("Updated permissions for user with ID: {}.", user_id_for_log); + + shard + .state + .apply(auth.user_id(), &EntryCommand::UpdatePermissions(self)) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply update permissions for user_id: {}, session: {session}", + user_id_for_log + ) + })?; + + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => { + return Err(err); + } + _ => unreachable!( + "Expected an UpdatePermissionsResponse inside of UpdatePermissions handler, impossible state" + ), + }, + } + Ok(HandlerResult::Finished) } } diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs index bdb9291360..b665537639 100644 --- a/core/server/src/binary/handlers/users/update_user_handler.rs +++ b/core/server/src/binary/handlers/users/update_user_handler.rs @@ -19,32 +19,32 @@ use std::rc::Rc; use crate::binary::command::{ - BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler, + AuthenticatedHandler, BinaryServerCommand, HandlerResult, ServerCommand, }; use crate::binary::handlers::users::COMPONENT; use crate::binary::handlers::utils::receive_and_validate; use crate::shard::IggyShard; -use crate::shard::transmission::event::ShardEvent; use crate::state::command::EntryCommand; +use crate::streaming::auth::Auth; use crate::streaming::session::Session; -use anyhow::Result; use err_trail::ErrContext; use iggy_common::update_user::UpdateUser; use iggy_common::{IggyError, SenderKind}; use tracing::info; use tracing::{debug, instrument}; -impl ServerCommandHandler for UpdateUser { +impl AuthenticatedHandler for UpdateUser { fn code(&self) -> u32 { iggy_common::UPDATE_USER_CODE } - #[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] + #[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = auth.user_id(), iggy_client_id = session.client_id))] async fn handle( self, sender: &mut SenderKind, _length: u32, + auth: Auth, session: &Session, shard: &Rc, ) -> Result { @@ -66,17 +66,10 @@ impl ServerCommandHandler for UpdateUser { info!("Updated user: {} with ID: {}.", user.username, user.id); - let event = ShardEvent::UpdatedUser { - user_id: self.user_id.clone(), - username: self.username.clone(), - status: self.status, - }; - shard.broadcast_event_to_all_shards(event).await?; - let user_id = self.user_id.clone(); shard .state - .apply(session.get_user_id(), &EntryCommand::UpdateUser(self)) + .apply(auth.user_id(), &EntryCommand::UpdateUser(self)) .await .error(|e: &IggyError| { format!( diff --git a/core/server/src/binary/macros.rs b/core/server/src/binary/macros.rs index a56c0bee0d..d88fbe75b2 100644 --- a/core/server/src/binary/macros.rs +++ b/core/server/src/binary/macros.rs @@ -16,37 +16,66 @@ * under the License. */ -/// This macro does 4 expansions in one go: +/// This macro generates the ServerCommand enum and associated dispatch logic. /// -/// 1) The `#[enum_dispatch(ServerCommandHandler)] pub enum ServerCommand` -/// with all variants. -/// 2) The `from_code_and_payload(code, payload)` function that matches each code. -/// 3) The `to_bytes()` function that matches each variant. -/// 4) The `validate()` function that matches each variant. +/// It performs the following expansions: +/// 1) The `pub enum ServerCommand` with all variants +/// 2) The `from_code_and_payload(code, payload)` function +/// 3) The `from_code_and_reader` async function +/// 4) The `to_bytes()` function +/// 5) The `validate()` function +/// 6) The `dispatch()` function that performs authentication and calls handlers +/// 7) The `code()` method #[macro_export] macro_rules! define_server_command_enum { ( - $( - // Macro pattern: - // variant name inner type numeric code display string show_payload? - $variant:ident ( $ty:ty ), $code:ident, $display_str:expr, $show_payload:expr - );* $(;)? + @unauth { + $( + $unauth_variant:ident ( $unauth_ty:ty ), $unauth_code:ident, $unauth_display:expr, $unauth_show_payload:expr + );* $(;)? + } + @auth { + $( + $auth_variant:ident ( $auth_ty:ty ), $auth_code:ident, $auth_display:expr, $auth_show_payload:expr + );* $(;)? + } ) => { - #[enum_dispatch(ServerCommandHandler)] #[derive(Debug, PartialEq, EnumString)] pub enum ServerCommand { + // Unauthenticated variants + $( + $unauth_variant($unauth_ty), + )* + // Authenticated variants $( - $variant($ty), + $auth_variant($auth_ty), )* } impl ServerCommand { + /// Returns the command code. + pub fn code(&self) -> u32 { + match self { + $( + ServerCommand::$unauth_variant(_) => $unauth_code, + )* + $( + ServerCommand::$auth_variant(_) => $auth_code, + )* + } + } + /// Constructs a `ServerCommand` from its numeric code and payload. pub fn from_code_and_payload(code: u32, payload: Bytes) -> Result { match code { $( - $code => Ok(ServerCommand::$variant( - <$ty>::from_bytes(payload)? + $unauth_code => Ok(ServerCommand::$unauth_variant( + <$unauth_ty>::from_bytes(payload)? + )), + )* + $( + $auth_code => Ok(ServerCommand::$auth_variant( + <$auth_ty>::from_bytes(payload)? )), )* _ => { @@ -64,8 +93,13 @@ macro_rules! define_server_command_enum { ) -> Result { match code { $( - $code => Ok(ServerCommand::$variant( - <$ty as BinaryServerCommand>::from_sender(sender, code, length).await? + $unauth_code => Ok(ServerCommand::$unauth_variant( + <$unauth_ty as BinaryServerCommand>::from_sender(sender, code, length).await? + )), + )* + $( + $auth_code => Ok(ServerCommand::$auth_variant( + <$auth_ty as BinaryServerCommand>::from_sender(sender, code, length).await? )), )* _ => Err(IggyError::InvalidCommand), @@ -76,16 +110,65 @@ macro_rules! define_server_command_enum { pub fn to_bytes(&self) -> Bytes { match self { $( - ServerCommand::$variant(payload) => as_bytes(payload), + ServerCommand::$unauth_variant(payload) => as_bytes(payload), + )* + $( + ServerCommand::$auth_variant(payload) => as_bytes(payload), )* } } - /// Validate the command by delegating to the inner command’s implementation. + /// Validate the command by delegating to the inner command's implementation. pub fn validate(&self) -> Result<(), IggyError> { match self { $( - ServerCommand::$variant(cmd) => <$ty as iggy_common::Validatable>::validate(cmd), + ServerCommand::$unauth_variant(cmd) => <$unauth_ty as iggy_common::Validatable>::validate(cmd), + )* + $( + ServerCommand::$auth_variant(cmd) => <$auth_ty as iggy_common::Validatable>::validate(cmd), + )* + } + } + + /// Dispatch the command to its handler. + /// + /// For authenticated commands, this performs authentication first and passes + /// the `Auth` proof token to the handler. + /// + /// For unauthenticated commands, the handler is called directly. + pub async fn dispatch( + self, + sender: &mut SenderKind, + length: u32, + session: &Session, + shard: &Rc, + ) -> Result { + match self { + // Unauthenticated commands - call handler directly + $( + ServerCommand::$unauth_variant(cmd) => { + <$unauth_ty as UnauthenticatedHandler>::handle( + cmd, + sender, + length, + session, + shard, + ).await + } + )* + // Authenticated commands - authenticate first, then call handler + $( + ServerCommand::$auth_variant(cmd) => { + let auth = shard.auth(session)?; + <$auth_ty as AuthenticatedHandler>::handle( + cmd, + sender, + length, + auth, + session, + shard, + ).await + } )* } } @@ -96,13 +179,20 @@ macro_rules! define_server_command_enum { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { $( - // If $show_payload is true, we display the command name + payload; - // otherwise, we just display the command name. - ServerCommand::$variant(payload) => { - if $show_payload { - write!(formatter, "{}|{payload:?}", $display_str) + ServerCommand::$unauth_variant(payload) => { + if $unauth_show_payload { + write!(formatter, "{}|{payload:?}", $unauth_display) + } else { + write!(formatter, "{}", $unauth_display) + } + }, + )* + $( + ServerCommand::$auth_variant(payload) => { + if $auth_show_payload { + write!(formatter, "{}|{payload:?}", $auth_display) } else { - write!(formatter, "{}", $display_str) + write!(formatter, "{}", $auth_display) } }, )* @@ -110,5 +200,4 @@ macro_rules! define_server_command_enum { } } }; - } diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs index 6e98081d6e..89dac029b4 100644 --- a/core/server/src/binary/mapper.rs +++ b/core/server/src/binary/mapper.rs @@ -16,23 +16,13 @@ * under the License. */ -use std::sync::{Arc, atomic::AtomicU64}; - -use crate::slab::Keyed; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; +use crate::metadata::ConsumerGroupMeta; use crate::streaming::clients::client_manager::Client; -use crate::streaming::partitions::partition::PartitionRoot; -use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; -use crate::streaming::streams::stream; -use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot, Member}; -use crate::streaming::topics::topic::{self, TopicRoot}; use crate::streaming::users::user::User; -use arcshift::SharedGetGuard; use bytes::{BufMut, Bytes, BytesMut}; use iggy_common::{ BytesSerializable, ConsumerOffsetInfo, PersonalAccessToken, Stats, TransportProtocol, UserId, }; -use slab::Slab; pub fn map_stats(stats: &Stats) -> Bytes { let mut bytes = BytesMut::with_capacity(104); @@ -153,145 +143,29 @@ pub fn map_personal_access_tokens(personal_access_tokens: Vec, stats: &Slab>) -> Bytes { +/// Map consumer group from SharedMetadata format. +pub fn map_consumer_group_from_meta(meta: &ConsumerGroupMeta) -> Bytes { let mut bytes = BytesMut::new(); - for (root, stat) in roots - .iter() - .map(|(_, val)| val) - .zip(stats.iter().map(|(_, val)| val)) - { - extend_stream(root, stat, &mut bytes); - } - bytes.freeze() -} -pub fn map_stream(root: &stream::StreamRoot, stats: &StreamStats) -> Bytes { - let mut bytes = BytesMut::new(); - extend_stream(root, stats, &mut bytes); - root.topics().with_components(|topics| { - let (roots, _, stats, ..) = topics.into_components(); - for (root, stat) in roots - .iter() - .map(|(_, val)| val) - .zip(stats.iter().map(|(_, val)| val)) - { - extend_topic(root, stat, &mut bytes); - } - }); - bytes.freeze() -} + // Header: id, partitions_count, members_count, name_len, name + bytes.put_u32_le(meta.id as u32); + bytes.put_u32_le(meta.partitions.len() as u32); + bytes.put_u32_le(meta.members.len() as u32); + bytes.put_u8(meta.name.len() as u8); + bytes.put_slice(meta.name.as_bytes()); -pub fn map_topics(roots: &Slab, stats: &Slab>) -> Bytes { - let mut bytes = BytesMut::new(); - for (root, stat) in roots - .iter() - .map(|(_, val)| val) - .zip(stats.iter().map(|(_, val)| val)) - { - extend_topic(root, stat, &mut bytes); - } - bytes.freeze() -} - -pub fn map_topic(root: &topic::TopicRoot, stats: &TopicStats) -> Bytes { - let mut bytes = BytesMut::new(); - extend_topic(root, stats, &mut bytes); - root.partitions().with_components(|partitions| { - let (roots, stats, _, offsets, _, _, _) = partitions.into_components(); - for (root, stat, offset) in roots - .iter() - .map(|(_, val)| val) - .zip(stats.iter().map(|(_, val)| val)) - .zip(offsets.iter().map(|(_, val)| val)) - .map(|((root, stat), offset)| (root, stat, offset)) - { - extend_partition(root, stat, offset, &mut bytes); - } - }); - - bytes.freeze() -} - -pub fn map_consumer_group(root: &ConsumerGroupRoot, members: &ConsumerGroupMembers) -> Bytes { - let mut bytes = BytesMut::new(); - let members = members.inner().shared_get(); - extend_consumer_group(root, &members, &mut bytes); - - for (_, member) in members.iter() { + // Members + for (_, member) in meta.members.iter() { bytes.put_u32_le(member.id as u32); bytes.put_u32_le(member.partitions.len() as u32); - for partition in &member.partitions { - bytes.put_u32_le(*partition as u32); + for &partition_id in &member.partitions { + bytes.put_u32_le(partition_id as u32); } } - bytes.freeze() -} -pub fn map_consumer_groups( - roots: &Slab, - members: &Slab, -) -> Bytes { - let mut bytes = BytesMut::new(); - for (root, member) in roots - .iter() - .map(|(_, val)| val) - .zip(members.iter().map(|(_, val)| val.inner().shared_get())) - { - extend_consumer_group(root, &member, &mut bytes); - } bytes.freeze() } -fn extend_stream(root: &stream::StreamRoot, stats: &StreamStats, bytes: &mut BytesMut) { - bytes.put_u32_le(root.id() as u32); - bytes.put_u64_le(root.created_at().into()); - bytes.put_u32_le(root.topics_count() as u32); - bytes.put_u64_le(stats.size_bytes_inconsistent()); - bytes.put_u64_le(stats.messages_count_inconsistent()); - bytes.put_u8(root.name().len() as u8); - bytes.put_slice(root.name().as_bytes()); -} - -fn extend_topic(root: &TopicRoot, stats: &TopicStats, bytes: &mut BytesMut) { - bytes.put_u32_le(root.id() as u32); - bytes.put_u64_le(root.created_at().into()); - bytes.put_u32_le(root.partitions().len() as u32); - bytes.put_u64_le(root.message_expiry().into()); - bytes.put_u8(root.compression_algorithm().as_code()); - bytes.put_u64_le(root.max_topic_size().into()); - bytes.put_u8(root.replication_factor()); - bytes.put_u64_le(stats.size_bytes_inconsistent()); - bytes.put_u64_le(stats.messages_count_inconsistent()); - bytes.put_u8(root.name().len() as u8); - bytes.put_slice(root.name().as_bytes()); -} - -fn extend_partition( - root: &PartitionRoot, - stats: &PartitionStats, - offset: &Arc, - bytes: &mut BytesMut, -) { - bytes.put_u32_le(root.id() as u32); - bytes.put_u64_le(root.created_at().into()); - bytes.put_u32_le(stats.segments_count_inconsistent()); - bytes.put_u64_le(offset.load(std::sync::atomic::Ordering::Relaxed)); - bytes.put_u64_le(stats.size_bytes_inconsistent()); - bytes.put_u64_le(stats.messages_count_inconsistent()); -} - -fn extend_consumer_group( - root: &ConsumerGroupRoot, - members: &SharedGetGuard<'_, Slab>, - bytes: &mut BytesMut, -) { - bytes.put_u32_le(root.id() as u32); - bytes.put_u32_le(root.partitions().len() as u32); - bytes.put_u32_le(members.len() as u32); - bytes.put_u8(root.key().len() as u8); - bytes.put_slice(root.key().as_bytes()); -} - fn extend_client(client: &Client, bytes: &mut BytesMut) { bytes.put_u32_le(client.session.client_id); bytes.put_u32_le(client.user_id.unwrap_or(u32::MAX)); diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs index 216e7c93aa..f3c0c86b4a 100644 --- a/core/server/src/bootstrap.rs +++ b/core/server/src/bootstrap.rs @@ -27,6 +27,7 @@ use crate::{ system::{INDEX_EXTENSION, LOG_EXTENSION, SystemConfig}, }, io::fs_utils::{self, DirEntry}, + metadata::{ConsumerGroupMeta, Metadata, UserMeta}, server_error::ServerError, shard::{ system::info::SystemInfo, @@ -35,37 +36,23 @@ use crate::{ frame::ShardFrame, }, }, - slab::{ - consumer_groups::ConsumerGroups, partitions::Partitions, streams::Streams, topics::Topics, - traits_ext::IntoComponents, users::Users, - }, - state::system::{StreamState, UserState}, + state::system::{StreamState, TopicState, UserState}, streaming::{ - partitions::{ - consumer_offset::ConsumerOffset, - helpers::create_message_deduplicator, - journal::MemoryMessageJournal, - log::SegmentedLog, - partition, - storage::{load_consumer_group_offsets, load_consumer_offsets}, - }, + partitions::{journal::MemoryMessageJournal, log::SegmentedLog}, persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, - polling_consumer::ConsumerGroupId, segments::{Segment, storage::Storage}, - stats::{PartitionStats, StreamStats, TopicStats}, + stats::PartitionStats, storage::SystemStorage, - streams::stream, - topics::{consumer_group, topic}, users::user::User, utils::{crypto, file::overwrite}, }, versioning::SemanticVersion, }; -use ahash::HashMap; use compio::{fs::create_dir_all, runtime::Runtime}; use err_trail::ErrContext; use iggy_common::{ - IggyByteSize, IggyError, PersonalAccessToken, + IggyByteSize, IggyError, + collections::SegmentedSlab, defaults::{ DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, @@ -74,154 +61,6 @@ use iggy_common::{ use std::{env, path::Path, sync::Arc}; use tracing::{info, warn}; -pub async fn load_streams( - state: impl IntoIterator, - config: &SystemConfig, -) -> Result { - let state: Vec = state.into_iter().collect(); - let mut stream_entries = Vec::with_capacity(state.len()); - - for stream_state in state { - let stream_id = stream_state.id as usize; - info!( - "Loading stream with ID: {}, name: {} from state...", - stream_state.id, stream_state.name - ); - - let stream_stats = Arc::new(StreamStats::default()); - let mut topic_entries = Vec::new(); - - for topic_state in stream_state.topics.into_values() { - let topic_id = topic_state.id as usize; - info!( - "Loading topic with ID: {}, name: {} from state...", - topic_state.id, topic_state.name - ); - - let topic_stats = Arc::new(TopicStats::new(stream_stats.clone())); - - // Build partitions - let mut partition_entries = Vec::new(); - for partition_state in topic_state.partitions.into_values() { - let partition_id = partition_state.id as usize; - info!( - "Loading partition with ID: {}, for topic with ID: {} from state...", - partition_id, topic_id - ); - - let partition = load_partition( - config, - stream_id, - topic_id, - partition_state, - topic_stats.clone(), - ) - .await?; - partition_entries.push((partition_id, partition)); - - info!( - "Loaded partition with ID: {}, for topic with ID: {} from state...", - partition_id, topic_id - ); - } - - // Build consumer groups - let partition_ids: Vec<_> = partition_entries.iter().map(|(id, _)| *id).collect(); - let cg_entries: Vec<_> = topic_state - .consumer_groups - .into_values() - .map(|cg_state| { - info!( - "Loading consumer group with ID: {}, name: {} for topic with ID: {} from state...", - cg_state.id, cg_state.name, topic_id - ); - let cg = consumer_group::ConsumerGroup::new( - cg_state.name.clone(), - Default::default(), - partition_ids.clone(), - ); - info!( - "Loaded consumer group with ID: {}, name: {} for topic with ID: {} from state...", - cg_state.id, cg_state.name, topic_id - ); - (cg_state.id as usize, cg) - }) - .collect(); - - // Build topic with pre-built partitions and consumer groups - let mut topic = topic::Topic::new( - topic_state.name.clone(), - topic_stats, - topic_state.created_at, - topic_state.replication_factor.unwrap_or(1), - topic_state.message_expiry, - topic_state.compression_algorithm, - topic_state.max_topic_size, - ); - - // Decompose, set nested containers, recompose - let (mut root, auxilary, stats) = topic.into_components(); - root.set_partitions(Partitions::from_entries(partition_entries)); - root.set_consumer_groups(ConsumerGroups::from_entries(cg_entries)); - topic = topic::Topic::new_with_components(root, auxilary, stats); - - topic_entries.push((topic_id, topic)); - info!( - "Loaded topic with ID: {}, name: {} from state...", - topic_state.id, topic_state.name - ); - } - - // Build stream with pre-built topics - let mut stream = stream::Stream::new( - stream_state.name.clone(), - stream_stats, - stream_state.created_at, - ); - - // Decompose, set nested containers, recompose - let (mut root, stats) = stream.into_components(); - root.set_topics(Topics::from_entries(topic_entries)); - stream = stream::Stream::new_with_components(root, stats); - - stream_entries.push((stream_id, stream)); - info!( - "Loaded stream with ID: {}, name: {} from state...", - stream_state.id, stream_state.name - ); - } - - Ok(Streams::from_entries(stream_entries)) -} - -pub fn load_users(state: impl IntoIterator) -> Users { - let users = Users::new(); - for user_state in state { - let UserState { - id, - username, - password_hash, - status, - created_at, - permissions, - personal_access_tokens, - } = user_state; - let mut user = User::with_password(id, &username, password_hash, status, permissions); - user.created_at = created_at; - user.personal_access_tokens = personal_access_tokens - .into_values() - .map(|token| { - ( - Arc::new(token.token_hash.clone()), - PersonalAccessToken::raw(id, &token.name, &token.token_hash, token.expiry_at), - ) - }) - .collect(); - users.insert(user); - } - users -} - pub fn create_shard_connections( shard_assignment: &[ShardInfo], ) -> (Vec>, Vec<(u16, StopSender)>) { @@ -620,86 +459,140 @@ pub async fn load_segments( Ok(log) } -async fn load_partition( - config: &SystemConfig, - stream_id: usize, - topic_id: usize, - partition_state: crate::state::system::PartitionState, - parent_stats: Arc, -) -> Result { - let stats = Arc::new(PartitionStats::new(parent_stats)); - let partition_id = partition_state.id; - - let partition_path = config.get_partition_path(stream_id, topic_id, partition_id as usize); - let log_files = collect_log_files(&partition_path).await?; - let should_increment_offset = !log_files.is_empty() - && log_files - .first() - .map(|entry| { - let log_file_name = entry - .path - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); - - let start_offset = log_file_name.parse::().unwrap(); - - let messages_file_path = config.get_messages_file_path( +/// Loads users directly into SharedMetadata without using slabs. +pub fn load_users_to_metadata( + state: impl IntoIterator, + shared_metadata: &Metadata, +) { + for user_state in state { + let UserState { + id, + username, + password_hash, + status, + created_at, + permissions, + personal_access_tokens: _, // Personal access tokens are handled separately + } = user_state; + + let user_meta = UserMeta { + id, + username: Arc::from(username.as_str()), + password_hash: Arc::from(password_hash.as_str()), + status, + permissions: permissions.map(Arc::new), + created_at, + }; + shared_metadata.add_user_with_id(id, user_meta); + } + + let metadata = shared_metadata.load(); + info!("Loaded into SharedMetadata: {} users", metadata.users.len()); +} + +/// Loads stream/topic/partition/consumer_group metadata into SharedMetadata. +/// Does NOT load partition data (logs, segments) - that's done per-shard via load_segments(). +pub fn load_metadata_only( + state: impl IntoIterator, + shared_metadata: &Metadata, +) { + for StreamState { + name, + created_at, + id, + topics, + } in state + { + info!("Loading stream with ID: {}, name: {} metadata...", id, name); + let stream_id = id as usize; + + // Register stream (creates stats + meta atomically) + let _stream_stats = + shared_metadata.register_stream(stream_id, Arc::from(name.as_str()), created_at); + + for TopicState { + id, + name, + created_at, + compression_algorithm, + message_expiry, + max_topic_size, + replication_factor, + consumer_groups, + partitions, + } in topics.into_values() + { + info!("Loading topic with ID: {}, name: {} metadata...", id, name); + let topic_id = id as usize; + let partitions_count = partitions.len() as u32; + + // Register topic (creates stats + meta atomically) + let _topic_stats = shared_metadata.register_topic( + stream_id, + topic_id, + Arc::from(name.as_str()), + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor.unwrap_or(1), + partitions_count, + ); + + // Register partitions (creates stats + meta atomically) + // Note: Partition data (logs, offsets) loaded per-shard via load_segments() + let mut partition_ids = Vec::new(); + for partition_state in partitions.into_values() { + let partition_id = partition_state.id as usize; + partition_ids.push(partition_id); + + let _partition_stats = shared_metadata.register_partition( stream_id, topic_id, - partition_id as usize, - start_offset, + partition_id, + partition_state.created_at, ); - let metadata = std::fs::metadata(&messages_file_path) - .expect("failed to get metadata for first segment in log"); - metadata.len() > 0 - }) - .unwrap_or_else(|| false); - - info!( - "Loading partition with ID: {} for stream with ID: {} and topic with ID: {}, for path: {} from disk...", - partition_id, stream_id, topic_id, partition_path - ); - - // Load consumer offsets - let message_deduplicator = create_message_deduplicator(config); - let consumer_offset_path = - config.get_consumer_offsets_path(stream_id, topic_id, partition_id as usize); - let consumer_group_offsets_path = - config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id as usize); - - let consumer_offset = Arc::new( - load_consumer_offsets(&consumer_offset_path)? - .into_iter() - .map(|offset| (offset.consumer_id as usize, offset)) - .collect::>() - .into(), - ); + } - let consumer_group_offset = Arc::new( - load_consumer_group_offsets(&consumer_group_offsets_path)? - .into_iter() - .collect::>() - .into(), - ); + // Sort partition IDs for consumer group round-robin + partition_ids.sort_unstable(); - let log = Default::default(); - let partition = partition::Partition::new( - partition_state.created_at, - should_increment_offset, - stats, - message_deduplicator, - Arc::new(Default::default()), - consumer_offset, - consumer_group_offset, - log, - ); + // Register consumer groups + for cg_state in consumer_groups.into_values() { + info!( + "Loading consumer group with ID: {}, name: {} for topic with ID: {} metadata...", + cg_state.id, cg_state.name, topic_id + ); + let group_id = cg_state.id as usize; + let cg_meta = ConsumerGroupMeta { + id: group_id, + name: Arc::from(cg_state.name.as_str()), + partitions: partition_ids.clone(), + members: SegmentedSlab::new(), + }; + shared_metadata.add_consumer_group_with_id(stream_id, topic_id, group_id, cg_meta); + } + } + } + let metadata = shared_metadata.load(); + let topics_count: usize = metadata.streams.iter().map(|(_, s)| s.topics.len()).sum(); + let partitions_count: usize = metadata + .streams + .iter() + .flat_map(|(_, s)| s.topics.iter().map(|(_, t)| t.partitions.len())) + .sum(); + let consumer_groups_count: usize = metadata + .streams + .iter() + .flat_map(|(_, s)| s.topics.iter().map(|(_, t)| t.consumer_groups.len())) + .sum(); info!( - "Loaded partition with ID: {} for stream with ID: {} and topic with ID: {}", - partition_id, stream_id, topic_id + "Loaded metadata: {} streams, {} topics, {} partitions, {} consumer groups (version: {})", + metadata.streams.len(), + topics_count, + partitions_count, + consumer_groups_count, + metadata.revision ); - - Ok(partition) } diff --git a/core/server/src/configs/system.rs b/core/server/src/configs/system.rs index c1b089a795..e3e39f3172 100644 --- a/core/server/src/configs/system.rs +++ b/core/server/src/configs/system.rs @@ -16,13 +16,9 @@ * under the License. */ -use crate::configs::server::MemoryPoolConfig; -use crate::slab::partitions; -use crate::slab::streams; -use crate::slab::topics; - use super::cache_indexes::CacheIndexesConfig; use super::sharding::ShardingConfig; +use crate::configs::server::MemoryPoolConfig; use iggy_common::IggyByteSize; use iggy_common::IggyError; use iggy_common::IggyExpiry; @@ -218,9 +214,9 @@ impl SystemConfig { pub fn get_partition_path( &self, - stream_id: streams::ContainerId, - topic_id: topics::ContainerId, - partition_id: partitions::ContainerId, + stream_id: usize, + topic_id: usize, + partition_id: usize, ) -> String { format!( "{}/{}", diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs index 5faf48adb2..99fa2b2340 100644 --- a/core/server/src/http/consumer_groups.rs +++ b/core/server/src/http/consumer_groups.rs @@ -21,7 +21,6 @@ use crate::http::error::CustomError; use crate::http::jwt::json_web_token::Identity; use crate::http::mapper; use crate::http::shared::AppState; -use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents}; use crate::state::command::EntryCommand; use crate::state::models::CreateConsumerGroupWithId; use crate::streaming::polling_consumer::ConsumerGroupId; @@ -64,45 +63,38 @@ async fn get_consumer_group( let identifier_group_id = Identifier::from_str_value(&group_id)?; let session = Session::stateless(identity.user_id, identity.ip_address); - - // Check permissions and existence - state.shard.shard().ensure_authenticated(&session)?; - let exists = state - .shard - .shard() - .ensure_consumer_group_exists( - &identifier_stream_id, - &identifier_topic_id, - &identifier_group_id, - ) - .is_ok(); - if !exists { - return Err(CustomError::ResourceNotFound); - } - - let numeric_topic_id = state.shard.shard().streams.with_topic_by_id( - &identifier_stream_id, - &identifier_topic_id, - crate::streaming::topics::helpers::get_topic_id(), - ); - let numeric_stream_id = state.shard.shard().streams.with_stream_by_id( - &identifier_stream_id, - crate::streaming::streams::helpers::get_stream_id(), - ); - - state - .shard - .shard() - .permissioner - .borrow() - .get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)?; - - let consumer_group = state.shard.shard().streams.with_consumer_group_by_id( - &identifier_stream_id, - &identifier_topic_id, - &identifier_group_id, - |(root, members)| mapper::map_consumer_group(root, members), - ); + let shard = state.shard.shard(); + + let numeric_stream_id = shard + .metadata + .get_stream_id(&identifier_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &identifier_topic_id) + .ok_or(CustomError::ResourceNotFound)?; + + let numeric_group_id = shard + .metadata + .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &identifier_group_id) + .ok_or(CustomError::ResourceNotFound)?; + + shard.permissioner.get_consumer_group( + session.get_user_id(), + numeric_stream_id, + numeric_topic_id, + )?; + + let metadata = shard.metadata.load(); + let cg_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .and_then(|t| t.consumer_groups.get(numeric_group_id)) + .ok_or(CustomError::ResourceNotFound)?; + + let consumer_group = mapper::map_consumer_group_details_from_metadata(cg_meta); Ok(Json(consumer_group)) } @@ -116,41 +108,31 @@ async fn get_consumer_groups( let identifier_topic_id = Identifier::from_str_value(&topic_id)?; let session = Session::stateless(identity.user_id, identity.ip_address); - - // Check permissions and existence - state.shard.shard().ensure_authenticated(&session)?; - state - .shard - .shard() - .ensure_topic_exists(&identifier_stream_id, &identifier_topic_id)?; - - let numeric_topic_id = state.shard.shard().streams.with_topic_by_id( - &identifier_stream_id, - &identifier_topic_id, - crate::streaming::topics::helpers::get_topic_id(), - ); - let numeric_stream_id = state.shard.shard().streams.with_stream_by_id( - &identifier_stream_id, - crate::streaming::streams::helpers::get_stream_id(), - ); - - state - .shard - .shard() - .permissioner - .borrow() - .get_consumer_groups(session.get_user_id(), numeric_stream_id, numeric_topic_id)?; - - let consumer_groups = state.shard.shard().streams.with_consumer_groups( - &identifier_stream_id, - &identifier_topic_id, - |cgs| { - cgs.with_components(|cgs| { - let (roots, members) = cgs.into_components(); - mapper::map_consumer_groups(roots, members) - }) - }, - ); + let shard = state.shard.shard(); + + let numeric_stream_id = shard + .metadata + .get_stream_id(&identifier_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &identifier_topic_id) + .ok_or(CustomError::ResourceNotFound)?; + + shard.permissioner.get_consumer_groups( + session.get_user_id(), + numeric_stream_id, + numeric_topic_id, + )?; + + let metadata = shard.metadata.load(); + let topic_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .ok_or(CustomError::ResourceNotFound)?; + let consumer_groups = mapper::map_consumer_groups_from_metadata(topic_meta); Ok(Json(consumer_groups)) } @@ -169,8 +151,7 @@ async fn create_consumer_group( let session = Session::stateless(identity.user_id, identity.ip_address); - // Create consumer group using the new API - let consumer_group = state.shard.shard().create_consumer_group( + let group_id = state.shard.shard().create_consumer_group( &session, &command.stream_id, &command.topic_id, @@ -178,36 +159,24 @@ async fn create_consumer_group( ) .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to create consumer group, stream ID: {}, topic ID: {}, name: {}", stream_id, topic_id, command.name))?; - let group_id = consumer_group.id(); - - // Send event for consumer group creation - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::CreatedConsumerGroup { - stream_id: command.stream_id.clone(), - topic_id: command.topic_id.clone(), - cg: consumer_group.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&command.stream_id) + .expect("Stream must exist"); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &command.topic_id) + .expect("Topic must exist"); + let metadata = shard.metadata.load(); + let cg_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .expect("Consumer group must exist after creation"); + let consumer_group_details = mapper::map_consumer_group_details_from_metadata(cg_meta); - // Get the created consumer group details - let group_id_identifier = Identifier::numeric(group_id as u32).unwrap(); - let consumer_group_details = state.shard.shard().streams.with_consumer_group_by_id( - &command.stream_id, - &command.topic_id, - &group_id_identifier, - |(root, members)| mapper::map_consumer_group(root, members), - ); - - // Apply state change let entry_command = EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { group_id: group_id as u32, command, @@ -239,8 +208,11 @@ async fn delete_consumer_group( let result = SendWrapper::new(async move { let session = Session::stateless(identity.user_id, identity.ip_address); - // Delete using the new API - let consumer_group = state.shard.shard().delete_consumer_group( + let shard = state.shard.shard(); + let (numeric_stream_id, numeric_topic_id) = + shard.resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?; + + let cg_meta = shard.delete_consumer_group( &session, &identifier_stream_id, &identifier_topic_id, @@ -248,27 +220,13 @@ async fn delete_consumer_group( ) .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {group_id} for topic with ID: {topic_id} in stream with ID: {stream_id}"))?; - let cg_id = consumer_group.id(); - - // Remove all consumer group members from ClientManager - let stream_id_usize = state.shard.shard().streams.with_stream_by_id( - &identifier_stream_id, - crate::streaming::streams::helpers::get_stream_id(), - ); - let topic_id_usize = state.shard.shard().streams.with_topic_by_id( - &identifier_stream_id, - &identifier_topic_id, - crate::streaming::topics::helpers::get_topic_id(), - ); + let cg_id = cg_meta.id; - // TODO: Tech debt, repeated code from `delete_consumer_group_handler.rs` - // Get members from the deleted consumer group and make them leave - let slab = consumer_group.members().inner().shared_get(); - for (_, member) in slab.iter() { - if let Err(err) = state.shard.shard().client_manager.leave_consumer_group( + for (_, member) in cg_meta.members.iter() { + if let Err(err) = shard.client_manager.leave_consumer_group( member.client_id, - stream_id_usize, - topic_id_usize, + numeric_stream_id, + numeric_topic_id, cg_id, ) { tracing::warn!( @@ -280,13 +238,11 @@ async fn delete_consumer_group( } let cg_id_spez = ConsumerGroupId(cg_id); - // Clean up consumer group offsets from all partitions using the specialized method - let partition_ids = consumer_group.partitions(); state.shard.shard().delete_consumer_group_offsets( cg_id_spez, &identifier_stream_id, &identifier_topic_id, - partition_ids, + &cg_meta.partitions, ).await.error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}", @@ -296,26 +252,6 @@ async fn delete_consumer_group( ) })?; - // Send event for consumer group deletion - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::DeletedConsumerGroup { - id: cg_id, - stream_id: identifier_stream_id.clone(), - topic_id: identifier_topic_id.clone(), - group_id: identifier_group_id.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - - // Apply state change let entry_command = EntryCommand::DeleteConsumerGroup(DeleteConsumerGroup { stream_id: identifier_stream_id, topic_id: identifier_topic_id, diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs index e7656a5544..dff4150fd4 100644 --- a/core/server/src/http/consumer_offsets.rs +++ b/core/server/src/http/consumer_offsets.rs @@ -118,6 +118,8 @@ async fn delete_consumer_offset( Path((stream_id, topic_id, consumer_id)): Path<(String, String, String)>, query: Query, ) -> Result { + let stream_id_ident = Identifier::from_str_value(&stream_id)?; + let topic_id_ident = Identifier::from_str_value(&topic_id)?; let consumer = Consumer::new(consumer_id.try_into()?); let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address)); state @@ -125,8 +127,8 @@ async fn delete_consumer_offset( .delete_consumer_offset( &session, consumer, - &query.stream_id, - &query.topic_id, + &stream_id_ident, + &topic_id_ident, query.partition_id, ) .await diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs index 7bcfdb65b6..0cc39d528d 100644 --- a/core/server/src/http/http_shard_wrapper.rs +++ b/core/server/src/http/http_shard_wrapper.rs @@ -149,11 +149,7 @@ impl HttpSafeShard { future.await } - pub async fn create_stream( - &self, - session: &Session, - name: String, - ) -> Result { + pub async fn create_stream(&self, session: &Session, name: String) -> Result { let future = SendWrapper::new(self.shard().create_stream(session, name)); future.await } @@ -302,31 +298,41 @@ impl HttpSafeShard { ) -> Result<(), IggyError> { self.shard().ensure_topic_exists(&stream_id, &topic_id)?; - let partition_id = self.shard().streams.with_topic_by_id( - &stream_id, - &topic_id, - |(root, auxilary, ..)| match partitioning.kind { - PartitioningKind::Balanced => { - let upperbound = root.partitions().len(); - let pid = auxilary.get_next_partition_id(upperbound); - Ok(pid) - } - PartitioningKind::PartitionId => Ok(u32::from_le_bytes( - partitioning.value[..partitioning.length as usize] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize), - PartitioningKind::MessagesKey => { - let upperbound = root.partitions().len(); - Ok( - topics::helpers::calculate_partition_id_by_messages_key_hash( - upperbound, - &partitioning.value, - ), - ) - } - }, - )?; + let numeric_stream_id = self + .shard() + .metadata + .get_stream_id(&stream_id) + .expect("Stream existence already verified"); + let numeric_topic_id = self + .shard() + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .expect("Topic existence already verified"); + let partition_id = match partitioning.kind { + PartitioningKind::Balanced => self + .shard() + .metadata + .get_next_partition_id(numeric_stream_id, numeric_topic_id) + .ok_or(IggyError::TopicIdNotFound( + stream_id.clone(), + topic_id.clone(), + ))?, + PartitioningKind::PartitionId => u32::from_le_bytes( + partitioning.value[..partitioning.length as usize] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize, + PartitioningKind::MessagesKey => { + let partitions_count = self + .shard() + .metadata + .partitions_count(numeric_stream_id, numeric_topic_id); + topics::helpers::calculate_partition_id_by_messages_key_hash( + partitions_count, + &partitioning.value, + ) + } + }; let future = SendWrapper::new(self.shard().append_messages( user_id, diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs index afe2435291..a5a945cb56 100644 --- a/core/server/src/http/mapper.rs +++ b/core/server/src/http/mapper.rs @@ -17,95 +17,17 @@ */ use crate::http::jwt::json_web_token::GeneratedToken; -use crate::slab::Keyed; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; +use crate::metadata::{ConsumerGroupMeta, InnerMetadata, PartitionMeta, StreamMeta, TopicMeta}; use crate::streaming::clients::client_manager::Client; -use crate::streaming::stats::TopicStats; -use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot}; -use crate::streaming::topics::topic::TopicRoot; use crate::streaming::users::user::User; use iggy_common::PersonalAccessToken; use iggy_common::{ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, IggyByteSize}; use iggy_common::{IdentityInfo, PersonalAccessTokenInfo, TokenInfo, TopicDetails}; use iggy_common::{UserInfo, UserInfoDetails}; -use slab::Slab; -use std::sync::Arc; - -/// Map TopicRoot with partitions to TopicDetails for HTTP responses -pub fn map_topic_details(root: &TopicRoot, stats: &TopicStats) -> TopicDetails { - let mut partitions = Vec::new(); - - // Get partition details similar to binary mapper - root.partitions().with_components(|partition_components| { - let (partition_roots, partition_stats, _, offsets, _, _, _) = - partition_components.into_components(); - for (partition_root, partition_stat, offset) in partition_roots - .iter() - .map(|(_, val)| val) - .zip(partition_stats.iter().map(|(_, val)| val)) - .zip(offsets.iter().map(|(_, val)| val)) - .map(|((root, stat), offset)| (root, stat, offset)) - { - partitions.push(iggy_common::Partition { - id: partition_root.id() as u32, - created_at: partition_root.created_at(), - segments_count: partition_stat.segments_count_inconsistent(), - current_offset: offset.load(std::sync::atomic::Ordering::Relaxed), - size: IggyByteSize::from(partition_stat.size_bytes_inconsistent()), - messages_count: partition_stat.messages_count_inconsistent(), - }); - } - }); - - // Sort partitions by ID - partitions.sort_by(|a, b| a.id.cmp(&b.id)); - - TopicDetails { - id: root.id() as u32, - created_at: root.created_at(), - name: root.name().clone(), - size: stats.size_bytes_inconsistent().into(), - messages_count: stats.messages_count_inconsistent(), - partitions_count: partitions.len() as u32, - partitions, - message_expiry: root.message_expiry(), - compression_algorithm: root.compression_algorithm(), - max_topic_size: root.max_topic_size(), - replication_factor: root.replication_factor(), - } -} - -/// Map TopicRoot and TopicStats to Topic for HTTP responses -pub fn map_topic(root: &TopicRoot, stats: &TopicStats) -> iggy_common::Topic { - iggy_common::Topic { - id: root.id() as u32, - created_at: root.created_at(), - name: root.name().clone(), - size: stats.size_bytes_inconsistent().into(), - partitions_count: root.partitions().len() as u32, - messages_count: stats.messages_count_inconsistent(), - message_expiry: root.message_expiry(), - compression_algorithm: root.compression_algorithm(), - max_topic_size: root.max_topic_size(), - replication_factor: root.replication_factor(), - } -} - -/// Map multiple topics from slab components to Vec for HTTP responses -pub fn map_topics_from_components( - roots: &Slab, - stats: &Slab>, -) -> Vec { - let mut topics = roots - .iter() - .map(|(_, root)| root) - .zip(stats.iter().map(|(_, stat)| stat)) - .map(|(root, stat)| map_topic(root, stat)) - .collect::>(); - topics.sort_by(|a, b| a.id.cmp(&b.id)); - topics -} +// ===================================================================== +// User mapping functions +// ===================================================================== pub fn map_user(user: &User) -> UserInfoDetails { UserInfoDetails { @@ -132,13 +54,17 @@ pub fn map_users(users: &[&User]) -> Vec { users_data } +// ===================================================================== +// Personal access token mapping functions +// ===================================================================== + pub fn map_personal_access_tokens( personal_access_tokens: &[PersonalAccessToken], ) -> Vec { let mut personal_access_tokens_data = Vec::with_capacity(personal_access_tokens.len()); for personal_access_token in personal_access_tokens { let personal_access_token = PersonalAccessTokenInfo { - name: personal_access_token.name.as_str().to_owned(), + name: (*personal_access_token.name).to_owned(), expiry_at: personal_access_token.expiry_at, }; personal_access_tokens_data.push(personal_access_token); @@ -147,6 +73,10 @@ pub fn map_personal_access_tokens( personal_access_tokens_data } +// ===================================================================== +// Client mapping functions +// ===================================================================== + pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails { iggy_common::ClientInfoDetails { client_id: client.session.client_id, @@ -183,51 +113,9 @@ pub fn map_clients(clients: &[Client]) -> Vec { all_clients } -pub fn map_consumer_groups( - roots: &slab::Slab, - members: &slab::Slab, -) -> Vec { - let mut groups = Vec::new(); - for (root, member) in roots - .iter() - .map(|(_, val)| val) - .zip(members.iter().map(|(_, val)| val)) - { - let members_guard = member.inner().shared_get(); - let consumer_group = iggy_common::ConsumerGroup { - id: root.id() as u32, - name: root.key().clone(), - partitions_count: root.partitions().len() as u32, - members_count: members_guard.len() as u32, - }; - groups.push(consumer_group); - } - groups.sort_by(|a, b| a.id.cmp(&b.id)); - groups -} - -pub fn map_consumer_group( - root: &ConsumerGroupRoot, - members: &ConsumerGroupMembers, -) -> ConsumerGroupDetails { - let members_guard = members.inner().shared_get(); - let mut consumer_group_details = ConsumerGroupDetails { - id: root.id() as u32, - name: root.key().clone(), - partitions_count: root.partitions().len() as u32, - members_count: members_guard.len() as u32, - members: Vec::new(), - }; - - for (_, member) in members_guard.iter() { - consumer_group_details.members.push(ConsumerGroupMember { - id: member.id as u32, - partitions_count: member.partitions.len() as u32, - partitions: member.partitions.iter().map(|p| *p as u32).collect(), - }); - } - consumer_group_details -} +// ===================================================================== +// Token mapping functions +// ===================================================================== pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> IdentityInfo { IdentityInfo { @@ -239,69 +127,223 @@ pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> Ide } } -/// Map StreamRoot and StreamStats to StreamDetails for HTTP responses -pub fn map_stream_details( - root: &crate::streaming::streams::stream::StreamRoot, - stats: &crate::streaming::stats::StreamStats, -) -> iggy_common::StreamDetails { - // Get topics using the new slab-based API - let topics = root.topics().with_components(|topic_ref| { - let (topic_roots, _topic_auxiliaries, topic_stats) = topic_ref.into_components(); - let mut topics_vec = Vec::new(); - - // Iterate over topics in the stream - for (topic_root, topic_stat) in topic_roots - .iter() - .map(|(_, root)| root) - .zip(topic_stats.iter().map(|(_, stat)| stat)) - { - topics_vec.push(map_topic(topic_root, topic_stat)); +// ===================================================================== +// SharedMetadata-based mapping functions +// ===================================================================== + +/// Map a stream from SharedMetadata to StreamDetails (with topics) +pub fn map_stream_details_from_metadata(stream_meta: &StreamMeta) -> iggy_common::StreamDetails { + // Get topic IDs sorted + let mut topic_ids: Vec<_> = stream_meta.topics.keys().collect(); + topic_ids.sort_unstable(); + + // Map topics + let mut topics = Vec::with_capacity(topic_ids.len()); + for topic_id in topic_ids { + if let Some(topic_meta) = stream_meta.topics.get(topic_id) { + topics.push(map_topic_from_metadata(topic_meta)); } + } - // Sort topics by ID for consistent ordering - topics_vec.sort_by(|a, b| a.id.cmp(&b.id)); - topics_vec - }); + // Aggregate stats + let (total_size, total_messages) = aggregate_stream_stats(stream_meta); iggy_common::StreamDetails { - id: root.id() as u32, - created_at: root.created_at(), - name: root.name().clone(), - topics_count: root.topics_count() as u32, - size: stats.size_bytes_inconsistent().into(), - messages_count: stats.messages_count_inconsistent(), + id: stream_meta.id as u32, + created_at: stream_meta.created_at, + name: stream_meta.name.to_string(), + topics_count: topics.len() as u32, + size: IggyByteSize::from(total_size), + messages_count: total_messages, topics, } } -/// Map StreamRoot and StreamStats to Stream for HTTP responses -pub fn map_stream( - root: &crate::streaming::streams::stream::StreamRoot, - stats: &crate::streaming::stats::StreamStats, -) -> iggy_common::Stream { +/// Map a stream from SharedMetadata to Stream (without topics) +pub fn map_stream_from_metadata(stream_meta: &StreamMeta) -> iggy_common::Stream { + let (total_size, total_messages) = aggregate_stream_stats(stream_meta); + iggy_common::Stream { - id: root.id() as u32, - created_at: root.created_at(), - name: root.name().clone(), - topics_count: root.topics_count() as u32, - size: stats.size_bytes_inconsistent().into(), - messages_count: stats.messages_count_inconsistent(), + id: stream_meta.id as u32, + created_at: stream_meta.created_at, + name: stream_meta.name.to_string(), + topics_count: stream_meta.topics.len() as u32, + size: IggyByteSize::from(total_size), + messages_count: total_messages, } } -/// Map multiple streams from slabs -pub fn map_streams_from_slabs( - roots: &slab::Slab, - stats: &slab::Slab>, -) -> Vec { - let mut streams = Vec::new(); - for (root, stat) in roots - .iter() - .map(|(_, val)| val) - .zip(stats.iter().map(|(_, val)| val)) - { - streams.push(map_stream(root, stat)); +/// Map all streams from SharedMetadata +pub fn map_streams_from_metadata(metadata: &InnerMetadata) -> Vec { + let mut stream_ids: Vec<_> = metadata.streams.keys().collect(); + stream_ids.sort_unstable(); + + let mut streams = Vec::with_capacity(stream_ids.len()); + for stream_id in stream_ids { + if let Some(stream_meta) = metadata.streams.get(stream_id) { + streams.push(map_stream_from_metadata(stream_meta)); + } } - streams.sort_by(|a, b| a.id.cmp(&b.id)); streams } + +/// Map a topic from SharedMetadata to Topic (without partitions) +pub fn map_topic_from_metadata(topic_meta: &TopicMeta) -> iggy_common::Topic { + let (total_size, total_messages) = aggregate_topic_stats(topic_meta); + + iggy_common::Topic { + id: topic_meta.id as u32, + created_at: topic_meta.created_at, + name: topic_meta.name.to_string(), + size: IggyByteSize::from(total_size), + partitions_count: topic_meta.partitions.len() as u32, + messages_count: total_messages, + message_expiry: topic_meta.message_expiry, + compression_algorithm: topic_meta.compression_algorithm, + max_topic_size: topic_meta.max_topic_size, + replication_factor: topic_meta.replication_factor, + } +} + +/// Map all topics for a stream from SharedMetadata +pub fn map_topics_from_metadata(stream_meta: &StreamMeta) -> Vec { + let mut topic_ids: Vec<_> = stream_meta.topics.keys().collect(); + topic_ids.sort_unstable(); + + let mut topics = Vec::with_capacity(topic_ids.len()); + for topic_id in topic_ids { + if let Some(topic_meta) = stream_meta.topics.get(topic_id) { + topics.push(map_topic_from_metadata(topic_meta)); + } + } + topics +} + +/// Map a topic from SharedMetadata to TopicDetails (with partitions) +pub fn map_topic_details_from_metadata(topic_meta: &TopicMeta) -> TopicDetails { + // Get partition IDs sorted + let mut partition_ids: Vec<_> = topic_meta.partitions.keys().collect(); + partition_ids.sort_unstable(); + + // Map partitions + let mut partitions = Vec::with_capacity(partition_ids.len()); + for partition_id in partition_ids { + if let Some(partition_meta) = topic_meta.partitions.get(partition_id) { + partitions.push(map_partition_from_metadata(partition_meta)); + } + } + + // Aggregate stats + let (total_size, total_messages) = aggregate_topic_stats(topic_meta); + + TopicDetails { + id: topic_meta.id as u32, + created_at: topic_meta.created_at, + name: topic_meta.name.to_string(), + size: IggyByteSize::from(total_size), + messages_count: total_messages, + partitions_count: partitions.len() as u32, + partitions, + message_expiry: topic_meta.message_expiry, + compression_algorithm: topic_meta.compression_algorithm, + max_topic_size: topic_meta.max_topic_size, + replication_factor: topic_meta.replication_factor, + } +} + +/// Map a partition from SharedMetadata +pub fn map_partition_from_metadata(partition_meta: &PartitionMeta) -> iggy_common::Partition { + let stats = &partition_meta.stats; + let segments_count = stats.segments_count_inconsistent(); + let size_bytes = stats.size_bytes_inconsistent(); + let messages_count = stats.messages_count_inconsistent(); + let current_offset = stats.current_offset(); + + iggy_common::Partition { + id: partition_meta.id as u32, + created_at: partition_meta.created_at, + segments_count, + current_offset, + size: IggyByteSize::from(size_bytes), + messages_count, + } +} + +/// Map a consumer group from SharedMetadata +pub fn map_consumer_group_from_metadata(cg_meta: &ConsumerGroupMeta) -> iggy_common::ConsumerGroup { + iggy_common::ConsumerGroup { + id: cg_meta.id as u32, + name: cg_meta.name.to_string(), + partitions_count: cg_meta.partitions.len() as u32, + members_count: cg_meta.members.len() as u32, + } +} + +/// Map a consumer group to ConsumerGroupDetails from SharedMetadata +pub fn map_consumer_group_details_from_metadata( + cg_meta: &ConsumerGroupMeta, +) -> ConsumerGroupDetails { + let members: Vec = cg_meta + .members + .iter() + .map(|(_, member)| ConsumerGroupMember { + id: member.id as u32, + partitions_count: member.partitions.len() as u32, + partitions: member.partitions.iter().map(|&p| p as u32).collect(), + }) + .collect(); + + ConsumerGroupDetails { + id: cg_meta.id as u32, + name: cg_meta.name.to_string(), + partitions_count: cg_meta.partitions.len() as u32, + members_count: members.len() as u32, + members, + } +} + +/// Map all consumer groups for a topic from SharedMetadata +pub fn map_consumer_groups_from_metadata( + topic_meta: &TopicMeta, +) -> Vec { + let mut group_ids: Vec<_> = topic_meta.consumer_groups.keys().collect(); + group_ids.sort_unstable(); + + let mut groups = Vec::with_capacity(group_ids.len()); + for group_id in group_ids { + if let Some(cg_meta) = topic_meta.consumer_groups.get(group_id) { + groups.push(map_consumer_group_from_metadata(cg_meta)); + } + } + groups +} + +// ===================================================================== +// Helper functions for stats aggregation +// ===================================================================== + +fn aggregate_stream_stats(stream_meta: &StreamMeta) -> (u64, u64) { + let mut total_size = 0u64; + let mut total_messages = 0u64; + + for (_, topic_meta) in stream_meta.topics.iter() { + for (_, partition_meta) in topic_meta.partitions.iter() { + total_size += partition_meta.stats.size_bytes_inconsistent(); + total_messages += partition_meta.stats.messages_count_inconsistent(); + } + } + + (total_size, total_messages) +} + +fn aggregate_topic_stats(topic_meta: &TopicMeta) -> (u64, u64) { + let mut total_size = 0u64; + let mut total_messages = 0u64; + + for (_, partition_meta) in topic_meta.partitions.iter() { + total_size += partition_meta.stats.size_bytes_inconsistent(); + total_messages += partition_meta.stats.messages_count_inconsistent(); + } + + (total_size, total_messages) +} diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs index 959e1fd2ca..10f3f27a4a 100644 --- a/core/server/src/http/partitions.rs +++ b/core/server/src/http/partitions.rs @@ -60,7 +60,7 @@ async fn create_partitions( let _parititon_guard = state.shard.shard().fs_locks.partition_lock.lock().await; let session = Session::stateless(identity.user_id, identity.ip_address); - let partitions = SendWrapper::new(state.shard.shard().create_partitions( + let partition_infos = SendWrapper::new(state.shard.shard().create_partitions( &session, &command.stream_id, &command.topic_id, @@ -74,7 +74,7 @@ async fn create_partitions( let event = ShardEvent::CreatedPartitions { stream_id: command.stream_id.clone(), topic_id: command.topic_id.clone(), - partitions, + partitions: partition_infos, }; let _responses = shard.broadcast_event_to_all_shards(event).await; Ok::<(), CustomError>(()) diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs index 9054777ea8..9e7afe76b0 100644 --- a/core/server/src/http/streams.rs +++ b/core/server/src/http/streams.rs @@ -20,7 +20,6 @@ use crate::http::COMPONENT; use crate::http::error::CustomError; use crate::http::jwt::json_web_token::Identity; use crate::http::shared::AppState; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; use crate::streaming::session::Session; use axum::extract::{Path, State}; use axum::http::StatusCode; @@ -58,34 +57,29 @@ async fn get_stream( Path(stream_id): Path, ) -> Result, CustomError> { let stream_id = Identifier::from_str_value(&stream_id)?; - let exists = state.shard.shard().ensure_stream_exists(&stream_id).is_ok(); - if !exists { - return Err(CustomError::ResourceNotFound); - } - // Use direct slab access for thread-safe stream retrieval - let stream_details = SendWrapper::new(|| { - state - .shard - .shard() - .streams - .with_stream_by_id(&stream_id, |(root, stats)| { - crate::http::mapper::map_stream_details(&root, &stats) - }) - })(); + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let metadata = shard.metadata.load(); + let stream_meta = metadata + .streams + .get(numeric_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let stream_details = crate::http::mapper::map_stream_details_from_metadata(stream_meta); Ok(Json(stream_details)) } #[debug_handler] async fn get_streams(State(state): State>) -> Result>, CustomError> { - // Use direct slab access for thread-safe streams retrieval - let streams = SendWrapper::new(|| { - state.shard.shard().streams.with_components(|stream_ref| { - let (roots, stats) = stream_ref.into_components(); - crate::http::mapper::map_streams_from_slabs(&roots, &stats) - }) - })(); + let shard = state.shard.shard(); + let metadata = shard.metadata.load(); + let streams = crate::http::mapper::map_streams_from_metadata(&metadata); Ok(Json(streams)) } @@ -104,7 +98,7 @@ async fn create_stream( let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await; // Create stream using wrapper method - let stream = state + let created_stream_id = state .shard .create_stream(&session, command.name.clone()) .await @@ -115,23 +109,6 @@ async fn create_stream( ) })?; - let created_stream_id = stream.root().id(); - - // Send event for stream creation - inlined from wrapper - { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::CreatedStream { - id: created_stream_id, - stream, - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - } - - // Apply state change using wrapper method let entry_command = EntryCommand::CreateStream(CreateStreamWithId { stream_id: created_stream_id as u32, command, @@ -148,16 +125,13 @@ async fn create_stream( ) })?; - // Get the created stream details using direct slab access - let response = SendWrapper::new(|| { - state - .shard - .shard() - .streams - .with_components_by_id(created_stream_id, |(root, stats)| { - crate::http::mapper::map_stream_details(&root, &stats) - }) - })(); + let shard = state.shard.shard(); + let metadata = shard.metadata.load(); + let stream_meta = metadata + .streams + .get(created_stream_id) + .expect("Stream must exist after creation"); + let response = crate::http::mapper::map_stream_details_from_metadata(stream_meta); Ok::, CustomError>(Json(response)) }); @@ -189,24 +163,6 @@ async fn update_stream( ) })?; - // Send event for stream update - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::UpdatedStream { - stream_id: command.stream_id.clone(), - name: command.name.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - - // Apply state change using wrapper method let entry_command = EntryCommand::UpdateStream(command); state.shard.apply_state(identity.user_id, &entry_command).await.error(|e: &IggyError| { format!( @@ -233,8 +189,8 @@ async fn delete_stream( let session = Session::stateless(identity.user_id, identity.ip_address); let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await; - // Delete stream and get the stream entity - let stream = { + // Delete stream + { let future = SendWrapper::new( state .shard @@ -247,26 +203,6 @@ async fn delete_stream( format!("{COMPONENT} (error: {e}) - failed to delete stream with ID: {stream_id}",) })?; - let stream_id_numeric = stream.root().id(); - - // Send event for stream deletion - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::DeletedStream { - id: stream_id_numeric, - stream_id: identifier_stream_id.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - - // Apply state change using wrapper method let entry_command = EntryCommand::DeleteStream(DeleteStream { stream_id: identifier_stream_id, }); @@ -319,7 +255,6 @@ async fn purge_stream( broadcast_future.await; } - // Apply state change using wrapper method let entry_command = EntryCommand::PurgeStream(PurgeStream { stream_id: identifier_stream_id, }); diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs index 8341c2cfaa..605b429abd 100644 --- a/core/server/src/http/topics.rs +++ b/core/server/src/http/topics.rs @@ -20,11 +20,9 @@ use crate::http::COMPONENT; use crate::http::error::CustomError; use crate::http::jwt::json_web_token::Identity; use crate::http::shared::AppState; -use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents}; use crate::state::command::EntryCommand; use crate::state::models::CreateTopicWithId; use crate::streaming::session::Session; -use crate::streaming::{streams, topics}; use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::routing::{delete, get}; @@ -68,41 +66,20 @@ async fn get_topic( let identity_topic_id = Identifier::from_str_value(&topic_id)?; let session = Session::stateless(identity.user_id, identity.ip_address); + let shard = state.shard.shard(); - // Check permissions and stream existence - state.shard.shard().ensure_authenticated(&session)?; - let stream_exists = state - .shard - .shard() - .ensure_stream_exists(&identity_stream_id) - .is_ok(); - if !stream_exists { - return Err(CustomError::ResourceNotFound); - } + let numeric_stream_id = shard + .metadata + .get_stream_id(&identity_stream_id) + .ok_or(CustomError::ResourceNotFound)?; - let topic_exists = state - .shard - .shard() - .ensure_topic_exists(&identity_stream_id, &identity_topic_id) - .is_ok(); - if !topic_exists { - return Err(CustomError::ResourceNotFound); - } + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &identity_topic_id) + .ok_or(CustomError::ResourceNotFound)?; - let numeric_stream_id = state - .shard - .shard() - .streams - .with_stream_by_id(&identity_stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = state.shard.shard().streams.with_topic_by_id( - &identity_stream_id, - &identity_topic_id, - topics::helpers::get_topic_id(), - ); - - state.shard.shard() + shard .permissioner - .borrow() .get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id) .error(|e: &IggyError| { format!( @@ -111,12 +88,14 @@ async fn get_topic( ) })?; - // Get topic details using the new API - let topic_details = state.shard.shard().streams.with_topic_by_id( - &identity_stream_id, - &identity_topic_id, - |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats), - ); + let metadata = shard.metadata.load(); + let topic_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .ok_or(CustomError::ResourceNotFound)?; + + let topic_details = crate::http::mapper::map_topic_details_from_metadata(topic_meta); Ok(Json(topic_details)) } @@ -127,22 +106,17 @@ async fn get_topics( Extension(identity): Extension, Path(stream_id): Path, ) -> Result>, CustomError> { - let stream_id = Identifier::from_str_value(&stream_id)?; + let stream_id_ident = Identifier::from_str_value(&stream_id)?; let session = Session::stateless(identity.user_id, identity.ip_address); + let shard = state.shard.shard(); - // Check permissions and stream existence - state.shard.shard().ensure_authenticated(&session)?; - state.shard.shard().ensure_stream_exists(&stream_id)?; - - let numeric_stream_id = state - .shard - .shard() - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id_ident) + .ok_or(CustomError::ResourceNotFound)?; - state.shard.shard() + shard .permissioner - .borrow() .get_topics(session.get_user_id(), numeric_stream_id) .error(|e: &IggyError| { format!( @@ -151,17 +125,12 @@ async fn get_topics( ) })?; - // Get topics using the new API - let topics = state - .shard - .shard() + let metadata = shard.metadata.load(); + let stream_meta = metadata .streams - .with_topics(&stream_id, |topics| { - topics.with_components(|topics| { - let (roots, _, stats) = topics.into_components(); - crate::http::mapper::map_topics_from_components(&roots, &stats) - }) - }); + .get(numeric_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + let topics = crate::http::mapper::map_topics_from_metadata(stream_meta); Ok(Json(topics)) } @@ -180,7 +149,7 @@ async fn create_topic( let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address)); let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await; - let topic = { + let topic_id = { let future = SendWrapper::new(state.shard.shard().create_topic( &session, &command.stream_id, @@ -196,26 +165,30 @@ async fn create_topic( format!("{COMPONENT} (error: {e}) - failed to create topic, stream ID: {stream_id}") })?; - // Update command with actual values from created topic - command.message_expiry = topic.root().message_expiry(); - command.max_topic_size = topic.root().max_topic_size(); - - let topic_id = topic.id(); + { + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&command.stream_id) + .expect("Stream must exist"); + let metadata = shard.metadata.load(); + if let Some(topic_meta) = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(topic_id)) + { + command.message_expiry = topic_meta.message_expiry; + command.max_topic_size = topic_meta.max_topic_size; + } + } - // Send events for topic creation let broadcast_future = SendWrapper::new(async { use crate::shard::transmission::event::ShardEvent; let shard = state.shard.shard(); - let event = ShardEvent::CreatedTopic { - stream_id: command.stream_id.clone(), - topic, - }; - let _responses = shard.broadcast_event_to_all_shards(event).await; - // Create partitions - let partitions = shard + let partition_infos = shard .create_partitions( &session, &command.stream_id, @@ -227,7 +200,7 @@ async fn create_topic( let event = ShardEvent::CreatedPartitions { stream_id: command.stream_id.clone(), topic_id: Identifier::numeric(topic_id as u32).unwrap(), - partitions, + partitions: partition_infos, }; let _responses = shard.broadcast_event_to_all_shards(event).await; @@ -240,18 +213,22 @@ async fn create_topic( ) })?; - // Create response using the same approach as binary handler let response = { - let topic_identifier = Identifier::numeric(topic_id as u32).unwrap(); - let topic_response = state.shard.shard().streams.with_topic_by_id( - &command.stream_id, - &topic_identifier, - |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats), - ); + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&command.stream_id) + .expect("Stream must exist"); + let metadata = shard.metadata.load(); + let topic_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(topic_id)) + .expect("Topic must exist after creation"); + let topic_response = crate::http::mapper::map_topic_details_from_metadata(topic_meta); Json(topic_response) }; - // Apply state change like in binary handler { let entry_command = EntryCommand::CreateTopic(CreateTopicWithId { topic_id: topic_id as u32, @@ -303,41 +280,30 @@ async fn update_topic( ) })?; - // TODO: Tech debt. - let topic_id = if name_changed { + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&command.stream_id) + .expect("Stream must exist"); + + let topic_id_for_lookup = if name_changed { Identifier::named(&command.name.clone()).unwrap() } else { command.topic_id.clone() }; - // Get the updated values from the topic - let (message_expiry, max_topic_size) = state.shard.shard().streams.with_topic_by_id( - &command.stream_id, - &topic_id, - |(root, _, _)| (root.message_expiry(), root.max_topic_size()), - ); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id_for_lookup) + .expect("Topic must exist after update"); - // Send event for topic update - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::UpdatedTopic { - stream_id: command.stream_id.clone(), - topic_id: command.topic_id.clone(), - name: command.name.clone(), - message_expiry: command.message_expiry, - compression_algorithm: command.compression_algorithm, - max_topic_size: command.max_topic_size, - replication_factor: command.replication_factor, - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } + let metadata = shard.metadata.load(); + let topic_meta = metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .expect("Topic metadata must exist"); + let (message_expiry, max_topic_size) = (topic_meta.message_expiry, topic_meta.max_topic_size); command.message_expiry = message_expiry; command.max_topic_size = max_topic_size; @@ -369,7 +335,7 @@ async fn delete_topic( let session = Session::stateless(identity.user_id, identity.ip_address); let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await; - let topic = { + { let future = SendWrapper::new(state.shard.shard().delete_topic( &session, &identifier_stream_id, @@ -382,26 +348,6 @@ async fn delete_topic( ) })?; - let topic_id_numeric = topic.root().id(); - - // Send event for topic deletion - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::DeletedTopic { - id: topic_id_numeric, - stream_id: identifier_stream_id.clone(), - topic_id: identifier_topic_id.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let entry_command = EntryCommand::DeleteTopic(DeleteTopic { stream_id: identifier_stream_id, @@ -447,7 +393,6 @@ async fn purge_topic( ) })?; - // Send event for topic purge { let broadcast_future = SendWrapper::new(async { use crate::shard::transmission::event::ShardEvent; diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs index 29ae19230a..47110dfb9b 100644 --- a/core/server/src/http/users.rs +++ b/core/server/src/http/users.rs @@ -137,26 +137,6 @@ async fn create_user( let user_id = user.id; let response = Json(mapper::map_user(&user)); - // Send event for user creation - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::CreatedUser { - user_id, - username: command.username.to_owned(), - password: command.password.to_owned(), - status: command.status, - permissions: command.permissions.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let username = command.username.clone(); let entry_command = EntryCommand::CreateUser(CreateUserWithId { @@ -212,24 +192,6 @@ async fn update_user( format!("{COMPONENT} (error: {e}) - failed to update user, user ID: {user_id}") })?; - // Send event for user update - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::UpdatedUser { - user_id: command.user_id.clone(), - username: command.username.clone(), - status: command.status, - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let username = command.username.clone(); let entry_command = EntryCommand::UpdateUser(command); @@ -271,23 +233,6 @@ async fn update_permissions( format!("{COMPONENT} (error: {e}) - failed to update permissions, user ID: {user_id}") })?; - // Send event for permissions update - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::UpdatedPermissions { - user_id: command.user_id.clone(), - permissions: command.permissions.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let entry_command = EntryCommand::UpdatePermissions(command); let future = SendWrapper::new( @@ -332,24 +277,6 @@ async fn change_password( format!("{COMPONENT} (error: {e}) - failed to change password, user ID: {user_id}") })?; - // Send event for password change - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::ChangedPassword { - user_id: command.user_id.clone(), - current_password: command.current_password.clone(), - new_password: command.new_password.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let entry_command = EntryCommand::ChangePassword(command); let future = SendWrapper::new( @@ -388,22 +315,6 @@ async fn delete_user( format!("{COMPONENT} (error: {e}) - failed to delete user with ID: {user_id}") })?; - // Send event for user deletion - { - let broadcast_future = SendWrapper::new(async { - use crate::shard::transmission::event::ShardEvent; - let event = ShardEvent::DeletedUser { - user_id: identifier_user_id.clone(), - }; - let _responses = state - .shard - .shard() - .broadcast_event_to_all_shards(event) - .await; - }); - broadcast_future.await; - } - { let entry_command = EntryCommand::DeleteUser(DeleteUser { user_id: identifier_user_id, diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs index 2919214b66..b6a7580065 100644 --- a/core/server/src/lib.rs +++ b/core/server/src/lib.rs @@ -37,10 +37,10 @@ pub mod diagnostics; pub mod http; pub mod io; pub mod log; +pub mod metadata; pub mod quic; pub mod server_error; pub mod shard; -pub mod slab; pub mod state; pub mod streaming; pub mod tcp; diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 3fe36e20d7..bf40787aa7 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -28,19 +28,17 @@ use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError, MemoryPool}; use server::SEMANTIC_VERSION; use server::args::Args; use server::bootstrap::{ - create_directories, create_shard_connections, create_shard_executor, load_config, load_streams, - load_users, resolve_persister, update_system_info, + create_directories, create_shard_connections, create_shard_executor, load_config, + load_metadata_only, load_users_to_metadata, resolve_persister, update_system_info, }; use server::configs::sharding::ShardAllocator; use server::diagnostics::{print_io_uring_permission_info, print_locked_memory_limit_info}; use server::io::fs_utils; use server::log::logger::Logging; +use server::metadata::Metadata; use server::server_error::ServerError; use server::shard::system::info::SystemInfo; use server::shard::{IggyShard, calculate_shard_assignment}; -use server::slab::traits_ext::{ - EntityComponentSystem, EntityComponentSystemMutCell, IntoComponents, -}; use server::state::file::FileState; use server::state::system::SystemState; use server::streaming::clients::client_manager::{Client, ClientManager}; @@ -278,8 +276,14 @@ fn main() -> Result<(), ServerError> { ); let state = SystemState::load(state).await?; let (streams_state, users_state) = state.decompose(); - let streams = load_streams(streams_state.into_values(), &config.system).await?; - let users = load_users(users_state.into_values()); + + // Create shared metadata structures (leaked for 'static lifetime) + let shared_metadata: &'static Metadata = + Box::leak(Box::new(Metadata::default())); + + // Load metadata directly into SharedMetadata (no slabs) + load_metadata_only(streams_state.into_values(), shared_metadata); + load_users_to_metadata(users_state.into_values(), shared_metadata); // ELEVENTH DISCRETE LOADING STEP. let shard_allocator = ShardAllocator::new(&config.system.sharding.cpu_allocation)?; @@ -320,41 +324,31 @@ fn main() -> Result<(), ServerError> { let client_manager: EternalPtr> = client_manager.into(); let client_manager = ClientManager::new(client_manager); - streams.with_components(|components| { - let (root, ..) = components.into_components(); - for (_, stream) in root.iter() { - stream.topics().with_components(|components| { - let (root, ..) = components.into_components(); - for (_, topic) in root.iter() { - topic.partitions().with_components(|components| { - let (root, ..) = components.into_components(); - for (_, partition) in root.iter() { - let stream_id = stream.id(); - let topic_id = topic.id(); - let partition_id = partition.id(); - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - let shard_id = ShardId::new(calculate_shard_assignment( - &ns, - shard_assignment.len() as u32, - )); - // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integratedds - let location = PartitionLocation::new(shard_id, LocalIdx::new(0)); - shards_table.insert(ns, location); - } - }); + // Populate shards_table from SharedMetadata partitions (hierarchical traversal) + { + let metadata = shared_metadata.load(); + for (stream_id, stream_meta) in metadata.streams.iter() { + for (topic_id, topic_meta) in stream_meta.topics.iter() { + for (partition_id, _partition_meta) in topic_meta.partitions.iter() { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + let shard_id = ShardId::new(calculate_shard_assignment( + &ns, + shard_assignment.len() as u32, + )); + // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integrated + let location = PartitionLocation::new(shard_id, LocalIdx::new(0)); + shards_table.insert(ns, location); } - }) + } } - }); + } for (id, assignment) in shard_assignment .into_iter() .enumerate() .map(|(idx, assignment)| (idx as u16, assignment)) { - let streams = streams.clone(); let shards_table = shards_table.clone(); - let users = users.clone(); let connections = connections.clone(); let config = config.clone(); let encryptor = encryptor.clone(); @@ -373,29 +367,6 @@ fn main() -> Result<(), ServerError> { ); let client_manager = client_manager.clone(); - // TODO: Explore decoupling the `Log` from `Partition` entity. - // Ergh... I knew this will backfire to include `Log` as part of the `Partition` entity, - // We have to initialize with a default log for every partition, once we `Clone` the Streams / Topics / Partitions, - // because `Clone` impl for `Partition` does not clone the actual log, just creates an empty one. - streams.with_components(|components| { - let (root, ..) = components.into_components(); - for (_, stream) in root.iter() { - stream.topics().with_components_mut(|components| { - let (mut root, ..) = components.into_components(); - for (_, topic) in root.iter_mut() { - let partitions_count = topic.partitions().len(); - for log_id in 0..partitions_count { - let id = topic.partitions_mut().insert_default_log(); - assert_eq!( - id, log_id, - "main: partition_insert_default_log: id mismatch when creating default log" - ); - } - } - }) - } - }); - let shard_done_tx = shard_done_tx.clone(); let handle = std::thread::Builder::new() .name(format!("shard-{id}")) @@ -414,9 +385,7 @@ fn main() -> Result<(), ServerError> { let builder = IggyShard::builder(); let shard = builder .id(id) - .streams(streams) .state(state) - .users(users) .shards_table(shards_table) .connections(connections) .clients_manager(client_manager) @@ -425,6 +394,7 @@ fn main() -> Result<(), ServerError> { .version(current_version) .metrics(metrics) .is_follower(is_follower) + .shared_metadata(shared_metadata.into()) .build(); let shard = Rc::new(shard); diff --git a/core/server/src/metadata/consumer_group.rs b/core/server/src/metadata/consumer_group.rs new file mode 100644 index 0000000000..96d290caa9 --- /dev/null +++ b/core/server/src/metadata/consumer_group.rs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group_member::ConsumerGroupMemberMeta; +use crate::metadata::{ConsumerGroupId, PartitionId, SLAB_SEGMENT_SIZE}; +use iggy_common::collections::SegmentedSlab; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct ConsumerGroupMeta { + pub id: ConsumerGroupId, + pub name: Arc, + pub partitions: Vec, + pub members: SegmentedSlab, +} + +impl ConsumerGroupMeta { + /// Rebalance partition assignments among members (round-robin). + pub fn rebalance_members(&mut self) { + let partition_count = self.partitions.len(); + let member_count = self.members.len(); + + if member_count == 0 || partition_count == 0 { + return; + } + + let mut members = std::mem::take(&mut self.members); + + // Clear all member partitions and rebuild assignments + let member_ids: Vec = members.iter().map(|(id, _)| id).collect(); + for &member_id in &member_ids { + if let Some(member) = members.get(member_id) { + let mut updated_member = member.clone(); + updated_member.partitions.clear(); + let (new_members, _) = members.update(member_id, updated_member); + members = new_members; + } + } + + for (i, &partition_id) in self.partitions.iter().enumerate() { + let member_idx = i % member_count; + if let Some(&member_id) = member_ids.get(member_idx) + && let Some(member) = members.get(member_id) + { + let mut updated_member = member.clone(); + updated_member.partitions.push(partition_id); + let (new_members, _) = members.update(member_id, updated_member); + members = new_members; + } + } + + self.members = members; + } +} diff --git a/core/server/src/metadata/consumer_group_member.rs b/core/server/src/metadata/consumer_group_member.rs new file mode 100644 index 0000000000..8543351cbb --- /dev/null +++ b/core/server/src/metadata/consumer_group_member.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{ClientId, ConsumerGroupMemberId, PartitionId}; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +#[derive(Clone, Debug)] +pub struct ConsumerGroupMemberMeta { + pub id: ConsumerGroupMemberId, + pub client_id: ClientId, + pub partitions: Vec, + pub partition_index: Arc, +} + +impl ConsumerGroupMemberMeta { + pub fn new(id: ConsumerGroupMemberId, client_id: ClientId) -> Self { + Self { + id, + client_id, + partitions: Vec::new(), + partition_index: Arc::new(AtomicUsize::new(0)), + } + } +} diff --git a/core/server/src/metadata/mod.rs b/core/server/src/metadata/mod.rs new file mode 100644 index 0000000000..c59029d176 --- /dev/null +++ b/core/server/src/metadata/mod.rs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared metadata module providing a single source of truth for all shards. +//! +//! This module provides an `ArcSwap`-based approach where all shards read from +//! a shared snapshot, and only shard 0 can write (swap in new snapshots). +//! +//! # Architecture +//! +//! - `GlobalMetadata` (snapshot.rs): Immutable snapshot with all metadata +//! - `SharedMetadata` (shared.rs): Thread-safe wrapper with ArcSwap +//! - Entity types: `StreamMeta`, `TopicMeta`, `PartitionMeta`, `UserMeta`, `ConsumerGroupMeta` +//! - Consumer offsets are stored in `PartitionMeta` for cross-shard visibility + +mod consumer_group; +mod consumer_group_member; +mod partition; +mod shared; +mod snapshot; +mod stream; +mod topic; +mod user; + +pub use consumer_group::ConsumerGroupMeta; +pub use consumer_group_member::ConsumerGroupMemberMeta; +pub use partition::PartitionMeta; +pub use shared::Metadata; +pub use snapshot::InnerMetadata; +pub use stream::StreamMeta; +pub use topic::TopicMeta; +pub use user::UserMeta; + +pub type StreamId = usize; +pub type TopicId = usize; +pub type PartitionId = usize; +pub type UserId = u32; +pub type ClientId = u32; +pub type ConsumerGroupId = usize; +pub type ConsumerGroupMemberId = usize; +pub type ConsumerGroupKey = (StreamId, TopicId, ConsumerGroupId); + +/// Segment size for SegmentedSlab (1024 entries per segment). +pub const SLAB_SEGMENT_SIZE: usize = 1024; diff --git a/core/server/src/metadata/partition.rs b/core/server/src/metadata/partition.rs new file mode 100644 index 0000000000..b2490b96ac --- /dev/null +++ b/core/server/src/metadata/partition.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::PartitionId; +use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets}; +use crate::streaming::stats::PartitionStats; +use iggy_common::IggyTimestamp; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct PartitionMeta { + pub id: PartitionId, + pub created_at: IggyTimestamp, + /// Monotonically increasing version to detect stale partition_store entries. + /// Set to the Metadata version when the partition was created. + pub revision_id: u64, + pub stats: Arc, + /// Consumer offsets for this partition // TODO: move to partition store + pub consumer_offsets: Option>, + /// Consumer group offsets for this partition // TODO: move to partition store + pub consumer_group_offsets: Option>, +} diff --git a/core/server/src/metadata/shared.rs b/core/server/src/metadata/shared.rs new file mode 100644 index 0000000000..983aeaabd6 --- /dev/null +++ b/core/server/src/metadata/shared.rs @@ -0,0 +1,1911 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{ + ConsumerGroupId, ConsumerGroupMemberMeta, ConsumerGroupMeta, InnerMetadata, PartitionId, + PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta, +}; +use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets}; +use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; +use arc_swap::{ArcSwap, Guard}; +use iggy_common::collections::SegmentedSlab; +use iggy_common::{ + CompressionAlgorithm, IdKind, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, + PersonalAccessToken, UserStatus, +}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +/// Thread-safe wrapper for GlobalMetadata using ArcSwap for lock-free reads. +/// Uses hierarchical structure: streams contain topics, topics contain partitions and consumer groups. +/// IDs are assigned by SegmentedSlab::insert() at each level. +pub struct Metadata { + inner: ArcSwap, +} + +impl Default for Metadata { + fn default() -> Self { + Self::new(InnerMetadata::new()) + } +} + +impl Metadata { + pub fn new(initial: InnerMetadata) -> Self { + Self { + inner: ArcSwap::from_pointee(initial), + } + } + + #[inline] + pub fn load(&self) -> Guard> { + self.inner.load() + } + + #[inline] + pub fn load_full(&self) -> Arc { + self.inner.load_full() + } + + /// Add a stream with a specific ID (for bootstrap/recovery). + pub fn add_stream_with_id(&self, id: StreamId, meta: StreamMeta) { + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + let entries: Vec<_> = new + .streams + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(std::iter::once((id, meta.clone()))) + .collect(); + new.streams = SegmentedSlab::from_entries(entries); + new.stream_index = new.stream_index.update(meta.name.clone(), id); + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a new stream with slab-assigned ID. Returns the assigned ID. + pub fn add_stream(&self, meta: StreamMeta) -> StreamId { + let assigned_id = Arc::new(AtomicUsize::new(0)); + let assigned_id_clone = assigned_id.clone(); + + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + let (streams, id) = new.streams.insert(meta.clone()); + assigned_id_clone.store(id, Ordering::Release); + new.streams = streams; + new.stream_index = new.stream_index.update(meta.name.clone(), id); + new.revision += 1; + Arc::new(new) + }); + + assigned_id.load(Ordering::Acquire) + } + + /// Atomically validates name uniqueness and updates stream name. + /// Returns Ok(()) if update succeeded, or appropriate error. + pub fn try_update_stream(&self, id: StreamId, new_name: Arc) -> Result<(), IggyError> { + let stream_not_found = Arc::new(AtomicBool::new(false)); + let name_conflict = Arc::new(AtomicBool::new(false)); + let unchanged = Arc::new(AtomicBool::new(false)); + + let stream_not_found_clone = stream_not_found.clone(); + let name_conflict_clone = name_conflict.clone(); + let unchanged_clone = unchanged.clone(); + let new_name_clone = new_name.clone(); + + self.inner.rcu(move |current| { + let Some(old_meta) = current.streams.get(id) else { + stream_not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + if old_meta.name == new_name_clone { + unchanged_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + if let Some(&existing_id) = current.stream_index.get(&new_name_clone) + && existing_id != id + { + name_conflict_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + stream_not_found_clone.store(false, Ordering::Release); + name_conflict_clone.store(false, Ordering::Release); + unchanged_clone.store(false, Ordering::Release); + + let mut new = (**current).clone(); + new.stream_index = new.stream_index.without(&old_meta.name); + + let mut updated = old_meta.clone(); + updated.name = new_name_clone.clone(); + let (streams, _) = new.streams.update(id, updated); + new.streams = streams; + new.stream_index = new.stream_index.update(new_name_clone.clone(), id); + new.revision += 1; + + Arc::new(new) + }); + + if stream_not_found.load(Ordering::Acquire) { + Err(IggyError::StreamIdNotFound( + Identifier::numeric(id as u32).unwrap(), + )) + } else if name_conflict.load(Ordering::Acquire) { + Err(IggyError::StreamNameAlreadyExists(new_name.to_string())) + } else { + Ok(()) + } + } + + /// Delete a stream and all its nested topics/partitions/consumer groups. + pub fn delete_stream(&self, id: StreamId) { + self.inner.rcu(|current| { + let mut new = (**current).clone(); + if let Some(stream) = new.streams.get(id) { + new.stream_index = new.stream_index.without(&stream.name); + } + let (streams, _) = new.streams.remove(id); + new.streams = streams; + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a topic with a specific ID (for bootstrap/recovery). + pub fn add_topic_with_id(&self, stream_id: StreamId, topic_id: TopicId, meta: TopicMeta) { + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + let entries: Vec<_> = updated_stream + .topics + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(std::iter::once((topic_id, meta.clone()))) + .collect(); + updated_stream.topics = SegmentedSlab::from_entries(entries); + updated_stream.topic_index = updated_stream + .topic_index + .update(meta.name.clone(), topic_id); + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a new topic with slab-assigned ID. Returns the assigned ID. + pub fn add_topic(&self, stream_id: StreamId, meta: TopicMeta) -> Option { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + let assigned_id_clone = assigned_id.clone(); + + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + let (topics, id) = updated_stream.topics.insert(meta.clone()); + assigned_id_clone.store(id, Ordering::Release); + updated_stream.topics = topics; + updated_stream.topic_index = + updated_stream.topic_index.update(meta.name.clone(), id); + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + + let id = assigned_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + /// Atomically validates name uniqueness and updates topic. + /// Returns Ok(()) if update succeeded, or appropriate error. + #[allow(clippy::too_many_arguments)] + pub fn try_update_topic( + &self, + stream_id: StreamId, + topic_id: TopicId, + new_name: Arc, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + ) -> Result<(), IggyError> { + let stream_not_found = Arc::new(AtomicBool::new(false)); + let topic_not_found = Arc::new(AtomicBool::new(false)); + let name_conflict = Arc::new(AtomicBool::new(false)); + + let stream_not_found_clone = stream_not_found.clone(); + let topic_not_found_clone = topic_not_found.clone(); + let name_conflict_clone = name_conflict.clone(); + let new_name_clone = new_name.clone(); + + self.inner.rcu(move |current| { + let Some(stream) = current.streams.get(stream_id) else { + stream_not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + let Some(old_meta) = stream.topics.get(topic_id) else { + topic_not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + if old_meta.name != new_name_clone + && let Some(&existing_id) = stream.topic_index.get(&new_name_clone) + && existing_id != topic_id + { + name_conflict_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + stream_not_found_clone.store(false, Ordering::Release); + topic_not_found_clone.store(false, Ordering::Release); + name_conflict_clone.store(false, Ordering::Release); + + let mut new = (**current).clone(); + let mut updated_stream = stream.clone(); + + if old_meta.name != new_name_clone { + updated_stream.topic_index = updated_stream.topic_index.without(&old_meta.name); + updated_stream.topic_index = updated_stream + .topic_index + .update(new_name_clone.clone(), topic_id); + } + + let updated_meta = TopicMeta { + id: topic_id, + name: new_name_clone.clone(), + created_at: old_meta.created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + partitions_count: old_meta.partitions_count, + stats: old_meta.stats.clone(), + partitions: old_meta.partitions.clone(), + consumer_groups: old_meta.consumer_groups.clone(), + consumer_group_index: old_meta.consumer_group_index.clone(), + partition_counter: old_meta.partition_counter.clone(), + }; + + let (topics, _) = updated_stream.topics.update(topic_id, updated_meta); + updated_stream.topics = topics; + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + new.revision += 1; + + Arc::new(new) + }); + + if stream_not_found.load(Ordering::Acquire) { + Err(IggyError::StreamIdNotFound( + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else if topic_not_found.load(Ordering::Acquire) { + Err(IggyError::TopicIdNotFound( + Identifier::numeric(topic_id as u32).unwrap(), + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else if name_conflict.load(Ordering::Acquire) { + Err(IggyError::TopicNameAlreadyExists( + new_name.to_string(), + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else { + Ok(()) + } + } + + /// Delete a topic and all its nested partitions/consumer groups. + pub fn delete_topic(&self, stream_id: StreamId, topic_id: TopicId) { + self.inner.rcu(|current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + updated_stream.topic_index = updated_stream.topic_index.without(&topic.name); + } + let (topics, _) = updated_stream.topics.remove(topic_id); + updated_stream.topics = topics; + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + /// Add partitions with specific IDs (for bootstrap/recovery). + pub fn add_partitions_with_ids( + &self, + stream_id: StreamId, + topic_id: TopicId, + partitions: Vec<(PartitionId, PartitionMeta)>, + ) { + if partitions.is_empty() { + return; + } + self.inner.rcu(move |current| { + let partitions = partitions.clone(); + let mut new = (**current).clone(); + let new_version = new.revision + 1; + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + let entries: Vec<_> = updated_topic + .partitions + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(partitions.into_iter().map(|(id, mut meta)| { + meta.revision_id = new_version; + (id, meta) + })) + .collect(); + updated_topic.partitions = SegmentedSlab::from_entries(entries); + updated_topic.partitions_count = updated_topic.partitions.len() as u32; + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision = new_version; + Arc::new(new) + }); + } + + /// Add new partitions with slab-assigned IDs. Returns the assigned IDs. + pub fn add_partitions( + &self, + stream_id: StreamId, + topic_id: TopicId, + partitions: Vec, + ) -> Vec { + if partitions.is_empty() { + return Vec::new(); + } + + let assigned_ids = Arc::new(std::sync::Mutex::new(Vec::new())); + let assigned_ids_clone = assigned_ids.clone(); + + self.inner.rcu(move |current| { + let partitions = partitions.clone(); + let mut new = (**current).clone(); + let new_version = new.revision + 1; + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + let mut ids = Vec::new(); + + for mut meta in partitions { + meta.revision_id = new_version; + let (parts, id) = updated_topic.partitions.insert(meta.clone()); + + meta.id = id; + let (parts, _) = parts.update(id, meta); + updated_topic.partitions = parts; + ids.push(id); + } + updated_topic.partitions_count = updated_topic.partitions.len() as u32; + + *assigned_ids_clone.lock().unwrap() = ids; + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision = new_version; + Arc::new(new) + }); + + Arc::try_unwrap(assigned_ids).unwrap().into_inner().unwrap() + } + + pub fn delete_partitions( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_ids: &[PartitionId], + ) { + if partition_ids.is_empty() { + return; + } + let partition_ids = partition_ids.to_vec(); + + self.inner.rcu(move |current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + for &partition_id in &partition_ids { + let (parts, _) = updated_topic.partitions.remove(partition_id); + updated_topic.partitions = parts; + } + updated_topic.partitions_count = updated_topic.partitions.len() as u32; + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a user with a specific ID (for bootstrap/recovery). + pub fn add_user_with_id(&self, id: UserId, meta: UserMeta) { + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + let entries: Vec<_> = new + .users + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(std::iter::once((id as usize, meta.clone()))) + .collect(); + new.users = SegmentedSlab::from_entries(entries); + new.user_index = new.user_index.update(meta.username.clone(), id); + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a new user with slab-assigned ID. Returns the assigned ID. + pub fn add_user(&self, meta: UserMeta) -> UserId { + let assigned_id = Arc::new(AtomicUsize::new(0)); + let assigned_id_clone = assigned_id.clone(); + + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + let (users, id) = new.users.insert(meta.clone()); + assigned_id_clone.store(id, Ordering::Release); + new.users = users; + new.user_index = new.user_index.update(meta.username.clone(), id as UserId); + new.revision += 1; + Arc::new(new) + }); + + assigned_id.load(Ordering::Acquire) as UserId + } + + /// Atomically validates username uniqueness and updates user. + /// Returns the updated UserMeta if successful. + pub fn try_update_user( + &self, + id: UserId, + new_username: Option>, + new_status: Option, + ) -> Result { + let user_not_found = Arc::new(AtomicBool::new(false)); + let name_conflict = Arc::new(AtomicBool::new(false)); + let updated_meta: Arc>> = + Arc::new(std::sync::Mutex::new(None)); + + let user_not_found_clone = user_not_found.clone(); + let name_conflict_clone = name_conflict.clone(); + let updated_meta_clone = updated_meta.clone(); + let new_username_clone = new_username.clone(); + + self.inner.rcu(move |current| { + let Some(old_meta) = current.users.get(id as usize) else { + user_not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + let final_username = new_username_clone + .clone() + .unwrap_or_else(|| old_meta.username.clone()); + + if final_username != old_meta.username + && let Some(&existing_id) = current.user_index.get(&final_username) + && existing_id != id + { + name_conflict_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + user_not_found_clone.store(false, Ordering::Release); + name_conflict_clone.store(false, Ordering::Release); + + let mut new = (**current).clone(); + + if final_username != old_meta.username { + new.user_index = new.user_index.without(&old_meta.username); + new.user_index = new.user_index.update(final_username.clone(), id); + } + + let meta = UserMeta { + id: old_meta.id, + username: final_username, + password_hash: old_meta.password_hash.clone(), + status: new_status.unwrap_or(old_meta.status), + permissions: old_meta.permissions.clone(), + created_at: old_meta.created_at, + }; + + *updated_meta_clone.lock().unwrap() = Some(meta.clone()); + + let (users, _) = new.users.update(id as usize, meta); + new.users = users; + new.revision += 1; + Arc::new(new) + }); + + if user_not_found.load(Ordering::Acquire) { + Err(IggyError::ResourceNotFound(format!("User {}", id))) + } else if name_conflict.load(Ordering::Acquire) { + Err(IggyError::UserAlreadyExists) + } else { + Ok(updated_meta.lock().unwrap().take().unwrap()) + } + } + + /// Updates user metadata directly. Use only when username is not changing + /// or when caller has already verified username uniqueness. + pub fn update_user_meta(&self, id: UserId, meta: UserMeta) { + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + + if let Some(old_meta) = new.users.get(id as usize) + && old_meta.username != meta.username + { + new.user_index = new.user_index.without(&old_meta.username); + new.user_index = new.user_index.update(meta.username.clone(), id); + } + + let (users, _) = new.users.update(id as usize, meta); + new.users = users; + new.revision += 1; + Arc::new(new) + }); + } + + pub fn delete_user(&self, id: UserId) { + self.inner.rcu(|current| { + let mut new = (**current).clone(); + if let Some(user) = new.users.get(id as usize) { + new.user_index = new.user_index.without(&user.username); + } + let (users, _) = new.users.remove(id as usize); + new.users = users; + new.personal_access_tokens = new.personal_access_tokens.without(&id); + new.revision += 1; + Arc::new(new) + }); + } + + pub fn add_personal_access_token(&self, user_id: UserId, pat: PersonalAccessToken) { + self.inner.rcu(move |current| { + let pat = pat.clone(); + let mut new = (**current).clone(); + let user_pats = new + .personal_access_tokens + .get(&user_id) + .cloned() + .unwrap_or_default(); + new.personal_access_tokens = new + .personal_access_tokens + .update(user_id, user_pats.update(pat.token.clone(), pat)); + new.revision += 1; + Arc::new(new) + }); + } + + pub fn delete_personal_access_token(&self, user_id: UserId, token_hash: &Arc) { + let token_hash = token_hash.clone(); + self.inner.rcu(move |current| { + let mut new = (**current).clone(); + if let Some(user_pats) = new.personal_access_tokens.get(&user_id) { + new.personal_access_tokens = new + .personal_access_tokens + .update(user_id, user_pats.without(&token_hash)); + } + new.revision += 1; + Arc::new(new) + }); + } + + pub fn get_user_personal_access_tokens(&self, user_id: UserId) -> Vec { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.values().cloned().collect()) + .unwrap_or_default() + } + + pub fn get_personal_access_token_by_hash( + &self, + token_hash: &str, + ) -> Option { + let token_hash_arc: Arc = Arc::from(token_hash); + let metadata = self.load(); + for (_, user_pats) in metadata.personal_access_tokens.iter() { + if let Some(pat) = user_pats.get(&token_hash_arc) { + return Some(pat.clone()); + } + } + None + } + + pub fn user_pat_count(&self, user_id: UserId) -> usize { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.len()) + .unwrap_or(0) + } + + pub fn user_has_pat_with_name(&self, user_id: UserId, name: &str) -> bool { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.values().any(|pat| &*pat.name == name)) + .unwrap_or(false) + } + + pub fn find_pat_token_hash_by_name(&self, user_id: UserId, name: &str) -> Option> { + self.load() + .personal_access_tokens + .get(&user_id) + .and_then(|pats| { + pats.iter() + .find(|(_, pat)| &*pat.name == name) + .map(|(hash, _)| hash.clone()) + }) + } + + /// Add a consumer group with a specific ID (for bootstrap/recovery). + pub fn add_consumer_group_with_id( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + meta: ConsumerGroupMeta, + ) { + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + let entries: Vec<_> = updated_topic + .consumer_groups + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(std::iter::once((group_id, meta.clone()))) + .collect(); + updated_topic.consumer_groups = SegmentedSlab::from_entries(entries); + updated_topic.consumer_group_index = updated_topic + .consumer_group_index + .update(meta.name.clone(), group_id); + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + /// Add a new consumer group with slab-assigned ID. Returns the assigned ID. + pub fn add_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + meta: ConsumerGroupMeta, + ) -> Option { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + let assigned_id_clone = assigned_id.clone(); + + self.inner.rcu(move |current| { + let meta = meta.clone(); + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + let (groups, id) = updated_topic.consumer_groups.insert(meta.clone()); + assigned_id_clone.store(id, Ordering::Release); + updated_topic.consumer_groups = groups; + updated_topic.consumer_group_index = updated_topic + .consumer_group_index + .update(meta.name.clone(), id); + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + + let id = assigned_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + pub fn delete_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) { + self.inner.rcu(|current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + if let Some(group) = updated_topic.consumer_groups.get(group_id) { + updated_topic.consumer_group_index = + updated_topic.consumer_group_index.without(&group.name); + } + let (groups, _) = updated_topic.consumer_groups.remove(group_id); + updated_topic.consumer_groups = groups; + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + pub fn join_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> Option { + let member_id = Arc::new(AtomicUsize::new(usize::MAX)); + let member_id_clone = member_id.clone(); + + self.inner.rcu(|current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + if let Some(group) = updated_topic.consumer_groups.get(group_id) { + let mut updated_group = group.clone(); + + let next_id = updated_group + .members + .iter() + .map(|(_, m)| m.id) + .max() + .map(|m| m + 1) + .unwrap_or(0); + + let new_member = ConsumerGroupMemberMeta::new(next_id, client_id); + let (members, _) = updated_group.members.insert(new_member); + updated_group.members = members; + updated_group.rebalance_members(); + + member_id_clone.store(next_id, Ordering::Release); + + let (groups, _) = updated_topic + .consumer_groups + .update(group_id, updated_group); + updated_topic.consumer_groups = groups; + } + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + + let id = member_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + pub fn leave_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> Option { + let member_id = Arc::new(AtomicUsize::new(usize::MAX)); + let member_id_clone = member_id.clone(); + + self.inner.rcu(|current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + if let Some(group) = updated_topic.consumer_groups.get(group_id) { + let mut updated_group = group.clone(); + + let member_to_remove: Option = updated_group + .members + .iter() + .find(|(_, m)| m.client_id == client_id) + .map(|(id, _)| id); + + if let Some(mid) = member_to_remove { + member_id_clone.store(mid, Ordering::Release); + let (members, _) = updated_group.members.remove(mid); + updated_group.members = members; + updated_group.rebalance_members(); + + let (groups, _) = updated_topic + .consumer_groups + .update(group_id, updated_group); + updated_topic.consumer_groups = groups; + } + } + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + + let id = member_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + pub fn is_consumer_group_member( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> bool { + let metadata = self.load(); + metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .map(|g| g.members.iter().any(|(_, m)| m.client_id == client_id)) + .unwrap_or(false) + } + + pub fn rebalance_consumer_groups_for_topic( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_ids: &[PartitionId], + ) { + let partition_ids = partition_ids.to_vec(); + + self.inner.rcu(move |current| { + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + let group_ids: Vec<_> = updated_topic + .consumer_groups + .iter() + .map(|(id, _)| id) + .collect(); + + for gid in group_ids { + if let Some(group) = updated_topic.consumer_groups.get(gid) { + let mut updated_group = group.clone(); + updated_group.partitions = partition_ids.clone(); + updated_group.rebalance_members(); + let (groups, _) = + updated_topic.consumer_groups.update(gid, updated_group); + updated_topic.consumer_groups = groups; + } + } + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + pub fn get_stream_id(&self, identifier: &Identifier) -> Option { + let metadata = self.load(); + match identifier.kind { + IdKind::Numeric => { + let stream_id = identifier.get_u32_value().ok()? as StreamId; + if metadata.streams.get(stream_id).is_some() { + Some(stream_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + metadata.stream_index.get(name.as_ref()).copied() + } + } + } + + pub fn stream_name_exists(&self, name: &str) -> bool { + self.load().stream_index.contains_key(name) + } + + pub fn get_topic_id(&self, stream_id: StreamId, identifier: &Identifier) -> Option { + let metadata = self.load(); + let stream = metadata.streams.get(stream_id)?; + + match identifier.kind { + IdKind::Numeric => { + let topic_id = identifier.get_u32_value().ok()? as TopicId; + if stream.topics.get(topic_id).is_some() { + Some(topic_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + stream.topic_index.get(&Arc::from(name.as_ref())).copied() + } + } + } + + pub fn get_user_id(&self, identifier: &Identifier) -> Option { + let metadata = self.load(); + match identifier.kind { + IdKind::Numeric => Some(identifier.get_u32_value().ok()? as UserId), + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + metadata.user_index.get(name.as_ref()).copied() + } + } + } + + pub fn get_consumer_group_id( + &self, + stream_id: StreamId, + topic_id: TopicId, + identifier: &Identifier, + ) -> Option { + let metadata = self.load(); + let stream = metadata.streams.get(stream_id)?; + let topic = stream.topics.get(topic_id)?; + + match identifier.kind { + IdKind::Numeric => { + let group_id = identifier.get_u32_value().ok()? as ConsumerGroupId; + if topic.consumer_groups.get(group_id).is_some() { + Some(group_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + topic + .consumer_group_index + .get(&Arc::from(name.as_ref())) + .copied() + } + } + } + + pub fn stream_exists(&self, id: StreamId) -> bool { + self.load().streams.get(id).is_some() + } + + pub fn topic_exists(&self, stream_id: StreamId, topic_id: TopicId) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .is_some() + } + + pub fn partition_exists( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .is_some() + } + + pub fn user_exists(&self, id: UserId) -> bool { + self.load().users.get(id as usize).is_some() + } + + pub fn consumer_group_exists( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .is_some() + } + + pub fn consumer_group_exists_by_name( + &self, + stream_id: StreamId, + topic_id: TopicId, + name: &str, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.consumer_group_index.contains_key(name)) + .unwrap_or(false) + } + + pub fn streams_count(&self) -> usize { + self.load().streams.len() + } + + pub fn topics_count(&self, stream_id: StreamId) -> usize { + self.load() + .streams + .get(stream_id) + .map(|s| s.topics.len()) + .unwrap_or(0) + } + + pub fn partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.partitions.len()) + .unwrap_or(0) + } + + pub fn get_next_partition_id(&self, stream_id: StreamId, topic_id: TopicId) -> Option { + let metadata = self.load(); + let topic = metadata.streams.get(stream_id)?.topics.get(topic_id)?; + let partitions_count = topic.partitions.len(); + + if partitions_count == 0 { + return None; + } + + let counter = &topic.partition_counter; + let mut partition_id = counter.fetch_add(1, Ordering::AcqRel); + if partition_id >= partitions_count { + partition_id %= partitions_count; + counter.store(partition_id + 1, Ordering::Relaxed); + } + Some(partition_id) + } + + pub fn get_next_member_partition_id( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + member_id: usize, + calculate: bool, + ) -> Option { + let metadata = self.load(); + let member = metadata + .streams + .get(stream_id)? + .topics + .get(topic_id)? + .consumer_groups + .get(group_id)? + .members + .get(member_id)?; + + let assigned_partitions = &member.partitions; + if assigned_partitions.is_empty() { + return None; + } + + let partitions_count = assigned_partitions.len(); + let counter = &member.partition_index; + + if calculate { + let current = counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some((current + 1) % partitions_count) + }) + .unwrap(); + Some(assigned_partitions[current % partitions_count]) + } else { + let current = counter.load(Ordering::Relaxed); + Some(assigned_partitions[current % partitions_count]) + } + } + + pub fn users_count(&self) -> usize { + self.load().users.len() + } + + pub fn username_exists(&self, username: &str) -> bool { + self.load().user_index.contains_key(username) + } + + pub fn consumer_groups_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.consumer_groups.len()) + .unwrap_or(0) + } + + pub fn get_stream_stats(&self, id: StreamId) -> Option> { + self.load().streams.get(id).map(|s| s.stats.clone()) + } + + pub fn get_topic_stats( + &self, + stream_id: StreamId, + topic_id: TopicId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.stats.clone()) + } + + pub fn get_partition_stats( + &self, + ns: &crate::shard::namespace::IggyNamespace, + ) -> Option> { + self.load() + .streams + .get(ns.stream_id()) + .and_then(|s| s.topics.get(ns.topic_id())) + .and_then(|t| t.partitions.get(ns.partition_id())) + .map(|p| p.stats.clone()) + } + + pub fn get_partition_stats_by_ids( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|p| p.stats.clone()) + } + + /// Set consumer offsets for a partition + pub fn set_partition_offsets( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + consumer_offsets: Arc, + consumer_group_offsets: Arc, + ) { + self.inner.rcu(move |current| { + let consumer_offsets = consumer_offsets.clone(); + let consumer_group_offsets = consumer_group_offsets.clone(); + let mut new = (**current).clone(); + + if let Some(stream) = new.streams.get(stream_id) { + let mut updated_stream = stream.clone(); + + if let Some(topic) = updated_stream.topics.get(topic_id) { + let mut updated_topic = topic.clone(); + + if let Some(partition) = updated_topic.partitions.get(partition_id) { + let mut updated_partition = partition.clone(); + updated_partition.consumer_offsets = Some(consumer_offsets); + updated_partition.consumer_group_offsets = Some(consumer_group_offsets); + + let (partitions, _) = updated_topic + .partitions + .update(partition_id, updated_partition); + updated_topic.partitions = partitions; + } + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + } + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + } + new.revision += 1; + Arc::new(new) + }); + } + + pub fn get_partition_consumer_offsets( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .and_then(|p| p.consumer_offsets.clone()) + } + + pub fn get_partition_consumer_group_offsets( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .and_then(|p| p.consumer_group_offsets.clone()) + } + + /// Register a stream with a specific ID (for bootstrap/recovery). + /// Unlike try_register_stream, this uses a pre-determined stream_id. + pub fn register_stream( + &self, + stream_id: StreamId, + name: Arc, + created_at: IggyTimestamp, + ) -> Arc { + let stats = Arc::new(StreamStats::default()); + let meta = StreamMeta::with_stats(stream_id, name, created_at, stats.clone()); + self.add_stream_with_id(stream_id, meta); + stats + } + + /// Atomically validates name uniqueness and registers stream. + pub fn try_register_stream( + &self, + name: Arc, + created_at: IggyTimestamp, + ) -> Result<(StreamId, Arc), IggyError> { + let stats = Arc::new(StreamStats::default()); + + let name_existed = Arc::new(AtomicBool::new(false)); + let assigned_id = Arc::new(AtomicUsize::new(0)); + let name_existed_clone = name_existed.clone(); + let assigned_id_clone = assigned_id.clone(); + let name_clone = name.clone(); + let stats_clone = stats.clone(); + + self.inner.rcu(move |current| { + if current.stream_index.contains_key(&name_clone) { + name_existed_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + name_existed_clone.store(false, Ordering::Release); + + let mut meta = + StreamMeta::with_stats(0, name_clone.clone(), created_at, stats_clone.clone()); + let mut new = (**current).clone(); + let (streams, id) = new.streams.insert(meta.clone()); + + meta.id = id; + let (streams, _) = streams.update(id, meta); + assigned_id_clone.store(id, Ordering::Release); + new.streams = streams; + new.stream_index = new.stream_index.update(name_clone.clone(), id); + new.revision += 1; + Arc::new(new) + }); + + if name_existed.load(Ordering::Acquire) { + Err(IggyError::StreamNameAlreadyExists(name.to_string())) + } else { + Ok((assigned_id.load(Ordering::Acquire), stats)) + } + } + + /// Atomically validates name uniqueness and registers topic. + #[allow(clippy::too_many_arguments)] + pub fn try_register_topic( + &self, + stream_id: StreamId, + name: Arc, + created_at: IggyTimestamp, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + partitions_count: u32, + ) -> Result<(TopicId, Arc), IggyError> { + let parent_stats = self.get_stream_stats(stream_id).ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + + let stats = Arc::new(TopicStats::new(parent_stats)); + + let name_existed = Arc::new(AtomicBool::new(false)); + let stream_not_found = Arc::new(AtomicBool::new(false)); + let assigned_id = Arc::new(AtomicUsize::new(0)); + let name_existed_clone = name_existed.clone(); + let stream_not_found_clone = stream_not_found.clone(); + let assigned_id_clone = assigned_id.clone(); + let name_clone = name.clone(); + let stats_clone = stats.clone(); + + self.inner.rcu(move |current| { + let Some(stream) = current.streams.get(stream_id) else { + stream_not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + if stream.topic_index.contains_key(&name_clone) { + name_existed_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + name_existed_clone.store(false, Ordering::Release); + stream_not_found_clone.store(false, Ordering::Release); + + let mut meta = TopicMeta { + id: 0, + name: name_clone.clone(), + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + partitions_count, + stats: stats_clone.clone(), + partitions: SegmentedSlab::new(), + consumer_groups: SegmentedSlab::new(), + consumer_group_index: imbl::HashMap::new(), + partition_counter: Arc::new(AtomicUsize::new(0)), + }; + + let mut new = (**current).clone(); + let mut updated_stream = stream.clone(); + let (topics, id) = updated_stream.topics.insert(meta.clone()); + + meta.id = id; + let (topics, _) = topics.update(id, meta); + assigned_id_clone.store(id, Ordering::Release); + updated_stream.topics = topics; + updated_stream.topic_index = updated_stream.topic_index.update(name_clone.clone(), id); + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + new.revision += 1; + Arc::new(new) + }); + + if stream_not_found.load(Ordering::Acquire) { + Err(IggyError::StreamIdNotFound( + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else if name_existed.load(Ordering::Acquire) { + Err(IggyError::TopicNameAlreadyExists( + name.to_string(), + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else { + Ok((assigned_id.load(Ordering::Acquire), stats)) + } + } + + /// Register a topic with a specific ID (for bootstrap/recovery). + /// Unlike try_register_topic, this uses a pre-determined topic_id. + #[allow(clippy::too_many_arguments)] + pub fn register_topic( + &self, + stream_id: StreamId, + topic_id: TopicId, + name: Arc, + created_at: IggyTimestamp, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + partitions_count: u32, + ) -> Arc { + let parent_stats = self + .get_stream_stats(stream_id) + .expect("Stream must exist when registering topic"); + + let stats = Arc::new(TopicStats::new(parent_stats)); + + let stats_clone = stats.clone(); + let name_clone = name.clone(); + + self.inner.rcu(move |current| { + let Some(stream) = current.streams.get(stream_id) else { + return Arc::clone(current); + }; + + let meta = TopicMeta { + id: topic_id, + name: name_clone.clone(), + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + partitions_count, + stats: stats_clone.clone(), + partitions: SegmentedSlab::new(), + consumer_groups: SegmentedSlab::new(), + consumer_group_index: imbl::HashMap::new(), + partition_counter: Arc::new(AtomicUsize::new(0)), + }; + + let mut new = (**current).clone(); + let mut updated_stream = stream.clone(); + let entries: Vec<_> = updated_stream + .topics + .iter() + .map(|(k, v)| (k, v.clone())) + .chain(std::iter::once((topic_id, meta))) + .collect(); + updated_stream.topics = SegmentedSlab::from_entries(entries); + updated_stream.topic_index = updated_stream + .topic_index + .update(name_clone.clone(), topic_id); + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + new.revision += 1; + Arc::new(new) + }); + + stats + } + + /// Atomically validates username uniqueness and registers user. + pub fn try_register_user( + &self, + username: Arc, + password_hash: Arc, + status: iggy_common::UserStatus, + permissions: Option>, + max_users: usize, + ) -> Result { + let name_existed = Arc::new(AtomicBool::new(false)); + let limit_reached = Arc::new(AtomicBool::new(false)); + let assigned_id = Arc::new(AtomicUsize::new(0)); + let name_existed_clone = name_existed.clone(); + let limit_reached_clone = limit_reached.clone(); + let assigned_id_clone = assigned_id.clone(); + let username_clone = username.clone(); + + self.inner.rcu(move |current| { + if current.user_index.contains_key(&username_clone) { + name_existed_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + if current.users.len() >= max_users { + limit_reached_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + name_existed_clone.store(false, Ordering::Release); + limit_reached_clone.store(false, Ordering::Release); + + let mut meta = UserMeta { + id: 0, + username: username_clone.clone(), + password_hash: password_hash.clone(), + status, + permissions: permissions.clone(), + created_at: IggyTimestamp::now(), + }; + + let mut new = (**current).clone(); + let (users, id) = new.users.insert(meta.clone()); + + meta.id = id as UserId; + let (users, _) = users.update(id, meta); + assigned_id_clone.store(id, Ordering::Release); + new.users = users; + new.user_index = new.user_index.update(username_clone.clone(), id as UserId); + new.revision += 1; + Arc::new(new) + }); + + if name_existed.load(Ordering::Acquire) { + Err(IggyError::UserAlreadyExists) + } else if limit_reached.load(Ordering::Acquire) { + Err(IggyError::UsersLimitReached) + } else { + Ok(assigned_id.load(Ordering::Acquire) as UserId) + } + } + + /// Atomically validates consumer group name uniqueness and registers the group. + pub fn try_register_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + name: Arc, + partitions: Vec, + ) -> Result { + let name_existed = Arc::new(AtomicBool::new(false)); + let not_found = Arc::new(AtomicBool::new(false)); + let assigned_id = Arc::new(AtomicUsize::new(0)); + let name_existed_clone = name_existed.clone(); + let not_found_clone = not_found.clone(); + let assigned_id_clone = assigned_id.clone(); + let name_clone = name.clone(); + + self.inner.rcu(move |current| { + let Some(stream) = current.streams.get(stream_id) else { + not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + let Some(topic) = stream.topics.get(topic_id) else { + not_found_clone.store(true, Ordering::Release); + return Arc::clone(current); + }; + + if topic.consumer_group_index.contains_key(&name_clone) { + name_existed_clone.store(true, Ordering::Release); + return Arc::clone(current); + } + + name_existed_clone.store(false, Ordering::Release); + not_found_clone.store(false, Ordering::Release); + + let mut meta = ConsumerGroupMeta { + id: 0, + name: name_clone.clone(), + partitions: partitions.clone(), + members: SegmentedSlab::new(), + }; + + let mut new = (**current).clone(); + let mut updated_stream = stream.clone(); + let mut updated_topic = topic.clone(); + + let (groups, id) = updated_topic.consumer_groups.insert(meta.clone()); + + meta.id = id; + let (groups, _) = groups.update(id, meta); + assigned_id_clone.store(id, Ordering::Release); + updated_topic.consumer_groups = groups; + updated_topic.consumer_group_index = updated_topic + .consumer_group_index + .update(name_clone.clone(), id); + + let (topics, _) = updated_stream.topics.update(topic_id, updated_topic); + updated_stream.topics = topics; + + let (streams, _) = new.streams.update(stream_id, updated_stream); + new.streams = streams; + new.revision += 1; + Arc::new(new) + }); + + if not_found.load(Ordering::Acquire) { + Err(IggyError::TopicIdNotFound( + Identifier::numeric(topic_id as u32).unwrap(), + Identifier::numeric(stream_id as u32).unwrap(), + )) + } else if name_existed.load(Ordering::Acquire) { + Err(IggyError::ConsumerGroupNameAlreadyExists( + name.to_string(), + Identifier::numeric(topic_id as u32).unwrap(), + )) + } else { + Ok(assigned_id.load(Ordering::Acquire)) + } + } + + pub fn register_partitions( + &self, + stream_id: StreamId, + topic_id: TopicId, + count: usize, + created_at: IggyTimestamp, + ) -> Vec> { + if count == 0 { + return Vec::new(); + } + + let parent_stats = self + .get_topic_stats(stream_id, topic_id) + .expect("Parent topic stats must exist before registering partitions"); + + let mut stats_list = Vec::with_capacity(count); + let mut metas = Vec::with_capacity(count); + + for _ in 0..count { + let stats = Arc::new(PartitionStats::new(parent_stats.clone())); + metas.push(PartitionMeta { + id: 0, + created_at, + revision_id: 0, + stats: stats.clone(), + consumer_offsets: None, + consumer_group_offsets: None, + }); + stats_list.push(stats); + } + + self.add_partitions(stream_id, topic_id, metas); + stats_list + } + + /// Register a single partition with a specific ID (for bootstrap/recovery). + pub fn register_partition( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + created_at: IggyTimestamp, + ) -> Arc { + let parent_stats = self + .get_topic_stats(stream_id, topic_id) + .expect("Parent topic stats must exist before registering partition"); + + let stats = Arc::new(PartitionStats::new(parent_stats)); + let meta = PartitionMeta { + id: partition_id, + created_at, + revision_id: 0, + stats: stats.clone(), + consumer_offsets: None, + consumer_group_offsets: None, + }; + + self.add_partitions_with_ids(stream_id, topic_id, vec![(partition_id, meta)]); + stats + } + + pub fn get_user(&self, id: UserId) -> Option { + self.load().users.get(id as usize).cloned() + } + + pub fn get_all_users(&self) -> Vec { + self.load().users.iter().map(|(_, u)| u.clone()).collect() + } + + pub fn get_stream(&self, id: StreamId) -> Option { + self.load().streams.get(id).cloned() + } + + pub fn get_topic(&self, stream_id: StreamId, topic_id: TopicId) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id).cloned()) + } + + pub fn get_partition( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id).cloned()) + } + + pub fn get_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id).cloned()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_stream_crud() { + let metadata = Metadata::default(); + + let stream_meta = StreamMeta::with_stats( + 0, + Arc::from("test-stream"), + IggyTimestamp::now(), + Arc::new(StreamStats::default()), + ); + let id = metadata.add_stream(stream_meta); + + assert!(metadata.stream_exists(id)); + assert_eq!(metadata.streams_count(), 1); + + metadata + .try_update_stream(id, Arc::from("renamed-stream")) + .unwrap(); + let loaded = metadata.load(); + assert_eq!( + loaded.streams.get(id).unwrap().name.as_ref(), + "renamed-stream" + ); + assert!(loaded.stream_index.contains_key("renamed-stream")); + assert!(!loaded.stream_index.contains_key("test-stream")); + + metadata.delete_stream(id); + assert!(!metadata.stream_exists(id)); + assert_eq!(metadata.streams_count(), 0); + } + + #[test] + fn test_cascade_delete() { + let metadata = Metadata::default(); + + let stream_stats = Arc::new(StreamStats::default()); + let stream_meta = StreamMeta::with_stats( + 0, + Arc::from("stream-1"), + IggyTimestamp::now(), + stream_stats.clone(), + ); + let stream_id = metadata.add_stream(stream_meta); + + let topic_stats = Arc::new(TopicStats::new(stream_stats)); + let topic_meta = TopicMeta { + id: 0, + name: Arc::from("topic-1"), + created_at: IggyTimestamp::now(), + message_expiry: IggyExpiry::NeverExpire, + compression_algorithm: CompressionAlgorithm::None, + max_topic_size: MaxTopicSize::Unlimited, + replication_factor: 1, + partitions_count: 0, + stats: topic_stats.clone(), + partitions: SegmentedSlab::new(), + consumer_groups: SegmentedSlab::new(), + consumer_group_index: imbl::HashMap::new(), + partition_counter: Arc::new(AtomicUsize::new(0)), + }; + let topic_id = metadata.add_topic(stream_id, topic_meta).unwrap(); + + let partitions = vec![ + PartitionMeta { + id: 0, + created_at: IggyTimestamp::now(), + revision_id: 0, + stats: Arc::new(PartitionStats::new(topic_stats.clone())), + consumer_offsets: None, + consumer_group_offsets: None, + }, + PartitionMeta { + id: 0, + created_at: IggyTimestamp::now(), + revision_id: 0, + stats: Arc::new(PartitionStats::new(topic_stats)), + consumer_offsets: None, + consumer_group_offsets: None, + }, + ]; + let partition_ids = metadata.add_partitions(stream_id, topic_id, partitions); + + assert!(metadata.stream_exists(stream_id)); + assert!(metadata.topic_exists(stream_id, topic_id)); + assert!(metadata.partition_exists(stream_id, topic_id, partition_ids[0])); + assert!(metadata.partition_exists(stream_id, topic_id, partition_ids[1])); + + metadata.delete_stream(stream_id); + + assert!(!metadata.stream_exists(stream_id)); + assert!(!metadata.topic_exists(stream_id, topic_id)); + assert!(!metadata.partition_exists(stream_id, topic_id, partition_ids[0])); + assert!(!metadata.partition_exists(stream_id, topic_id, partition_ids[1])); + } +} diff --git a/core/server/src/metadata/snapshot.rs b/core/server/src/metadata/snapshot.rs new file mode 100644 index 0000000000..9c314aae66 --- /dev/null +++ b/core/server/src/metadata/snapshot.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{SLAB_SEGMENT_SIZE, StreamId, StreamMeta, UserId, UserMeta}; +use iggy_common::PersonalAccessToken; +use iggy_common::collections::SegmentedSlab; +use imbl::HashMap as ImHashMap; +use std::sync::Arc; + +/// Immutable metadata snapshot with hierarchical structure. +/// Streams contain topics, topics contain partitions and consumer groups. +/// Uses SegmentedSlab for O(1) access with structural sharing. +#[derive(Clone, Default)] +pub struct InnerMetadata { + /// Streams indexed by StreamId (slab-assigned) + pub streams: SegmentedSlab, + + /// Users indexed by UserId (slab-assigned) + pub users: SegmentedSlab, + + /// Forward indexes (name → ID) + pub stream_index: ImHashMap, StreamId>, + pub user_index: ImHashMap, UserId>, + + /// user_id -> (token_hash -> PAT) + pub personal_access_tokens: ImHashMap, PersonalAccessToken>>, + + /// Monotonic revision for cache invalidation + pub revision: u64, +} + +impl InnerMetadata { + pub fn new() -> Self { + Self::default() + } +} diff --git a/core/server/src/metadata/stream.rs b/core/server/src/metadata/stream.rs new file mode 100644 index 0000000000..279ffadab4 --- /dev/null +++ b/core/server/src/metadata/stream.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::topic::TopicMeta; +use crate::metadata::{SLAB_SEGMENT_SIZE, StreamId, TopicId}; +use crate::streaming::stats::StreamStats; +use iggy_common::IggyTimestamp; +use iggy_common::collections::SegmentedSlab; +use imbl::HashMap as ImHashMap; +use std::sync::Arc; + +/// Stream metadata stored in the shared snapshot. +#[derive(Clone, Debug)] +pub struct StreamMeta { + pub id: StreamId, + pub name: Arc, + pub created_at: IggyTimestamp, + pub stats: Arc, + pub topics: SegmentedSlab, + pub topic_index: ImHashMap, TopicId>, +} + +impl StreamMeta { + pub fn new(id: StreamId, name: Arc, created_at: IggyTimestamp) -> Self { + Self { + id, + name, + created_at, + stats: Arc::new(StreamStats::default()), + topics: SegmentedSlab::new(), + topic_index: ImHashMap::new(), + } + } + + pub fn with_stats( + id: StreamId, + name: Arc, + created_at: IggyTimestamp, + stats: Arc, + ) -> Self { + Self { + id, + name, + created_at, + stats, + topics: SegmentedSlab::new(), + topic_index: ImHashMap::new(), + } + } +} diff --git a/core/server/src/metadata/topic.rs b/core/server/src/metadata/topic.rs new file mode 100644 index 0000000000..8f75926ce1 --- /dev/null +++ b/core/server/src/metadata/topic.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group::ConsumerGroupMeta; +use crate::metadata::partition::PartitionMeta; +use crate::metadata::{ConsumerGroupId, SLAB_SEGMENT_SIZE, TopicId}; +use crate::streaming::stats::TopicStats; +use iggy_common::collections::SegmentedSlab; +use iggy_common::{CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize}; +use imbl::HashMap as ImHashMap; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +/// Topic metadata stored in the shared snapshot. +#[derive(Clone, Debug)] +pub struct TopicMeta { + pub id: TopicId, + pub name: Arc, + pub created_at: IggyTimestamp, + pub message_expiry: IggyExpiry, + pub compression_algorithm: CompressionAlgorithm, + pub max_topic_size: MaxTopicSize, + pub replication_factor: u8, + pub partitions_count: u32, + pub stats: Arc, + pub partitions: SegmentedSlab, + pub consumer_groups: SegmentedSlab, + pub consumer_group_index: ImHashMap, ConsumerGroupId>, + pub partition_counter: Arc, +} diff --git a/core/server/src/metadata/user.rs b/core/server/src/metadata/user.rs new file mode 100644 index 0000000000..c82d49a441 --- /dev/null +++ b/core/server/src/metadata/user.rs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::UserId; +use iggy_common::{IggyTimestamp, Permissions, UserStatus}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct UserMeta { + pub id: UserId, + pub username: Arc, + pub password_hash: Arc, + pub status: UserStatus, + pub permissions: Option>, + pub created_at: IggyTimestamp, +} diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs index 67a2d49375..78a5331de0 100644 --- a/core/server/src/quic/listener.rs +++ b/core/server/src/quic/listener.rs @@ -16,7 +16,7 @@ * under the License. */ -use crate::binary::command::{ServerCommand, ServerCommandHandler}; +use crate::binary::command::ServerCommand; use crate::server_error::ConnectionError; use crate::shard::IggyShard; use crate::shard::task_registry::ShutdownToken; @@ -187,7 +187,7 @@ async fn handle_stream( trace!("Received a QUIC command: {command}, payload size: {length}"); - match command.handle(&mut sender, length, session, &shard).await { + match command.dispatch(&mut sender, length, session, &shard).await { Ok(_) => { trace!( "Command was handled successfully, session: {:?}. QUIC response was sent.", diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs index 135a9b4bce..332ebaee33 100644 --- a/core/server/src/shard/builder.rs +++ b/core/server/src/shard/builder.rs @@ -17,31 +17,33 @@ */ use super::{ - IggyShard, TaskRegistry, transmission::connector::ShardConnector, - transmission::frame::ShardFrame, + IggyShard, TaskRegistry, shard_local_partitions::ShardLocalPartitions, + transmission::connector::ShardConnector, transmission::frame::ShardFrame, }; +use crate::metadata::Metadata; use crate::{ configs::server::ServerConfig, - slab::{streams::Streams, users::Users}, state::file::FileState, streaming::{ clients::client_manager::ClientManager, diagnostics::metrics::Metrics, - utils::ptr::EternalPtr, + users::permissioner::Permissioner, utils::ptr::EternalPtr, }, versioning::SemanticVersion, }; use dashmap::DashMap; use iggy_common::EncryptorKind; use iggy_common::sharding::{IggyNamespace, PartitionLocation}; -use std::{cell::Cell, rc::Rc, sync::atomic::AtomicBool}; +use std::{ + cell::{Cell, RefCell}, + rc::Rc, + sync::atomic::AtomicBool, +}; #[derive(Default)] pub struct IggyShardBuilder { id: Option, - streams: Option, shards_table: Option>>, state: Option, - users: Option, client_manager: Option, connections: Option>>, config: Option, @@ -49,6 +51,7 @@ pub struct IggyShardBuilder { version: Option, metrics: Option, is_follower: bool, + shared_metadata: Option>, } impl IggyShardBuilder { @@ -90,21 +93,11 @@ impl IggyShardBuilder { self } - pub fn streams(mut self, streams: Streams) -> Self { - self.streams = Some(streams); - self - } - pub fn state(mut self, state: FileState) -> Self { self.state = Some(state); self } - pub fn users(mut self, users: Users) -> Self { - self.users = Some(users); - self - } - pub fn metrics(mut self, metrics: Metrics) -> Self { self.metrics = Some(metrics); self @@ -115,18 +108,22 @@ impl IggyShardBuilder { self } + pub fn shared_metadata(mut self, shared_metadata: EternalPtr) -> Self { + self.shared_metadata = Some(shared_metadata); + self + } + // TODO: Too much happens in there, some of those bootstrapping logic should be moved outside. pub fn build(self) -> IggyShard { let id = self.id.unwrap(); - let streams = self.streams.unwrap(); let shards_table = self.shards_table.unwrap(); let state = self.state.unwrap(); - let users = self.users.unwrap(); let config = self.config.unwrap(); let connections = self.connections.unwrap(); let encryptor = self.encryptor; let client_manager = self.client_manager.unwrap(); let version = self.version.unwrap(); + let shared_metadata = self.shared_metadata.expect("shared_metadata is required"); let (stop_receiver, frame_receiver) = connections .iter() .filter(|c| c.id == id) @@ -150,12 +147,17 @@ impl IggyShardBuilder { // Trigger initial check in case servers bind before task starts let _ = config_writer_notify.try_send(()); + // Create per-shard stores (wrapped in RefCell for interior mutability) + let partition_store = RefCell::new(ShardLocalPartitions::new()); + let permissioner = Permissioner::new(shared_metadata.clone()); + IggyShard { id, shards, shards_table, - streams, // TODO: Fixme - users, + metadata: shared_metadata, + partition_store, + pending_partition_inits: RefCell::new(std::collections::HashSet::new()), fs_locks: Default::default(), encryptor, config, @@ -173,7 +175,7 @@ impl IggyShardBuilder { config_writer_notify, config_writer_receiver, task_registry, - permissioner: Default::default(), + permissioner, client_manager, } } diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs index 087b7a54d1..37f527587a 100644 --- a/core/server/src/shard/handlers.rs +++ b/core/server/src/shard/handlers.rs @@ -19,21 +19,21 @@ use super::*; use crate::{ shard::{ IggyShard, - namespace::IggyFullNamespace, + namespace::IggyNamespace, transmission::{ event::ShardEvent, frame::ShardResponse, message::{ShardMessage, ShardRequest, ShardRequestPayload}, }, }, - streaming::{session::Session, traits::MainOps}, + streaming::session::Session, tcp::{ connection_handler::{ConnectionAction, handle_connection, handle_error}, tcp_listener::cleanup_connection, }, }; use compio_net::TcpStream; -use iggy_common::{Identifier, IggyError, SenderKind, TransportProtocol}; +use iggy_common::{Identifier, IggyError, IggyTimestamp, SenderKind, TransportProtocol}; use nix::sys::stat::SFlag; use std::os::fd::{FromRawFd, IntoRawFd}; use tracing::info; @@ -63,73 +63,232 @@ async fn handle_request( let partition_id = request.partition_id; match request.payload { ShardRequestPayload::SendMessages { batch } => { - let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id); let batch = shard.maybe_encrypt_messages(batch)?; let messages_count = batch.count(); - shard + + let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?; + let namespace = IggyNamespace::new(stream, topic, partition_id); + + let metadata = shard.metadata.load(); + let partition_meta = metadata .streams - .append_messages(&shard.config.system, &shard.task_registry, &ns, batch) + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.partitions.get(partition_id)); + + let needs_init = { + let store = shard.partition_store.borrow(); + match (store.get(&namespace), partition_meta) { + (Some(data), Some(meta)) if data.revision_id == meta.revision_id => false, + (Some(_), _) => { + drop(store); + shard.partition_store.borrow_mut().remove(&namespace); + true + } + (None, _) => true, + } + }; + + if needs_init { + let created_at = partition_meta + .map(|m| m.created_at) + .unwrap_or_else(IggyTimestamp::now); + + shard + .init_partition_directly(stream, topic, partition_id, created_at) + .await?; + } + + shard + .append_messages_to_partition_store(&namespace, batch, &shard.config.system) .await?; + shard.metrics.increment_messages(messages_count as u64); Ok(ShardResponse::SendMessages) } ShardRequestPayload::PollMessages { args, consumer } => { let auto_commit = args.auto_commit; - let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id); - let (metadata, batches) = shard.streams.poll_messages(&ns, consumer, args).await?; + + let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?; + let namespace = IggyNamespace::new(stream, topic, partition_id); + + let metadata = shard.metadata.load(); + let partition_meta = metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.partitions.get(partition_id)); + + let needs_init = { + let store = shard.partition_store.borrow(); + match (store.get(&namespace), partition_meta) { + (Some(data), Some(meta)) if data.revision_id == meta.revision_id => false, + (Some(_), _) => { + drop(store); + shard.partition_store.borrow_mut().remove(&namespace); + true + } + (None, _) => true, + } + }; + + if needs_init { + let created_at = partition_meta + .map(|m| m.created_at) + .unwrap_or_else(IggyTimestamp::now); + + shard + .init_partition_directly(stream, topic, partition_id, created_at) + .await?; + } + + let (metadata, batches) = shard + .poll_messages_from_partition_store(&namespace, consumer, args) + .await?; if auto_commit && !batches.is_empty() { let offset = batches .last_offset() .expect("Batch set should have at least one batch"); shard - .streams - .auto_commit_consumer_offset( - &shard.config.system, - ns.stream_id(), - ns.topic_id(), - partition_id, - consumer, - offset, - ) + .auto_commit_consumer_offset_from_partition_store(&namespace, consumer, offset) .await?; } Ok(ShardResponse::PollMessages((metadata, batches))) } ShardRequestPayload::FlushUnsavedBuffer { fsync } => { + let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?; shard - .flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync) + .flush_unsaved_buffer_base(stream, topic, partition_id, fsync) .await?; Ok(ShardResponse::FlushUnsavedBuffer) } ShardRequestPayload::DeleteSegments { segments_count } => { + let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?; shard - .delete_segments_base(&stream_id, &topic_id, partition_id, segments_count) + .delete_segments_base(stream, topic, partition_id, segments_count) .await?; Ok(ShardResponse::DeleteSegments) } - ShardRequestPayload::CreateStream { user_id, name } => { - assert_eq!(shard.id, 0, "CreateStream should only be handled by shard0"); + ShardRequestPayload::CreatePartitions { + user_id, + stream_id, + topic_id, + partitions_count, + } => { + assert_eq!( + shard.id, 0, + "CreatePartitions should only be handled by shard0" + ); let session = Session::stateless( user_id, std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), ); - // Acquire stream lock to serialize filesystem operations - let _stream_guard = shard.fs_locks.stream_lock.lock().await; + let _partition_guard = shard.fs_locks.partition_lock.lock().await; - let stream = shard.create_stream(&session, name.clone()).await?; - let created_stream_id = stream.id(); + let partition_infos = shard + .create_partitions(&session, &stream_id, &topic_id, partitions_count) + .await?; + let partition_ids = partition_infos.iter().map(|p| p.id).collect::>(); - let event = ShardEvent::CreatedStream { - id: created_stream_id, - stream: stream.clone(), + let event = ShardEvent::CreatedPartitions { + stream_id: stream_id.clone(), + topic_id: topic_id.clone(), + partitions: partition_infos, }; + shard.broadcast_event_to_all_shards(event).await?; + + // Rebalance consumer groups using SharedMetadata + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id) + .expect("Stream must exist"); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .expect("Topic must exist"); + shard.metadata.rebalance_consumer_groups_for_topic( + numeric_stream_id, + numeric_topic_id, + &partition_ids, + ); + + Ok(ShardResponse::CreatePartitionsResponse(partition_ids)) + } + ShardRequestPayload::DeletePartitions { + user_id, + stream_id, + topic_id, + partitions_count, + } => { + assert_eq!( + shard.id, 0, + "DeletePartitions should only be handled by shard0" + ); + + let session = Session::stateless( + user_id, + std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), + ); + let _partition_guard = shard.fs_locks.partition_lock.lock().await; + + let deleted_partition_ids = shard + .delete_partitions(&session, &stream_id, &topic_id, partitions_count) + .await?; + + let event = ShardEvent::DeletedPartitions { + stream_id: stream_id.clone(), + topic_id: topic_id.clone(), + partitions_count, + partition_ids: deleted_partition_ids.clone(), + }; shard.broadcast_event_to_all_shards(event).await?; - Ok(ShardResponse::CreateStreamResponse(stream)) + // Rebalance consumer groups using SharedMetadata + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id) + .expect("Stream must exist"); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .expect("Topic must exist"); + let remaining_partition_ids: Vec<_> = { + let metadata = shard.metadata.load(); + metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .map(|t| t.partitions.keys().collect()) + .unwrap_or_default() + }; + shard.metadata.rebalance_consumer_groups_for_topic( + numeric_stream_id, + numeric_topic_id, + &remaining_partition_ids, + ); + + Ok(ShardResponse::DeletePartitionsResponse( + deleted_partition_ids, + )) + } + ShardRequestPayload::CreateStream { user_id, name } => { + assert_eq!(shard.id, 0, "CreateStream should only be handled by shard0"); + + let session = Session::stateless( + user_id, + std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), + ); + + // Acquire stream lock to serialize filesystem operations + let _stream_guard = shard.fs_locks.stream_lock.lock().await; + + let created_stream_id = shard.create_stream(&session, name.clone()).await?; + + Ok(ShardResponse::CreateStreamResponse(created_stream_id)) } ShardRequestPayload::CreateTopic { user_id, @@ -151,7 +310,7 @@ async fn handle_request( // Acquire topic lock to serialize filesystem operations let _topic_guard = shard.fs_locks.topic_lock.lock().await; - let topic = shard + let topic_id_num = shard .create_topic( &session, &stream_id, @@ -163,29 +322,23 @@ async fn handle_request( ) .await?; - let topic_id = topic.id(); - let event = ShardEvent::CreatedTopic { - stream_id: stream_id.clone(), - topic: topic.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - let partitions = shard + let partition_infos = shard .create_partitions( &session, &stream_id, - &Identifier::numeric(topic_id as u32).unwrap(), + &Identifier::numeric(topic_id_num as u32).unwrap(), partitions_count, ) .await?; let event = ShardEvent::CreatedPartitions { stream_id: stream_id.clone(), - topic_id: Identifier::numeric(topic_id as u32).unwrap(), - partitions, + topic_id: Identifier::numeric(topic_id_num as u32).unwrap(), + partitions: partition_infos, }; shard.broadcast_event_to_all_shards(event).await?; - Ok(ShardResponse::CreateTopicResponse(topic)) + Ok(ShardResponse::CreateTopicResponse(topic_id_num)) } ShardRequestPayload::UpdateTopic { user_id, @@ -215,17 +368,6 @@ async fn handle_request( replication_factor, )?; - let event = ShardEvent::UpdatedTopic { - stream_id: stream_id.clone(), - topic_id: topic_id.clone(), - name, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - }; - shard.broadcast_event_to_all_shards(event).await?; - Ok(ShardResponse::UpdateTopicResponse) } ShardRequestPayload::DeleteTopic { @@ -241,17 +383,10 @@ async fn handle_request( ); let _topic_guard = shard.fs_locks.topic_lock.lock().await; - let topic = shard.delete_topic(&session, &stream_id, &topic_id).await?; - let topic_id_num = topic.root().id(); + let topic_info = shard.delete_topic(&session, &stream_id, &topic_id).await?; + let topic_id_num = topic_info.id; - let event = ShardEvent::DeletedTopic { - id: topic_id_num, - stream_id: stream_id.clone(), - topic_id: topic_id.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; - - Ok(ShardResponse::DeleteTopicResponse(topic)) + Ok(ShardResponse::DeleteTopicResponse(topic_id_num)) } ShardRequestPayload::CreateUser { user_id, @@ -270,16 +405,6 @@ async fn handle_request( let user = shard.create_user(&session, &username, &password, status, permissions.clone())?; - let created_user_id = user.id; - - let event = ShardEvent::CreatedUser { - user_id: created_user_id, - username: username.clone(), - password: password.clone(), - status, - permissions: permissions.clone(), - }; - shard.broadcast_event_to_all_shards(event).await?; Ok(ShardResponse::CreateUserResponse(user)) } ShardRequestPayload::GetStats { .. } => { @@ -299,10 +424,24 @@ async fn handle_request( ); let _user_guard = shard.fs_locks.user_lock.lock().await; let user = shard.delete_user(&session, &user_id)?; - let event = ShardEvent::DeletedUser { user_id }; - shard.broadcast_event_to_all_shards(event).await?; Ok(ShardResponse::DeletedUser(user)) } + ShardRequestPayload::UpdateStream { + user_id, + stream_id, + name, + } => { + assert_eq!(shard.id, 0, "UpdateStream should only be handled by shard0"); + + let session = Session::stateless( + user_id, + std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), + ); + + shard.update_stream(&session, &stream_id, name.clone())?; + + Ok(ShardResponse::UpdateStreamResponse) + } ShardRequestPayload::DeleteStream { user_id, stream_id } => { assert_eq!(shard.id, 0, "DeleteStream should only be handled by shard0"); @@ -311,13 +450,49 @@ async fn handle_request( std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), ); let _stream_guard = shard.fs_locks.stream_lock.lock().await; - let stream = shard.delete_stream(&session, &stream_id).await?; - let event = ShardEvent::DeletedStream { - id: stream.id(), - stream_id, - }; - shard.broadcast_event_to_all_shards(event).await?; - Ok(ShardResponse::DeleteStreamResponse(stream)) + let stream_info = shard.delete_stream(&session, &stream_id).await?; + let stream_id_num = stream_info.id; + + Ok(ShardResponse::DeleteStreamResponse(stream_id_num)) + } + ShardRequestPayload::UpdatePermissions { + session_user_id, + user_id, + permissions, + } => { + assert_eq!( + shard.id, 0, + "UpdatePermissions should only be handled by shard0" + ); + + let session = Session::stateless( + session_user_id, + std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), + ); + let _user_guard = shard.fs_locks.user_lock.lock().await; + shard.update_permissions(&session, &user_id, permissions)?; + + Ok(ShardResponse::UpdatePermissionsResponse) + } + ShardRequestPayload::ChangePassword { + session_user_id, + user_id, + current_password, + new_password, + } => { + assert_eq!( + shard.id, 0, + "ChangePassword should only be handled by shard0" + ); + + let session = Session::stateless( + session_user_id, + std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0), + ); + let _user_guard = shard.fs_locks.user_lock.lock().await; + shard.change_password(&session, &user_id, ¤t_password, &new_password)?; + + Ok(ShardResponse::ChangePasswordResponse) } ShardRequestPayload::SocketTransfer { fd, @@ -351,13 +526,47 @@ async fn handle_request( let registry = shard.task_registry.clone(); let registry_clone = registry.clone(); - let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id); let batch = shard.maybe_encrypt_messages(initial_data)?; let messages_count = batch.count(); + // Get numeric IDs for partition_store lookup + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id) + .expect("Stream must exist"); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .expect("Topic must exist"); + + // Use partition_store - initialize on-demand if not yet ready + let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + if !shard.partition_store.borrow().contains(&namespace) { + // Partition not yet initialized (race with CreatedPartitions event) + // Initialize it on-demand using SharedMetadata + let created_at = { + let metadata = shard.metadata.load(); + metadata + .streams + .get(numeric_stream_id) + .and_then(|s| s.topics.get(numeric_topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|meta| meta.created_at) + .unwrap_or_else(IggyTimestamp::now) + }; + + shard + .init_partition_directly( + numeric_stream_id, + numeric_topic_id, + partition_id, + created_at, + ) + .await?; + } + shard - .streams - .append_messages(&shard.config.system, &shard.task_registry, &ns, batch) + .append_messages_to_partition_store(&namespace, batch, &shard.config.system) .await?; shard.metrics.increment_messages(messages_count as u64); @@ -405,19 +614,25 @@ pub async fn handle_event(shard: &Rc, event: ShardEvent) -> Result<() ShardEvent::DeletedPartitions { stream_id, topic_id, - partitions_count, + partitions_count: _, partition_ids, } => { - shard.delete_partitions_bypass_auth( - &stream_id, - &topic_id, - partitions_count, - partition_ids, - )?; - Ok(()) - } - ShardEvent::UpdatedStream { stream_id, name } => { - shard.update_stream_bypass_auth(&stream_id, &name)?; + // SharedMetadata was already updated by the request handler before broadcasting. + // Here we only need to clean up local partition_store entries on all shards. + let numeric_stream_id = shard.metadata.get_stream_id(&stream_id).unwrap_or_default(); + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .unwrap_or_default(); + let mut store = shard.partition_store.borrow_mut(); + for partition_id in partition_ids { + let ns = crate::shard::namespace::IggyNamespace::new( + numeric_stream_id, + numeric_topic_id, + partition_id, + ); + store.remove(&ns); + } Ok(()) } ShardEvent::PurgedStream { stream_id } => { @@ -431,59 +646,6 @@ pub async fn handle_event(shard: &Rc, event: ShardEvent) -> Result<() shard.purge_topic_bypass_auth(&stream_id, &topic_id).await?; Ok(()) } - ShardEvent::CreatedUser { - user_id, - username, - password, - status, - permissions, - } => { - shard.create_user_bypass_auth( - user_id, - &username, - &password, - status, - permissions.clone(), - )?; - Ok(()) - } - ShardEvent::DeletedUser { user_id } => { - shard.delete_user_bypass_auth(&user_id)?; - Ok(()) - } - ShardEvent::ChangedPassword { - user_id, - current_password, - new_password, - } => { - shard.change_password_bypass_auth(&user_id, ¤t_password, &new_password)?; - Ok(()) - } - ShardEvent::CreatedPersonalAccessToken { - personal_access_token, - } => { - shard.create_personal_access_token_bypass_auth(personal_access_token.to_owned())?; - Ok(()) - } - ShardEvent::DeletedPersonalAccessToken { user_id, name } => { - shard.delete_personal_access_token_bypass_auth(user_id, &name)?; - Ok(()) - } - ShardEvent::UpdatedUser { - user_id, - username, - status, - } => { - shard.update_user_bypass_auth(&user_id, username.to_owned(), status)?; - Ok(()) - } - ShardEvent::UpdatedPermissions { - user_id, - permissions, - } => { - shard.update_permissions_bypass_auth(&user_id, permissions.to_owned())?; - Ok(()) - } ShardEvent::AddressBound { protocol, address } => { info!( "Received AddressBound event for {:?} with address: {}", @@ -509,92 +671,88 @@ pub async fn handle_event(shard: &Rc, event: ShardEvent) -> Result<() } Ok(()) } - ShardEvent::CreatedStream { id, stream } => { - let stream_id = shard.create_stream_bypass_auth(stream); - assert_eq!(stream_id, id); - Ok(()) - } - ShardEvent::DeletedStream { id, stream_id } => { - let stream = shard.delete_stream_bypass_auth(&stream_id); - assert_eq!(stream.id(), id); - Ok(()) - } - ShardEvent::CreatedTopic { stream_id, topic } => { - let topic_id_from_event = topic.id(); - let topic_id = shard.create_topic_bypass_auth(&stream_id, topic.clone()); - assert_eq!(topic_id, topic_id_from_event); - Ok(()) - } + // ======================================================================== + // PARTITION EVENTS - Need to initialize logs in partition_store + // ======================================================================== ShardEvent::CreatedPartitions { stream_id, topic_id, partitions, } => { - shard - .create_partitions_bypass_auth(&stream_id, &topic_id, partitions) - .await?; - Ok(()) - } - ShardEvent::DeletedTopic { - id, - stream_id, - topic_id, - } => { - let topic = shard.delete_topic_bypass_auth(&stream_id, &topic_id); - assert_eq!(topic.id(), id); - Ok(()) - } - ShardEvent::UpdatedTopic { - stream_id, - topic_id, - name, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - } => { - shard.update_topic_bypass_auth( - &stream_id, - &topic_id, - name.clone(), - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - )?; - Ok(()) - } - ShardEvent::CreatedConsumerGroup { - stream_id, - topic_id, - cg, - } => { - let cg_id = cg.id(); - let id = shard.create_consumer_group_bypass_auth(&stream_id, &topic_id, cg); - assert_eq!(id, cg_id); - Ok(()) - } - ShardEvent::DeletedConsumerGroup { - id, - stream_id, - topic_id, - group_id, - } => { - let cg = shard.delete_consumer_group_bypass_auth(&stream_id, &topic_id, &group_id); - assert_eq!(cg.id(), id); + let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { + Some(id) => id, + None => { + tracing::warn!( + "CreatedPartitions: stream {:?} not found in SharedMetadata", + stream_id + ); + return Ok(()); + } + }; + let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { + Some(id) => id, + None => { + tracing::warn!( + "CreatedPartitions: topic {:?} not found in SharedMetadata for stream {}", + topic_id, + numeric_stream_id + ); + return Ok(()); + } + }; + let shards_count = shard.get_available_shards_count(); + for partition_info in partitions { + let ns = crate::shard::namespace::IggyNamespace::new( + numeric_stream_id, + numeric_topic_id, + partition_info.id, + ); + let owner_shard_id = crate::shard::calculate_shard_assignment(&ns, shards_count); + + if shard.id == owner_shard_id as u16 { + shard + .init_partition_directly( + numeric_stream_id, + numeric_topic_id, + partition_info.id, + partition_info.created_at, + ) + .await?; + } + } Ok(()) } + + // ======================================================================== + // OPERATIONAL EVENTS - Work directly with partition_store + // ======================================================================== ShardEvent::FlushUnsavedBuffer { stream_id, topic_id, partition_id, fsync, } => { - shard - .flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync) - .await?; + let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { + Some(id) => id, + None => return Ok(()), + }; + let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { + Some(id) => id, + None => return Ok(()), + }; + + let ns = crate::shard::namespace::IggyNamespace::new( + numeric_stream_id, + numeric_topic_id, + partition_id, + ); + if shard.partition_store.borrow().get(&ns).is_some() { + shard + .flush_unsaved_buffer_from_partition_store(&ns, fsync) + .await?; + } Ok(()) } } diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs index 8939e87b09..d8bb358362 100644 --- a/core/server/src/shard/mod.rs +++ b/core/server/src/shard/mod.rs @@ -18,6 +18,7 @@ inner() * or more contributor license agreements. See the NOTICE file pub mod builder; pub mod namespace; +pub mod shard_local_partitions; pub mod system; pub mod task_registry; pub mod tasks; @@ -33,8 +34,11 @@ use self::tasks::{continuous, periodic}; use crate::{ configs::server::ServerConfig, io::fs_locks::FsLocks, - shard::{task_registry::TaskRegistry, transmission::frame::ShardFrame}, - slab::{streams::Streams, traits_ext::EntityMarker, users::Users}, + metadata::Metadata, + shard::{ + namespace::IggyNamespace, shard_local_partitions::ShardLocalPartitions, + task_registry::TaskRegistry, transmission::frame::ShardFrame, + }, state::file::FileState, streaming::{ clients::client_manager::ClientManager, diagnostics::metrics::Metrics, session::Session, @@ -44,13 +48,16 @@ use crate::{ }; use builder::IggyShardBuilder; use dashmap::DashMap; -use iggy_common::sharding::{IggyNamespace, PartitionLocation}; -use iggy_common::{EncryptorKind, Identifier, IggyError}; +use iggy_common::sharding::PartitionLocation; +use iggy_common::{EncryptorKind, IggyError}; use std::{ cell::{Cell, RefCell}, net::SocketAddr, rc::Rc, - sync::atomic::{AtomicBool, Ordering}, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, time::{Duration, Instant}, }; use tracing::{debug, error, info, instrument}; @@ -65,7 +72,11 @@ pub struct IggyShard { shards: Vec>, _version: SemanticVersion, - pub(crate) streams: Streams, + // Shared metadata structures (source of truth) + pub(crate) metadata: EternalPtr, + pub(crate) partition_store: RefCell, + pub(crate) pending_partition_inits: RefCell>, + pub(crate) shards_table: EternalPtr>, pub(crate) state: FileState, @@ -73,8 +84,7 @@ pub struct IggyShard { pub(crate) encryptor: Option, pub(crate) config: ServerConfig, pub(crate) client_manager: ClientManager, - pub(crate) permissioner: RefCell, - pub(crate) users: Users, + pub(crate) permissioner: Permissioner, pub(crate) metrics: Metrics, pub(crate) is_follower: bool, pub messages_receiver: Cell>>, @@ -185,6 +195,8 @@ impl IggyShard { async fn load_segments(&self) -> Result<(), IggyError> { use crate::bootstrap::load_segments; + use crate::shard::shard_local_partitions::PartitionData; + for shard_entry in self.shards_table.iter() { let (namespace, location) = shard_entry.pair(); @@ -202,33 +214,98 @@ impl IggyShard { self.config .system .get_partition_path(stream_id, topic_id, partition_id); - let stats = self.streams.with_partition_by_id( - &Identifier::numeric(stream_id as u32).unwrap(), - &Identifier::numeric(topic_id as u32).unwrap(), - partition_id, - |(_, stats, ..)| stats.clone(), + + let metadata = self.metadata.load(); + let partition_meta = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .expect("Partition must exist in SharedMetadata"); + let created_at = partition_meta.created_at; + let stats = partition_meta.stats.clone(); + drop(metadata); + + use crate::streaming::partitions::helpers::create_message_deduplicator; + use crate::streaming::partitions::storage::{ + load_consumer_group_offsets, load_consumer_offsets, + }; + use ahash::HashMap; + + let consumer_offset_path = + self.config + .system + .get_consumer_offsets_path(stream_id, topic_id, partition_id); + let consumer_group_offsets_path = self + .config + .system + .get_consumer_group_offsets_path(stream_id, topic_id, partition_id); + + let consumer_offsets = Arc::new( + load_consumer_offsets(&consumer_offset_path) + .unwrap_or_default() + .into_iter() + .map(|offset| (offset.consumer_id as usize, offset)) + .collect::>() + .into(), + ); + + let consumer_group_offsets = Arc::new( + load_consumer_group_offsets(&consumer_group_offsets_path) + .unwrap_or_default() + .into_iter() + .collect::>() + .into(), ); + + let message_deduplicator = + create_message_deduplicator(&self.config.system).map(Arc::new); + match load_segments( &self.config.system, stream_id, topic_id, partition_id, partition_path, - stats, + stats.clone(), ) .await { Ok(loaded_log) => { - self.streams.with_partition_by_id_mut( - &Identifier::numeric(stream_id as u32).unwrap(), - &Identifier::numeric(topic_id as u32).unwrap(), - partition_id, - |(_, _, _, offset, .., log)| { - *log = loaded_log; - let current_offset = log.active_segment().end_offset; - offset.store(current_offset, Ordering::Relaxed); - }, + let current_offset = loaded_log.active_segment().end_offset; + stats.set_current_offset(current_offset); + + // Only increment offset if we have messages (current_offset > 0). + // When current_offset is 0 and we have no messages, first message + // should get offset 0. + let should_increment_offset = current_offset > 0; + + let revision_id = self + .metadata + .load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|meta| meta.revision_id) + .unwrap_or(0); + + let partition_data = PartitionData::with_log( + loaded_log, + stats, + Arc::new(AtomicU64::new(current_offset)), + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + created_at, + revision_id, + should_increment_offset, ); + + self.partition_store + .borrow_mut() + .insert(*namespace, partition_data); + info!( "Successfully loaded segments for stream: {}, topic: {}, partition: {}", stream_id, topic_id, partition_id @@ -249,11 +326,8 @@ impl IggyShard { } async fn load_users(&self) -> Result<(), IggyError> { - let users_list = self.users.values(); - let users_count = users_list.len(); - self.permissioner - .borrow_mut() - .init(&users_list.iter().collect::>()); + // Permissioner reads directly from SharedMetadata, no initialization needed + let users_count = self.metadata.users_count(); self.metrics.increment_users(users_count as u32); info!("Initialized {} user(s).", users_count); Ok(()) @@ -295,4 +369,27 @@ impl IggyShard { Err(IggyError::Unauthenticated) } } + + /// Authenticates a session and returns a proof token. + /// + /// This is the ONLY way to create an [`Auth`] token. The returned token + /// proves that authentication was successful and can be passed to shard + /// methods that require authentication. + /// + /// # Errors + /// - [`IggyError::StaleClient`] if the session is inactive + /// - [`IggyError::Unauthenticated`] if the session is not authenticated + pub fn auth(&self, session: &Session) -> Result { + if !session.is_active() { + error!("{COMPONENT} - session is inactive, session: {session}"); + return Err(IggyError::StaleClient); + } + + if !session.is_authenticated() { + error!("{COMPONENT} - unauthenticated access attempt, session: {session}"); + return Err(IggyError::Unauthenticated); + } + + Ok(crate::streaming::auth::Auth::new(session.get_user_id())) + } } diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs index 2895871796..392fa2b154 100644 --- a/core/server/src/shard/namespace.rs +++ b/core/server/src/shard/namespace.rs @@ -16,18 +16,20 @@ * under the License. */ -use crate::slab::partitions; use iggy_common::Identifier; +// Re-export IggyNamespace from iggy_common for backwards compatibility +pub use iggy_common::sharding::IggyNamespace; + #[derive(Debug)] pub struct IggyFullNamespace { stream: Identifier, topic: Identifier, - partition: partitions::ContainerId, + partition: usize, } impl IggyFullNamespace { - pub fn new(stream: Identifier, topic: Identifier, partition: partitions::ContainerId) -> Self { + pub fn new(stream: Identifier, topic: Identifier, partition: usize) -> Self { Self { stream, topic, @@ -43,11 +45,11 @@ impl IggyFullNamespace { &self.topic } - pub fn partition_id(&self) -> partitions::ContainerId { + pub fn partition_id(&self) -> usize { self.partition } - pub fn decompose(self) -> (Identifier, Identifier, partitions::ContainerId) { + pub fn decompose(self) -> (Identifier, Identifier, usize) { (self.stream, self.topic, self.partition) } } diff --git a/core/server/src/shard/shard_local_partitions.rs b/core/server/src/shard/shard_local_partitions.rs new file mode 100644 index 0000000000..9b84b69c66 --- /dev/null +++ b/core/server/src/shard/shard_local_partitions.rs @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-shard partition data storage. +//! +//! Each shard runs on a single-threaded compio runtime, so per-shard data +//! needs NO synchronization. This uses a plain HashMap for maximum performance. + +use super::namespace::IggyNamespace; +use crate::streaming::{ + deduplication::message_deduplicator::MessageDeduplicator, + partitions::{ + journal::MemoryMessageJournal, + log::SegmentedLog, + partition::{ConsumerGroupOffsets, ConsumerOffsets}, + }, + stats::PartitionStats, +}; +use iggy_common::IggyTimestamp; +use std::{ + collections::HashMap, + sync::{Arc, atomic::AtomicU64}, +}; + +/// Per-shard partition data - mutable, single-threaded access. +#[derive(Debug)] +pub struct PartitionData { + pub log: SegmentedLog, + pub offset: Arc, + pub consumer_offsets: Arc, + pub consumer_group_offsets: Arc, + pub message_deduplicator: Option>, + pub stats: Arc, + pub created_at: IggyTimestamp, + pub revision_id: u64, + pub should_increment_offset: bool, +} + +impl PartitionData { + /// Create new partition data with default log. + #[allow(clippy::too_many_arguments)] + pub fn new( + stats: Arc, + offset: Arc, + consumer_offsets: Arc, + consumer_group_offsets: Arc, + message_deduplicator: Option>, + created_at: IggyTimestamp, + revision_id: u64, + should_increment_offset: bool, + ) -> Self { + Self { + log: SegmentedLog::default(), + offset, + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + stats, + created_at, + revision_id, + should_increment_offset, + } + } + + /// Create partition data with existing log (e.g., loaded from disk). + #[allow(clippy::too_many_arguments)] + pub fn with_log( + log: SegmentedLog, + stats: Arc, + offset: Arc, + consumer_offsets: Arc, + consumer_group_offsets: Arc, + message_deduplicator: Option>, + created_at: IggyTimestamp, + revision_id: u64, + should_increment_offset: bool, + ) -> Self { + Self { + log, + offset, + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + stats, + created_at, + revision_id, + should_increment_offset, + } + } +} + +/// Per-shard partition data storage. +/// Single-threaded (compio runtime) - NO synchronization needed! +#[derive(Debug, Default)] +pub struct ShardLocalPartitions { + partitions: HashMap, +} + +impl ShardLocalPartitions { + pub fn new() -> Self { + Self { + partitions: HashMap::new(), + } + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + partitions: HashMap::with_capacity(capacity), + } + } + + #[inline] + pub fn get(&self, ns: &IggyNamespace) -> Option<&PartitionData> { + self.partitions.get(ns) + } + + #[inline] + pub fn get_mut(&mut self, ns: &IggyNamespace) -> Option<&mut PartitionData> { + self.partitions.get_mut(ns) + } + + #[inline] + pub fn insert(&mut self, ns: IggyNamespace, data: PartitionData) { + self.partitions.insert(ns, data); + } + + #[inline] + pub fn remove(&mut self, ns: &IggyNamespace) -> Option { + self.partitions.remove(ns) + } + + #[inline] + pub fn contains(&self, ns: &IggyNamespace) -> bool { + self.partitions.contains_key(ns) + } + + #[inline] + pub fn len(&self) -> usize { + self.partitions.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.partitions.is_empty() + } + + /// Iterate over all namespaces owned by this shard. + pub fn namespaces(&self) -> impl Iterator { + self.partitions.keys() + } + + /// Iterate over all partition data. + pub fn iter(&self) -> impl Iterator { + self.partitions.iter() + } + + /// Iterate over all partition data mutably. + pub fn iter_mut(&mut self) -> impl Iterator { + self.partitions.iter_mut() + } + + /// Remove multiple partitions at once. + pub fn remove_many(&mut self, namespaces: &[IggyNamespace]) -> Vec { + namespaces + .iter() + .filter_map(|ns| self.partitions.remove(ns)) + .collect() + } + + /// Get partition data, initializing if not present. + /// Returns None if initialization fails. + pub fn get_or_init(&mut self, ns: IggyNamespace, init: F) -> &mut PartitionData + where + F: FnOnce() -> PartitionData, + { + self.partitions.entry(ns).or_insert_with(init) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::streaming::stats::{StreamStats, TopicStats}; + + fn create_test_partition_data() -> PartitionData { + let stream_stats = Arc::new(StreamStats::default()); + let topic_stats = Arc::new(TopicStats::new(stream_stats)); + let partition_stats = Arc::new(PartitionStats::new(topic_stats)); + + PartitionData::new( + partition_stats, + Arc::new(AtomicU64::new(0)), + Arc::new(ConsumerOffsets::with_capacity(10)), + Arc::new(ConsumerGroupOffsets::with_capacity(10)), + None, + IggyTimestamp::now(), + 1, // revision_id + true, + ) + } + + #[test] + fn test_basic_operations() { + let mut store = ShardLocalPartitions::new(); + let ns = IggyNamespace::new(1, 1, 0); + + assert!(!store.contains(&ns)); + assert!(store.is_empty()); + + store.insert(ns, create_test_partition_data()); + + assert!(store.contains(&ns)); + assert_eq!(store.len(), 1); + assert!(store.get(&ns).is_some()); + assert!(store.get_mut(&ns).is_some()); + + let removed = store.remove(&ns); + assert!(removed.is_some()); + assert!(!store.contains(&ns)); + assert!(store.is_empty()); + } + + #[test] + fn test_iteration() { + let mut store = ShardLocalPartitions::new(); + let ns1 = IggyNamespace::new(1, 1, 0); + let ns2 = IggyNamespace::new(1, 1, 1); + let ns3 = IggyNamespace::new(1, 2, 0); + + store.insert(ns1, create_test_partition_data()); + store.insert(ns2, create_test_partition_data()); + store.insert(ns3, create_test_partition_data()); + + let namespaces: Vec<_> = store.namespaces().collect(); + assert_eq!(namespaces.len(), 3); + + let pairs: Vec<_> = store.iter().collect(); + assert_eq!(pairs.len(), 3); + } + + #[test] + fn test_remove_many() { + let mut store = ShardLocalPartitions::new(); + let ns1 = IggyNamespace::new(1, 1, 0); + let ns2 = IggyNamespace::new(1, 1, 1); + let ns3 = IggyNamespace::new(1, 2, 0); + + store.insert(ns1, create_test_partition_data()); + store.insert(ns2, create_test_partition_data()); + store.insert(ns3, create_test_partition_data()); + + let removed = store.remove_many(&[ns1, ns2]); + assert_eq!(removed.len(), 2); + assert!(!store.contains(&ns1)); + assert!(!store.contains(&ns2)); + assert!(store.contains(&ns3)); + } + + #[test] + fn test_get_or_init() { + let mut store = ShardLocalPartitions::new(); + let ns = IggyNamespace::new(1, 1, 0); + + assert!(!store.contains(&ns)); + + let _ = store.get_or_init(ns, create_test_partition_data); + assert!(store.contains(&ns)); + + // Second call should not reinitialize + let data = store.get_or_init(ns, || panic!("Should not be called")); + assert!(data.should_increment_offset); + } +} diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs index 2259e5b118..132ae8ad39 100644 --- a/core/server/src/shard/system/clients.rs +++ b/core/server/src/shard/system/clients.rs @@ -73,9 +73,7 @@ impl IggyShard { session: &Session, client_id: u32, ) -> Result, IggyError> { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .get_client(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -88,9 +86,7 @@ impl IggyShard { } pub fn get_clients(&self, session: &Session) -> Result, IggyError> { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .get_clients(session.get_user_id()) .error(|e: &IggyError| { format!( diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs index 1f9610a22e..2a7c766483 100644 --- a/core/server/src/shard/system/consumer_groups.rs +++ b/core/server/src/shard/system/consumer_groups.rs @@ -17,21 +17,14 @@ */ use super::COMPONENT; +use crate::metadata::ConsumerGroupMeta; use crate::shard::IggyShard; -use crate::slab::consumer_groups; -use crate::slab::traits_ext::EntityMarker; -use crate::slab::traits_ext::Insert; -use crate::streaming::partitions; use crate::streaming::session::Session; -use crate::streaming::streams; -use crate::streaming::topics; -use crate::streaming::topics::consumer_group; -use crate::streaming::topics::consumer_group::MEMBERS_CAPACITY; -use arcshift::ArcShift; use err_trail::ErrContext; use iggy_common::Identifier; use iggy_common::IggyError; -use slab::Slab; +use iggy_common::collections::SegmentedSlab; +use std::sync::Arc; impl IggyShard { pub fn create_consumer_group( @@ -40,72 +33,44 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, name: String, - ) -> Result { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - let exists = self - .streams - .with_topic_by_id(stream_id, topic_id, |(root, ..)| { - root.consumer_groups() - .exists(&name.clone().try_into().unwrap()) - }); - if exists { - return Err(IggyError::ConsumerGroupNameAlreadyExists( - name, - topic_id.clone(), - )); - } + ) -> Result { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.permissioner + .create_consumer_group(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}", + session.get_user_id(), + stream, + topic + ) + })?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self + let mut partitions: Vec = { + let metadata = self.metadata.load(); + metadata .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().create_consumer_group( - session.get_user_id(), - stream_id, - topic_id, - ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?; - } - let cg = self.create_and_insert_consumer_group_mem(stream_id, topic_id, name); - Ok(cg) - } + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| t.partitions.keys().collect()) + .unwrap_or_default() + }; + partitions.sort_unstable(); - fn create_and_insert_consumer_group_mem( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - name: String, - ) -> consumer_group::ConsumerGroup { - let partitions = self.streams.with_topics(stream_id, |topics| { - topics.with_partitions(topic_id, partitions::helpers::get_partition_ids()) - }); - let members = ArcShift::new(Slab::with_capacity(MEMBERS_CAPACITY)); - let mut cg = consumer_group::ConsumerGroup::new(name, members, partitions); - let id = self.insert_consumer_group_mem(stream_id, topic_id, cg.clone()); - cg.update_id(id); - cg - } + let id = self + .metadata + .try_register_consumer_group(stream, topic, Arc::from(name.as_str()), partitions) + .map_err(|e| { + if let IggyError::ConsumerGroupNameAlreadyExists(_, _) = &e { + // Re-wrap to use caller's topic_id for the error message + IggyError::ConsumerGroupNameAlreadyExists(name.clone(), topic_id.clone()) + } else { + e + } + })?; - fn insert_consumer_group_mem( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - cg: consumer_group::ConsumerGroup, - ) -> consumer_groups::ContainerId { - self.streams - .with_consumer_groups_mut(stream_id, topic_id, |container| container.insert(cg)) - } - - pub fn create_consumer_group_bypass_auth( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - cg: consumer_group::ConsumerGroup, - ) -> usize { - self.insert_consumer_group_mem(stream_id, topic_id, cg) + Ok(id) } pub fn delete_consumer_group( @@ -114,23 +79,22 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, group_id: &Identifier, - ) -> Result { - self.ensure_authenticated(session)?; - self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().delete_consumer_group( - session.get_user_id(), - stream_id, - topic_id, - ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to delete consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?; - } - let cg = self.delete_consumer_group_base(stream_id, topic_id, group_id); + ) -> Result { + let (stream, topic, group) = + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; + + self.permissioner + .delete_consumer_group(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to delete consumer group for user {} on stream ID: {}, topic ID: {}", + session.get_user_id(), + stream, + topic + ) + })?; + + let cg = self.delete_consumer_group_base(stream, topic, group); Ok(cg) } @@ -139,41 +103,40 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, group_id: &Identifier, - ) -> consumer_group::ConsumerGroup { - self.delete_consumer_group_base(stream_id, topic_id, group_id) + ) -> Result { + let (stream, topic, group) = + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; + Ok(self.delete_consumer_group_base(stream, topic, group)) } fn delete_consumer_group_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, - group_id: &Identifier, - ) -> consumer_group::ConsumerGroup { - // Get numeric IDs before deletion for ClientManager cleanup - let stream_id_value = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let topic_id_value = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let group_id_value = self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - group_id, - topics::helpers::get_consumer_group_id(), - ); - - let cg = self.streams.with_consumer_groups_mut( - stream_id, - topic_id, - topics::helpers::delete_consumer_group(group_id), - ); - - // Clean up ClientManager state + stream: usize, + topic: usize, + group: usize, + ) -> ConsumerGroupMeta { + let cg_meta = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.consumer_groups.get(group)) + .cloned() + .unwrap_or_else(|| ConsumerGroupMeta { + id: group, + name: Arc::from(""), + partitions: Vec::new(), + members: SegmentedSlab::new(), + }) + }; + self.client_manager - .delete_consumer_group(stream_id_value, topic_id_value, group_id_value); + .delete_consumer_group(stream, topic, group); - cg + self.metadata.delete_consumer_group(stream, topic, group); + + cg_meta } pub fn join_consumer_group( @@ -183,55 +146,34 @@ impl IggyShard { topic_id: &Identifier, group_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().join_consumer_group( - session.get_user_id(), - stream_id, - topic_id, - ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to join consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?; - } + let (stream, topic, group) = + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; + + self.permissioner + .join_consumer_group(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to join consumer group for user {} on stream ID: {}, topic ID: {}", + session.get_user_id(), + stream, + topic + ) + })?; + let client_id = session.client_id; - self.streams.with_consumer_group_by_id_mut( - stream_id, - topic_id, - group_id, - topics::helpers::join_consumer_group(client_id), - ); - - // Update ClientManager state - let stream_id_value = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let topic_id_value = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let group_id_value = self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - group_id, - topics::helpers::get_consumer_group_id(), - ); - - self.client_manager.join_consumer_group( - session.client_id, - stream_id_value, - topic_id_value, - group_id_value, - ) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}", - session.client_id - ) - })?; + + self.metadata + .join_consumer_group(stream, topic, group, client_id); + + self.client_manager + .join_consumer_group(client_id, stream, topic, group) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}", + client_id + ) + })?; + Ok(()) } @@ -242,21 +184,20 @@ impl IggyShard { topic_id: &Identifier, group_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().leave_consumer_group( - session.get_user_id(), - stream_id, - topic_id, - ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to leave consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?; - } + let (stream, topic, _group) = + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; + + self.permissioner + .leave_consumer_group(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to leave consumer group for user {} on stream ID: {}, topic ID: {}", + session.get_user_id(), + stream_id, + topic_id + ) + })?; + self.leave_consumer_group_base(stream_id, topic_id, group_id, session.client_id) } @@ -267,46 +208,68 @@ impl IggyShard { group_id: &Identifier, client_id: u32, ) -> Result<(), IggyError> { - let Some(_) = self.streams.with_consumer_group_by_id_mut( - stream_id, - topic_id, - group_id, - topics::helpers::leave_consumer_group(client_id), - ) else { + let (stream, topic, group) = + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; + + let member_id = self + .metadata + .leave_consumer_group(stream, topic, group, client_id); + + if member_id.is_none() { return Err(IggyError::ConsumerGroupMemberNotFound( client_id, group_id.clone(), topic_id.clone(), )); - }; + } - self.streams.with_consumer_group_by_id_mut( - stream_id, - topic_id, - group_id, - topics::helpers::rebalance_consumer_group(), - ); + self.client_manager + .leave_consumer_group(client_id, stream, topic, group) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}", + client_id + ) + })?; - // Update ClientManager state - let stream_id_value = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let topic_id_value = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let group_id_value = self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - group_id, - topics::helpers::get_consumer_group_id(), - ); - - self.client_manager.leave_consumer_group( - client_id, - stream_id_value, - topic_id_value, - group_id_value, - ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}", client_id))?; Ok(()) } + + pub fn get_consumer_group_from_shared_metadata( + &self, + stream_id: usize, + topic_id: usize, + group_id: usize, + ) -> bytes::Bytes { + use bytes::{BufMut, BytesMut}; + + let metadata = self.metadata.load(); + + let Some(cg_meta) = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + else { + return bytes::Bytes::new(); + }; + + let mut bytes = BytesMut::new(); + + bytes.put_u32_le(cg_meta.id as u32); + bytes.put_u32_le(cg_meta.partitions.len() as u32); + bytes.put_u32_le(cg_meta.members.len() as u32); + bytes.put_u8(cg_meta.name.len() as u8); + bytes.put_slice(cg_meta.name.as_bytes()); + + for (_, member) in cg_meta.members.iter() { + bytes.put_u32_le(member.id as u32); + bytes.put_u32_le(member.partitions.len() as u32); + for &partition_id in &member.partitions { + bytes.put_u32_le(partition_id as u32); + } + } + + bytes.freeze() + } } diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs index 1777675a12..619202f376 100644 --- a/core/server/src/shard/system/consumer_offsets.rs +++ b/core/server/src/shard/system/consumer_offsets.rs @@ -1,34 +1,33 @@ /* Licensed to the Apache Software Foundation (ASF) under one - polling_consumer: &PollingConsumer, -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ use super::COMPONENT; use crate::{ shard::IggyShard, streaming::{ - partitions, + partitions::consumer_offset::ConsumerOffset, polling_consumer::{ConsumerGroupId, PollingConsumer}, session::Session, - streams, topics, }, }; use err_trail::ErrContext; -use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError}; +use iggy_common::{Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError}; +use std::sync::atomic::Ordering; impl IggyShard { pub async fn store_consumer_offset( @@ -40,26 +39,16 @@ impl IggyShard { partition_id: Option, offset: u64, ) -> Result<(PollingConsumer, usize), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().store_consumer_offset( - session.get_user_id(), - stream_id, - topic_id - ).error(|e: &IggyError| { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.permissioner + .store_consumer_offset(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { format!( - "{COMPONENT} (error: {e}) - permission denied to store consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}", + "{COMPONENT} (error: {e}) - permission denied to store consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic} and stream with ID: {stream}", session.get_user_id(), ) })?; - } let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( stream_id, topic_id, @@ -73,14 +62,8 @@ impl IggyShard { }; self.ensure_partition_exists(stream_id, topic_id, partition_id)?; - self.store_consumer_offset_base( - stream_id, - topic_id, - &polling_consumer, - partition_id, - offset, - ); - self.persist_consumer_offset_to_disk(stream_id, topic_id, &polling_consumer, partition_id) + self.store_consumer_offset_base(stream, topic, &polling_consumer, partition_id, offset); + self.persist_consumer_offset_to_disk(stream, topic, &polling_consumer, partition_id) .await?; Ok((polling_consumer, partition_id)) } @@ -93,26 +76,16 @@ impl IggyShard { topic_id: &Identifier, partition_id: Option, ) -> Result, IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().get_consumer_offset( - session.get_user_id(), - stream_id, - topic_id - ).error(|e: &IggyError| { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.permissioner + .get_consumer_offset(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { format!( - "{COMPONENT} (error: {e}) - permission denied to get consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}", + "{COMPONENT} (error: {e}) - permission denied to get consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic} and stream with ID: {stream}", session.get_user_id() ) })?; - } let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( stream_id, topic_id, @@ -126,20 +99,46 @@ impl IggyShard { }; self.ensure_partition_exists(stream_id, topic_id, partition_id)?; + // Get the partition's current offset from stats (messages_count - 1, or 0 if empty) + use iggy_common::sharding::IggyNamespace; + let ns = IggyNamespace::new(stream, topic, partition_id); + let partition_current_offset = self + .metadata + .get_partition_stats(&ns) + .map(|s| { + let count = s.messages_count_inconsistent(); + if count > 0 { count - 1 } else { 0 } + }) + .unwrap_or(0); + let offset = match polling_consumer { - PollingConsumer::Consumer(id, _) => self.streams.with_partition_by_id( - stream_id, - topic_id, - partition_id, - partitions::helpers::get_consumer_offset(id), - ), + PollingConsumer::Consumer(id, _) => { + let offsets = + self.metadata + .get_partition_consumer_offsets(stream, topic, partition_id); + offsets.and_then(|co| { + let guard = co.pin(); + guard.get(&id).map(|item| ConsumerOffsetInfo { + partition_id: partition_id as u32, + current_offset: partition_current_offset, + stored_offset: item.offset.load(Ordering::Relaxed), + }) + }) + } PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - self.streams.with_partition_by_id( - stream_id, - topic_id, - partition_id, - partitions::helpers::get_consumer_group_offset(consumer_group_id), - ) + let offsets = + self.metadata + .get_partition_consumer_group_offsets(stream, topic, partition_id); + offsets.and_then(|co| { + let guard = co.pin(); + guard + .get(&consumer_group_id) + .map(|item| ConsumerOffsetInfo { + partition_id: partition_id as u32, + current_offset: partition_current_offset, + stored_offset: item.offset.load(Ordering::Relaxed), + }) + }) } }; Ok(offset) @@ -153,26 +152,16 @@ impl IggyShard { topic_id: &Identifier, partition_id: Option, ) -> Result<(PollingConsumer, usize), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().delete_consumer_offset( - session.get_user_id(), - stream_id, - topic_id - ).error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - permission denied to delete consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}", - session.get_user_id(), - ) - })?; - } + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.permissioner + .delete_consumer_offset(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to delete consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic} and stream with ID: {stream}", + session.get_user_id(), + ) + })?; let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( stream_id, topic_id, @@ -187,7 +176,7 @@ impl IggyShard { self.ensure_partition_exists(stream_id, topic_id, partition_id)?; let path = - self.delete_consumer_offset_base(stream_id, topic_id, &polling_consumer, partition_id)?; + self.delete_consumer_offset_base(stream, topic, &polling_consumer, partition_id)?; self.delete_consumer_offset_from_disk(&path).await?; Ok((polling_consumer, partition_id)) } @@ -199,44 +188,33 @@ impl IggyShard { topic_id: &Identifier, partition_ids: &[usize], ) -> Result<(), IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + for &partition_id in partition_ids { - // Skip if partition was deleted - let partition_exists = self - .streams - .with_partitions(stream_id, topic_id, |p| p.exists(partition_id)); - if !partition_exists { - continue; - } + let offsets = + self.metadata + .get_partition_consumer_group_offsets(stream, topic, partition_id); - // Skip if offset does not exist - let has_offset = self - .streams - .with_partition_by_id( - stream_id, - topic_id, - partition_id, - partitions::helpers::get_consumer_group_offset(cg_id), - ) - .is_some(); - if !has_offset { + let Some(offsets) = offsets else { continue; - } + }; - let path = self.streams - .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_group_offset(cg_id)) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete consumer group offset for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}", - cg_id, partition_id, topic_id, stream_id - ) - })?; + let path = { + let guard = offsets.pin(); + guard.get(&cg_id).map(|item| item.path.clone()) + }; - self.delete_consumer_offset_from_disk(&path).await.error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}", - cg_id, partition_id, topic_id, stream_id - ) - })?; + if let Some(path) = path { + offsets.pin().remove(&cg_id); + self.delete_consumer_offset_from_disk(&path) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}", + cg_id, partition_id, topic_id, stream_id + ) + })?; + } } Ok(()) @@ -244,128 +222,143 @@ impl IggyShard { fn store_consumer_offset_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, polling_consumer: &PollingConsumer, partition_id: usize, offset: u64, ) { - let stream_id_num = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let topic_id_num = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - match polling_consumer { PollingConsumer::Consumer(id, _) => { - self.streams.with_partition_by_id( - stream_id, - topic_id, - partition_id, - partitions::helpers::store_consumer_offset( - *id, - stream_id_num, - topic_id_num, - partition_id, - offset, - &self.config.system, - ), - ); + let offsets = + self.metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id); + + if let Some(offsets) = offsets { + let guard = offsets.pin(); + if let Some(existing) = guard.get(id) { + existing.offset.store(offset, Ordering::Relaxed); + } else { + let dir_path = self.config.system.get_consumer_offsets_path( + stream_id, + topic_id, + partition_id, + ); + let path = format!("{}/{}", dir_path, id); + let consumer_offset = + ConsumerOffset::new(ConsumerKind::Consumer, *id as u32, offset, path); + drop(guard); + offsets.pin().insert(*id, consumer_offset); + } + } } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - self.streams.with_partition_by_id( + PollingConsumer::ConsumerGroup(cg_id, _) => { + let offsets = self.metadata.get_partition_consumer_group_offsets( stream_id, topic_id, partition_id, - partitions::helpers::store_consumer_group_offset( - *consumer_group_id, - stream_id_num, - topic_id_num, - partition_id, - offset, - &self.config.system, - ), ); + + if let Some(offsets) = offsets { + let guard = offsets.pin(); + if let Some(existing) = guard.get(cg_id) { + existing.offset.store(offset, Ordering::Relaxed); + } else { + let dir_path = self.config.system.get_consumer_group_offsets_path( + stream_id, + topic_id, + partition_id, + ); + let path = format!("{}/{}", dir_path, cg_id.0); + let consumer_offset = ConsumerOffset::new( + ConsumerKind::ConsumerGroup, + cg_id.0 as u32, + offset, + path, + ); + drop(guard); + offsets.pin().insert(*cg_id, consumer_offset); + } + } } } } fn delete_consumer_offset_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, polling_consumer: &PollingConsumer, partition_id: usize, ) -> Result { match polling_consumer { PollingConsumer::Consumer(id, _) => { - self.streams - .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_offset(*id)).error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete consumer offset for consumer with ID: {id} in topic with ID: {topic_id} and stream with ID: {stream_id}", - ) - }) + let offsets = self + .metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + + let guard = offsets.pin(); + let offset = guard + .remove(id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + Ok(offset.path.clone()) } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - self.streams - .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_group_offset(*consumer_group_id)).error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete consumer group offset for group with ID: {consumer_group_id:?} in topic with ID: {topic_id} and stream with ID: {stream_id}", - ) - }) + PollingConsumer::ConsumerGroup(cg_id, _) => { + let offsets = self + .metadata + .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + + let guard = offsets.pin(); + let offset = guard + .remove(cg_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + Ok(offset.path.clone()) } } } async fn persist_consumer_offset_to_disk( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, polling_consumer: &PollingConsumer, partition_id: usize, ) -> Result<(), IggyError> { - match polling_consumer { + use crate::streaming::partitions::storage::persist_offset; + + let (offset_value, path) = match polling_consumer { PollingConsumer::Consumer(id, _) => { - let (offset_value, path) = self.streams.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(id) - .expect("persist_consumer_offset_to_disk: offset not found"); - let offset = item.offset.load(std::sync::atomic::Ordering::Relaxed); - let path = item.path.clone(); - (offset, path) - }, - ); - partitions::storage::persist_offset(&path, offset_value).await + let offsets = self + .metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + + let guard = offsets.pin(); + let item = guard + .get(id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + (item.offset.load(Ordering::Relaxed), item.path.clone()) } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - let (offset_value, path) = self.streams.with_partition_by_id( - stream_id, - topic_id, - partition_id, - move |(.., offsets, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(consumer_group_id) - .expect("persist_consumer_offset_to_disk: offset not found"); - ( - item.offset.load(std::sync::atomic::Ordering::Relaxed), - item.path.clone(), - ) - }, - ); - partitions::storage::persist_offset(&path, offset_value).await + PollingConsumer::ConsumerGroup(cg_id, _) => { + let offsets = self + .metadata + .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + + let guard = offsets.pin(); + let item = guard + .get(cg_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + (item.offset.load(Ordering::Relaxed), item.path.clone()) } - } + }; + persist_offset(&path, offset_value).await } pub async fn delete_consumer_offset_from_disk(&self, path: &str) -> Result<(), IggyError> { - partitions::storage::delete_persisted_offset(path).await + crate::streaming::partitions::storage::delete_persisted_offset(path).await } pub fn store_consumer_offset_bypass_auth( @@ -375,13 +368,9 @@ impl IggyShard { polling_consumer: &PollingConsumer, partition_id: usize, offset: u64, - ) { - self.store_consumer_offset_base( - stream_id, - topic_id, - polling_consumer, - partition_id, - offset, - ); + ) -> Result<(), IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + self.store_consumer_offset_base(stream, topic, polling_consumer, partition_id, offset); + Ok(()) } } diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs index 02e8d9d0e4..b7e06c9ecb 100644 --- a/core/server/src/shard/system/messages.rs +++ b/core/server/src/shard/system/messages.rs @@ -19,20 +19,19 @@ use super::COMPONENT; use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata; use crate::shard::IggyShard; -use crate::shard::namespace::IggyFullNamespace; use crate::shard::transmission::frame::ShardResponse; use crate::shard::transmission::message::{ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult, }; +use crate::streaming::partitions::journal::Journal; +use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet}; -use crate::streaming::traits::MainOps; -use crate::streaming::{partitions, streams, topics}; use err_trail::ErrContext; use iggy_common::PooledBuffer; use iggy_common::sharding::IggyNamespace; use iggy_common::{ BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError, - PollingKind, PollingStrategy, + PollingStrategy, }; use std::sync::atomic::Ordering; use tracing::error; @@ -46,36 +45,20 @@ impl IggyShard { partition_id: usize, batch: IggyMessagesBatchMut, ) -> Result<(), IggyError> { - self.ensure_topic_exists(&stream_id, &topic_id)?; + let (stream, topic, _) = self.resolve_partition_id(&stream_id, &topic_id, partition_id)?; - let numeric_stream_id = self - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - - let numeric_topic_id = - self.streams - .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id()); - - // Validate permissions for given user on stream and topic. self.permissioner - .borrow() - .append_messages( - user_id, - numeric_stream_id, - numeric_topic_id, - ) + .append_messages(user_id, stream, topic) .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}", user_id, numeric_stream_id as u32, numeric_topic_id as u32) + format!("{COMPONENT} (error: {e}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}", user_id, stream as u32, topic as u32) })?; if batch.count() == 0 { return Ok(()); } - self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?; - // TODO(tungtose): DRY this code - let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + let namespace = IggyNamespace::new(stream, topic, partition_id); let payload = ShardRequestPayload::SendMessages { batch }; let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload); let message = ShardMessage::Request(request); @@ -85,20 +68,52 @@ impl IggyShard { { ShardSendRequestResult::Recoil(message) => { if let ShardMessage::Request(ShardRequest { - stream_id, - topic_id, + stream_id: _, + topic_id: _, partition_id, payload, }) = message && let ShardRequestPayload::SendMessages { batch } = payload { - let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id); - // Encrypt messages if encryptor is enabled in configuration. let batch = self.maybe_encrypt_messages(batch)?; let messages_count = batch.count(); - self.streams - .append_messages(&self.config.system, &self.task_registry, &ns, batch) + + let namespace = IggyNamespace::new(stream, topic, partition_id); + + let metadata = self.metadata.load(); + let partition_meta = metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.partitions.get(partition_id)); + + let needs_init = { + let store = self.partition_store.borrow(); + match (store.get(&namespace), partition_meta) { + (Some(data), Some(meta)) if data.revision_id == meta.revision_id => { + false + } + (Some(_), _) => { + drop(store); + self.partition_store.borrow_mut().remove(&namespace); + true + } + (None, _) => true, + } + }; + + if needs_init { + let created_at = partition_meta + .map(|m| m.created_at) + .unwrap_or_else(iggy_common::IggyTimestamp::now); + + self.init_partition_directly(stream, topic, partition_id, created_at) + .await?; + } + + self.append_messages_to_partition_store(&namespace, batch, &self.config.system) .await?; + self.metrics.increment_messages(messages_count as u64); Ok(()) } else { @@ -130,23 +145,15 @@ impl IggyShard { maybe_partition_id: Option, args: PollingArgs, ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - self.ensure_topic_exists(&stream_id, &topic_id)?; - - let numeric_stream_id = self - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id()); + let (stream, topic) = self.resolve_topic_id(&stream_id, &topic_id)?; self.permissioner - .borrow() - .poll_messages(user_id, numeric_stream_id, numeric_topic_id) + .poll_messages(user_id, stream, topic) .error(|e: &IggyError| format!( "{COMPONENT} (error: {e}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}", user_id, stream_id, - numeric_topic_id + topic ))?; // Resolve partition ID @@ -164,22 +171,22 @@ impl IggyShard { self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?; - let current_offset = self.streams.with_partition_by_id( - &stream_id, - &topic_id, - partition_id, - |(_, _, _, offset, ..)| offset.load(Ordering::Relaxed), - ); - if args.strategy.kind == PollingKind::Offset && args.strategy.value > current_offset - || args.count == 0 - { + let namespace = IggyNamespace::new(stream, topic, partition_id); + + if args.count == 0 { + let current_offset = self + .partition_store + .borrow() + .get(&namespace) + .map(|data| data.offset.load(Ordering::Relaxed)) + .unwrap_or(0); return Ok(( IggyPollMetadata::new(partition_id as u32, current_offset), IggyMessagesBatchSet::empty(), )); } - let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + // Offset validation is done by the owning shard after routing let payload = ShardRequestPayload::PollMessages { consumer, args }; let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload); let message = ShardMessage::Request(request); @@ -195,27 +202,50 @@ impl IggyShard { }) = message && let ShardRequestPayload::PollMessages { consumer, args } = payload { - let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id); + let metadata = self.metadata.load(); + let partition_meta = metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.partitions.get(partition_id)); + + let needs_init = { + let store = self.partition_store.borrow(); + match (store.get(&namespace), partition_meta) { + (Some(data), Some(meta)) if data.revision_id == meta.revision_id => { + false + } + (Some(_), _) => { + drop(store); + self.partition_store.borrow_mut().remove(&namespace); + true + } + (None, _) => true, + } + }; + + if needs_init { + let created_at = partition_meta + .map(|m| m.created_at) + .unwrap_or_else(iggy_common::IggyTimestamp::now); + self.init_partition_directly(stream, topic, partition_id, created_at) + .await?; + } + let auto_commit = args.auto_commit; - let (metadata, batches) = - self.streams.poll_messages(&ns, consumer, args).await?; - let stream_id = ns.stream_id(); - let topic_id = ns.topic_id(); + + let (metadata, batches) = self + .poll_messages_from_partition_store(&namespace, consumer, args) + .await?; if auto_commit && !batches.is_empty() { let offset = batches .last_offset() .expect("Batch set should have at least one batch"); - self.streams - .auto_commit_consumer_offset( - &self.config.system, - stream_id, - topic_id, - partition_id, - consumer, - offset, - ) - .await?; + self.auto_commit_consumer_offset_from_partition_store( + &namespace, consumer, offset, + ) + .await?; } Ok((metadata, batches)) } else { @@ -250,27 +280,15 @@ impl IggyShard { partition_id: usize, fsync: bool, ) -> Result<(), IggyError> { - self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?; - - let numeric_stream_id = self - .streams - .with_stream_by_id(&stream_id, streams::helpers::get_stream_id()); - - let numeric_topic_id = - self.streams - .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id()); + let (stream, topic, _) = self.resolve_partition_id(&stream_id, &topic_id, partition_id)?; - // Validate permissions for given user on stream and topic. self.permissioner - .borrow() - .append_messages(user_id, numeric_stream_id, numeric_topic_id) + .append_messages(user_id, stream, topic) .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, numeric_stream_id as u32, numeric_topic_id as u32) + format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, stream as u32, topic as u32) })?; - self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?; - - let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + let namespace = IggyNamespace::new(stream, topic, partition_id); let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync }; let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload); let message = ShardMessage::Request(request); @@ -280,14 +298,14 @@ impl IggyShard { { ShardSendRequestResult::Recoil(message) => { if let ShardMessage::Request(ShardRequest { - stream_id, - topic_id, partition_id, payload, + .. }) = message && let ShardRequestPayload::FlushUnsavedBuffer { fsync } = payload { - self.flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync) + let namespace = IggyNamespace::new(stream, topic, partition_id); + self.flush_unsaved_buffer_from_partition_store(&namespace, fsync) .await?; Ok(()) } else { @@ -310,38 +328,389 @@ impl IggyShard { pub(crate) async fn flush_unsaved_buffer_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream: usize, + topic: usize, partition_id: usize, fsync: bool, - ) -> Result<(), IggyError> { - let batches = self.streams.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - partitions::helpers::commit_journal(), - ); + ) -> Result { + let namespace = IggyNamespace::new(stream, topic, partition_id); + self.flush_unsaved_buffer_from_partition_store(&namespace, fsync) + .await + } - self.streams - .persist_messages_to_disk( - stream_id, - topic_id, - partition_id, - batches, - &self.config.system, - ) + /// Flushes unsaved messages from the partition store to disk. + /// Returns the number of messages saved. + pub(crate) async fn flush_unsaved_buffer_from_partition_store( + &self, + namespace: &IggyNamespace, + fsync: bool, + ) -> Result { + let batches = { + let mut store = self.partition_store.borrow_mut(); + let Some(partition_data) = store.get_mut(namespace) else { + return Ok(0); + }; + if !partition_data.log.has_segments() { + return Ok(0); + } + let batches = partition_data.log.journal_mut().commit(); + partition_data.log.ensure_indexes(); + batches.append_indexes_to(partition_data.log.active_indexes_mut().unwrap()); + batches + }; + + let saved_count = self + .persist_messages_to_disk_from_partition_store(namespace, batches) .await?; - // Ensure all data is flushed to disk before returning if fsync { - self.streams - .fsync_all_messages(stream_id, topic_id, partition_id) + self.fsync_all_messages_from_partition_store(namespace) .await?; } + Ok(saved_count) + } + + pub(crate) async fn fsync_all_messages_from_partition_store( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + let storage = { + let store = self.partition_store.borrow(); + let Some(partition_data) = store.get(namespace) else { + return Ok(()); + }; + if !partition_data.log.has_segments() { + return Ok(()); + } + partition_data.log.active_storage().clone() + }; + + if storage.messages_writer.is_none() || storage.index_writer.is_none() { + return Ok(()); + } + + if let Some(ref messages_writer) = storage.messages_writer + && let Err(e) = messages_writer.fsync().await + { + tracing::error!( + "Failed to fsync messages writer for partition {:?}: {}", + namespace, + e + ); + return Err(e); + } + + if let Some(ref index_writer) = storage.index_writer + && let Err(e) = index_writer.fsync().await + { + tracing::error!( + "Failed to fsync index writer for partition {:?}: {}", + namespace, + e + ); + return Err(e); + } + Ok(()) } + pub(crate) async fn auto_commit_consumer_offset_from_partition_store( + &self, + namespace: &IggyNamespace, + consumer: PollingConsumer, + offset: u64, + ) -> Result<(), IggyError> { + let (offset_value, path) = { + let store = self.partition_store.borrow(); + let partition_data = store.get(namespace).ok_or_else(|| { + IggyError::PartitionNotFound( + namespace.partition_id(), + Identifier::numeric(namespace.topic_id() as u32).unwrap(), + Identifier::numeric(namespace.stream_id() as u32).unwrap(), + ) + })?; + + match consumer { + PollingConsumer::Consumer(consumer_id, _) => { + tracing::trace!( + "Auto-committing offset {} for consumer {} on partition {:?}", + offset, + consumer_id, + namespace + ); + let hdl = partition_data.consumer_offsets.pin(); + let item = hdl.get_or_insert( + consumer_id, + crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer( + consumer_id as u32, + &self.config.system.get_consumer_offsets_path( + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ), + ), + ); + item.offset.store(offset, Ordering::Relaxed); + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + PollingConsumer::ConsumerGroup(consumer_group_id, _) => { + tracing::trace!( + "Auto-committing offset {} for consumer group {} on partition {:?}", + offset, + consumer_group_id.0, + namespace + ); + let hdl = partition_data.consumer_group_offsets.pin(); + let item = hdl.get_or_insert( + consumer_group_id, + crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer_group( + consumer_group_id, + &self.config.system.get_consumer_group_offsets_path( + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ), + ), + ); + item.offset.store(offset, Ordering::Relaxed); + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + } + }; + + crate::streaming::partitions::storage::persist_offset(&path, offset_value).await?; + Ok(()) + } + + pub async fn append_messages_to_partition_store( + &self, + namespace: &IggyNamespace, + mut batch: IggyMessagesBatchMut, + config: &crate::configs::system::SystemConfig, + ) -> Result<(), IggyError> { + let (current_offset, current_position, segment_start_offset, message_deduplicator) = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let current_offset = if partition_data.should_increment_offset { + partition_data.offset.load(Ordering::Relaxed) + 1 + } else { + 0 + }; + + let segment = partition_data.log.active_segment(); + let current_position = segment.current_position; + let segment_start_offset = segment.start_offset; + let message_deduplicator = partition_data.message_deduplicator.clone(); + + ( + current_offset, + current_position, + segment_start_offset, + message_deduplicator, + ) + }; + + batch + .prepare_for_persistence( + segment_start_offset, + current_offset, + current_position, + message_deduplicator.as_ref(), + ) + .await; + + let (journal_messages_count, journal_size) = { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(namespace) + .expect("partition_store: partition must exist"); + + let segment = partition_data.log.active_segment_mut(); + + if segment.end_offset == 0 { + segment.start_timestamp = batch.first_timestamp().unwrap(); + } + + let batch_messages_size = batch.size(); + let batch_messages_count = batch.count(); + + partition_data + .stats + .increment_size_bytes(batch_messages_size as u64); + partition_data + .stats + .increment_messages_count(batch_messages_count as u64); + + segment.end_timestamp = batch.last_timestamp().unwrap(); + segment.end_offset = batch.last_offset().unwrap(); + + let (journal_messages_count, journal_size) = + partition_data.log.journal_mut().append(batch)?; + + let last_offset = if batch_messages_count == 0 { + current_offset + } else { + current_offset + batch_messages_count as u64 - 1 + }; + + if partition_data.should_increment_offset { + partition_data.offset.store(last_offset, Ordering::Relaxed); + } else { + partition_data.should_increment_offset = true; + partition_data.offset.store(last_offset, Ordering::Relaxed); + } + partition_data.stats.set_current_offset(last_offset); + partition_data.log.active_segment_mut().current_position += batch_messages_size; + + (journal_messages_count, journal_size) + }; + + let unsaved_messages_count_exceeded = + journal_messages_count >= config.partition.messages_required_to_save; + let unsaved_messages_size_exceeded = journal_size + >= config + .partition + .size_of_messages_required_to_save + .as_bytes_u64() as u32; + + let is_full = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + partition_data.log.active_segment().is_full() + }; + + if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded { + let batches = { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(namespace) + .expect("partition_store: partition must exist"); + let batches = partition_data.log.journal_mut().commit(); + partition_data.log.ensure_indexes(); + batches.append_indexes_to(partition_data.log.active_indexes_mut().unwrap()); + batches + }; + + self.persist_messages_to_disk_from_partition_store(namespace, batches) + .await?; + + if is_full { + self.rotate_segment_in_partition_store(namespace).await?; + } + } + + Ok(()) + } + + async fn persist_messages_to_disk_from_partition_store( + &self, + namespace: &IggyNamespace, + batches: IggyMessagesBatchSet, + ) -> Result { + let batch_count = batches.count(); + + if batch_count == 0 { + return Ok(0); + } + + // Track segment_index to handle concurrent segment rotations. + // Another task may rotate to a new segment during our await points, + // so we must use the original segment index throughout. + let (messages_writer, index_writer, segment_index) = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + if !partition_data.log.has_segments() { + return Ok(0); + } + + let segment_index = partition_data.log.segments().len() - 1; + let messages_writer = partition_data + .log + .active_storage() + .messages_writer + .as_ref() + .expect("Messages writer not initialized") + .clone(); + let index_writer = partition_data + .log + .active_storage() + .index_writer + .as_ref() + .expect("Index writer not initialized") + .clone(); + (messages_writer, index_writer, segment_index) + }; + + let guard = messages_writer.lock.lock().await; + let saved = messages_writer.as_ref().save_batch_set(batches).await?; + + let unsaved_indexes_slice = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + partition_data.log.indexes()[segment_index] + .as_ref() + .expect("indexes must exist for segment being persisted") + .unsaved_slice() + }; + + index_writer + .as_ref() + .save_indexes(unsaved_indexes_slice) + .await?; + + tracing::trace!( + "Persisted {} messages on disk for partition: {:?}, total bytes written: {}.", + batch_count, + namespace, + saved + ); + + { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(namespace) + .expect("partition_store: partition must exist"); + + let indexes = partition_data.log.indexes_mut()[segment_index] + .as_mut() + .expect("indexes must exist for segment being persisted"); + indexes.mark_saved(); + + let segment = &mut partition_data.log.segments_mut()[segment_index]; + segment.size = + iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64()); + } + + drop(guard); + Ok(batch_count) + } + + pub async fn poll_messages_from_partition_store( + &self, + namespace: &IggyNamespace, + consumer: crate::streaming::polling_consumer::PollingConsumer, + args: PollingArgs, + ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + crate::streaming::partition_ops::poll_messages( + &self.partition_store, + namespace, + consumer, + args, + ) + .await + } + async fn decrypt_messages( &self, batches: IggyMessagesBatchSet, diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs index aacbd52c0d..9bd74ee0b9 100644 --- a/core/server/src/shard/system/partitions.rs +++ b/core/server/src/shard/system/partitions.rs @@ -17,23 +17,25 @@ */ use super::COMPONENT; +use crate::metadata::PartitionMeta; use crate::shard::IggyShard; use crate::shard::calculate_shard_assignment; -use crate::slab::traits_ext::EntityMarker; -use crate::slab::traits_ext::IntoComponents; -use crate::streaming::partitions; -use crate::streaming::partitions::partition; +use crate::shard::namespace::IggyNamespace; +use crate::shard::shard_local_partitions::PartitionData; +use crate::shard::transmission::event::PartitionInfo; +use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets}; use crate::streaming::partitions::storage::create_partition_file_hierarchy; use crate::streaming::partitions::storage::delete_partitions_from_disk; use crate::streaming::segments::Segment; use crate::streaming::segments::storage::create_segment_storage; use crate::streaming::session::Session; -use crate::streaming::streams; -use crate::streaming::topics; +use crate::streaming::stats::PartitionStats; use err_trail::ErrContext; use iggy_common::Identifier; use iggy_common::IggyError; -use iggy_common::sharding::{IggyNamespace, LocalIdx, PartitionLocation, ShardId}; +use iggy_common::IggyTimestamp; +use iggy_common::sharding::{LocalIdx, PartitionLocation, ShardId}; +use std::sync::Arc; use tracing::info; impl IggyShard { @@ -44,10 +46,15 @@ impl IggyShard { topic_id: usize, operation: &str, ) -> Result<(), IggyError> { - let permissioner = self.permissioner.borrow(); let result = match operation { - "create" => permissioner.create_partitions(session.get_user_id(), stream_id, topic_id), - "delete" => permissioner.delete_partitions(session.get_user_id(), stream_id, topic_id), + "create" => { + self.permissioner + .create_partitions(session.get_user_id(), stream_id, topic_id) + } + "delete" => { + self.permissioner + .delete_partitions(session.get_user_id(), stream_id, topic_id) + } _ => return Err(IggyError::InvalidCommand), }; @@ -67,41 +74,43 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, partitions_count: u32, - ) -> Result, IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - let numeric_stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - - // Claude garbage, rework this. - self.validate_partition_permissions( - session, - numeric_stream_id, - numeric_topic_id, - "create", - )?; - let parent_stats = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_stats()); - let partitions = partition::create_and_insert_partitions_mem( - &self.streams, - stream_id, - topic_id, - parent_stats, - partitions_count, - &self.config.system, - ); + ) -> Result, IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.validate_partition_permissions(session, stream, topic, "create")?; + + let created_at = IggyTimestamp::now(); + let shards_count = self.get_available_shards_count(); + + let parent_stats = self + .metadata + .get_topic_stats(stream, topic) + .expect("Parent topic stats must exist"); + + let metas: Vec = (0..partitions_count) + .map(|_| PartitionMeta { + id: 0, + created_at, + revision_id: 0, + stats: Arc::new(PartitionStats::new(parent_stats.clone())), + consumer_offsets: None, + consumer_group_offsets: None, + }) + .collect(); + + let assigned_ids = self.metadata.add_partitions(stream, topic, metas); + + let partition_infos: Vec = assigned_ids + .iter() + .map(|&id| PartitionInfo { id, created_at }) + .collect(); self.metrics.increment_partitions(partitions_count); self.metrics.increment_segments(partitions_count); - let shards_count = self.get_available_shards_count(); - for (partition_id, stats) in partitions.iter().map(|p| (p.id(), p.stats())) { - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + for info in &partition_infos { + let partition_id = info.id; + let ns = IggyNamespace::new(stream, topic, partition_id); let shard_id = ShardId::new(calculate_shard_assignment(&ns, shards_count)); let is_current_shard = self.id == *shard_id; // TODO(hubcio): LocalIdx(0) is wrong.. When IggyPartitions is integrated into @@ -109,19 +118,15 @@ impl IggyShard { let location = PartitionLocation::new(shard_id, LocalIdx::new(0)); self.insert_shard_table_record(ns, location); - create_partition_file_hierarchy( - numeric_stream_id as usize, - numeric_topic_id as usize, - partition_id, - &self.config.system, - ) - .await?; - stats.increment_segments_count(1); + create_partition_file_hierarchy(stream, topic, partition_id, &self.config.system) + .await?; + if is_current_shard { - self.init_log(stream_id, topic_id, partition_id).await?; + self.init_partition_directly(stream, topic, partition_id, created_at) + .await?; } } - Ok(partitions) + Ok(partition_infos) } pub async fn init_log( @@ -130,94 +135,183 @@ impl IggyShard { topic_id: &Identifier, partition_id: usize, ) -> Result<(), IggyError> { - let numeric_stream_id = self + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + let created_at = self + .metadata + .load() .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|meta| meta.created_at) + .unwrap_or_else(IggyTimestamp::now); + + self.init_partition_directly(stream, topic, partition_id, created_at) + .await + } + + /// Thread-safety: Uses `pending_partition_inits` to prevent TOCTOU races where + /// multiple tasks could see needs_init=true and all try to initialize. Only the + /// first task to mark the partition as pending will proceed; others return early. + pub async fn init_partition_directly( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + created_at: IggyTimestamp, + ) -> Result<(), IggyError> { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + // Check if partition already exists in partition_store + // If it does, verify revision_id matches SharedMetadata + // If not, the old entry is stale and needs to be removed + let needs_init = { + let store = self.partition_store.borrow(); + let metadata = self.metadata.load(); + let partition_meta = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)); + + match (store.get(&ns), partition_meta) { + (Some(data), Some(meta)) if data.revision_id == meta.revision_id => false, + (Some(_), _) => { + // Stale entry with different revision_id + drop(store); + self.partition_store.borrow_mut().remove(&ns); + true + } + (None, _) => true, + } + }; + + if !needs_init { + return Ok(()); + } + + { + let mut pending = self.pending_partition_inits.borrow_mut(); + if pending.contains(&ns) { + return Ok(()); + } + pending.insert(ns); + } + + let result = self + .init_partition_directly_inner(stream_id, topic_id, partition_id, created_at, ns) + .await; - let start_offset = 0; + self.pending_partition_inits.borrow_mut().remove(&ns); + + result + } + + async fn init_partition_directly_inner( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + created_at: IggyTimestamp, + ns: IggyNamespace, + ) -> Result<(), IggyError> { info!( - "Initializing log for partition ID: {} for topic ID: {} for stream ID: {} with start offset: {}", - partition_id, numeric_topic_id, numeric_stream_id, start_offset + "Initializing partition in partition_store: partition ID: {} for topic ID: {} for stream ID: {}", + partition_id, topic_id, stream_id ); - let segment = Segment::new( - start_offset, - self.config.system.segment.size, - self.config.system.segment.message_expiry, - ); + let stats = self + .metadata + .get_partition_stats_by_ids(stream_id, topic_id, partition_id) + .expect("Partition stats must exist in SharedMetadata"); - let numeric_stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - - let messages_size = 0; - let indexes_size = 0; - let storage = create_segment_storage( + let partition_path = + self.config + .system + .get_partition_path(stream_id, topic_id, partition_id); + + let mut loaded_log = crate::bootstrap::load_segments( &self.config.system, - numeric_stream_id, - numeric_topic_id, + stream_id, + topic_id, partition_id, - messages_size, - indexes_size, - start_offset, + partition_path, + stats.clone(), ) .await?; - self.streams - .with_partition_by_id_mut(stream_id, topic_id, partition_id, |(.., log)| { - log.add_persisted_segment(segment, storage); - }); - info!( - "Initialized log for partition ID: {} for topic ID: {} for stream ID: {} with start offset: {}", - partition_id, numeric_topic_id, numeric_stream_id, start_offset - ); + // If no segments exist on disk (newly created partition), create an initial segment + if !loaded_log.has_segments() { + info!( + "No segments found on disk for partition ID: {} for topic ID: {} for stream ID: {}, creating initial segment", + partition_id, topic_id, stream_id + ); - Ok(()) - } + let start_offset = 0; + let segment = Segment::new( + start_offset, + self.config.system.segment.size, + self.config.system.segment.message_expiry, + ); - pub async fn create_partitions_bypass_auth( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partitions: Vec, - ) -> Result<(), IggyError> { - let numeric_stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let shards_count = self.get_available_shards_count(); - for partition in partitions { - let actual_id = partition.id(); - let id = self.streams.with_partitions_mut( + let storage = create_segment_storage( + &self.config.system, stream_id, topic_id, - partitions::helpers::insert_partition(partition), - ); - assert_eq!( - id, actual_id, - "create_partitions_bypass_auth: partition mismatch ID, wrong creation order ?!" - ); - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, id); - // TODO(hubcio): when IggyPartitions is integrated, this fallback path should - // either be removed or use proper index resolution. - let location = self.find_shard_table_record(&ns).unwrap_or_else(|| { - tracing::warn!("WARNING: missing shard table record for namespace: {:?}, in the event handler for `CreatedPartitions` event.", ns); - let shard_id = ShardId::new(calculate_shard_assignment(&ns, shards_count)); - PartitionLocation::new(shard_id, LocalIdx::new(0)) - }); - if self.id == *location.shard_id { - self.init_log(stream_id, topic_id, id).await?; - } + partition_id, + 0, // messages_size + 0, // indexes_size + start_offset, + ) + .await?; + + loaded_log.add_persisted_segment(segment, storage); + stats.increment_segments_count(1); } + let current_offset = loaded_log.active_segment().end_offset; + + let revision_id = self + .metadata + .load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|meta| meta.revision_id) + .unwrap_or(0); + + let consumer_offsets = Arc::new(ConsumerOffsets::with_capacity(0)); + let consumer_group_offsets = Arc::new(ConsumerGroupOffsets::with_capacity(0)); + + self.metadata.set_partition_offsets( + stream_id, + topic_id, + partition_id, + consumer_offsets.clone(), + consumer_group_offsets.clone(), + ); + + let partition_data = PartitionData::with_log( + loaded_log, + stats, + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(current_offset)), + consumer_offsets, + consumer_group_offsets, + None, // message_deduplicator + created_at, + revision_id, + current_offset > 0, // should_increment_offset - true if we have messages + ); + + self.partition_store.borrow_mut().insert(ns, partition_data); + + info!( + "Initialized partition in partition_store: partition ID: {} for topic ID: {} for stream ID: {} with offset: {}", + partition_id, topic_id, stream_id, current_offset + ); + Ok(()) } @@ -228,58 +322,77 @@ impl IggyShard { topic_id: &Identifier, partitions_count: u32, ) -> Result, IggyError> { - self.ensure_authenticated(session)?; self.ensure_partitions_exist(stream_id, topic_id, partitions_count)?; - let numeric_stream_id = self + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.validate_partition_permissions(session, stream, topic, "delete")?; + + let all_partition_ids: Vec = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| { + let mut ids: Vec<_> = t.partitions.keys().collect(); + ids.sort_unstable(); + ids + }) + .unwrap_or_default() + }; + + let partitions_to_delete: Vec = all_partition_ids + .into_iter() + .rev() + .take(partitions_count as usize) + .collect(); + + let topic_stats = self + .metadata + .load() .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - - self.validate_partition_permissions( - session, - numeric_stream_id, - numeric_topic_id, - "delete", - )?; - - let partitions = self.delete_partitions_base(stream_id, topic_id, partitions_count); - let parent = partitions - .first() - .map(|p| p.stats().parent().clone()) - .expect("delete_partitions: no partitions to deletion"); - // Reassign the partitions count as it could get clamped by the `delete_partitions_base` method. - - let mut deleted_ids = Vec::with_capacity(partitions.len()); - let mut total_messages_count = 0; - let mut total_segments_count = 0; - let mut total_size_bytes = 0; - - for partition in partitions { - let (root, stats, _, _, _, _, _) = partition.into_components(); - let partition_id = root.id(); - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); - self.remove_shard_table_record(&ns); + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| t.stats.clone()); + + let mut deleted_ids = Vec::with_capacity(partitions_to_delete.len()); + let mut total_messages_count: u64 = 0; + let mut total_segments_count: u32 = 0; + let mut total_size_bytes: u64 = 0; + + for partition_id in &partitions_to_delete { + let ns = IggyNamespace::new(stream, topic, *partition_id); + + if let Some(stats) = + self.metadata + .get_partition_stats_by_ids(stream, topic, *partition_id) + { + total_segments_count += stats.segments_count_inconsistent(); + total_messages_count += stats.messages_count_inconsistent(); + total_size_bytes += stats.size_bytes_inconsistent(); + } - self.delete_partition_dir(numeric_stream_id, numeric_topic_id, partition_id) + self.remove_shard_table_record(&ns); + self.partition_store.borrow_mut().remove(&ns); + self.delete_partition_dir(stream, topic, *partition_id) .await?; - let segments_count = stats.segments_count_inconsistent(); - let messages_count = stats.messages_count_inconsistent(); - let size_bytes = stats.size_bytes_inconsistent(); - total_messages_count += messages_count; - total_segments_count += segments_count; - total_size_bytes += size_bytes; - - deleted_ids.push(partition_id); + + deleted_ids.push(*partition_id); } - self.metrics.decrement_partitions(partitions_count); + self.metadata + .delete_partitions(stream, topic, &partitions_to_delete); + + self.metrics + .decrement_partitions(partitions_to_delete.len() as u32); self.metrics.decrement_segments(total_segments_count); - parent.decrement_messages_count(total_messages_count); - parent.decrement_size_bytes(total_size_bytes); - parent.decrement_segments_count(total_segments_count); + + if let Some(parent) = topic_stats { + parent.decrement_messages_count(total_messages_count); + parent.decrement_size_bytes(total_size_bytes); + parent.decrement_segments_count(total_segments_count); + } Ok(deleted_ids) } @@ -295,15 +408,42 @@ impl IggyShard { fn delete_partitions_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream: usize, + topic: usize, partitions_count: u32, - ) -> Vec { - self.streams.with_partitions_mut( - stream_id, - topic_id, - partitions::helpers::delete_partitions(partitions_count), - ) + ) -> Vec { + let all_partition_ids: Vec = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| { + let mut ids: Vec<_> = t.partitions.keys().collect(); + ids.sort_unstable(); + ids + }) + .unwrap_or_default() + }; + + let partition_ids: Vec = all_partition_ids + .into_iter() + .rev() + .take(partitions_count as usize) + .collect(); + + { + let mut store = self.partition_store.borrow_mut(); + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic, partition_id); + store.remove(&ns); + } + } + + self.metadata + .delete_partitions(stream, topic, &partition_ids); + + partition_ids } pub fn delete_partitions_bypass_auth( @@ -314,19 +454,18 @@ impl IggyShard { partition_ids: Vec, ) -> Result<(), IggyError> { self.ensure_partitions_exist(stream_id, topic_id, partitions_count)?; + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; if partitions_count as usize != partition_ids.len() { return Err(IggyError::InvalidPartitionsCount); } - let partitions = self.delete_partitions_base(stream_id, topic_id, partitions_count); - for (deleted_partition_id, actual_deleted_partition_id) in partitions - .iter() - .map(|p| p.id()) - .zip(partition_ids.into_iter()) + let deleted_ids = self.delete_partitions_base(stream, topic, partitions_count); + for (deleted_partition_id, actual_deleted_partition_id) in + deleted_ids.iter().zip(partition_ids.iter()) { assert_eq!( - deleted_partition_id, actual_deleted_partition_id, + *deleted_partition_id, *actual_deleted_partition_id, "delete_partitions_bypass_auth: partition mismatch ID" ); } diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs index 7b26aff1ec..45bf1fdea3 100644 --- a/core/server/src/shard/system/personal_access_tokens.rs +++ b/core/server/src/shard/system/personal_access_tokens.rs @@ -32,17 +32,15 @@ impl IggyShard { &self, session: &Session, ) -> Result, IggyError> { - self.ensure_authenticated(session)?; let user_id = session.get_user_id(); - let user = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + + let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") })?; + info!("Loading personal access tokens for user with ID: {user_id}...",); - let personal_access_tokens: Vec<_> = user - .personal_access_tokens - .iter() - .map(|pat| pat.clone()) - .collect(); + + let personal_access_tokens = self.metadata.get_user_personal_access_tokens(user_id); info!( "Loaded {} personal access tokens for user with ID: {user_id}.", @@ -57,23 +55,22 @@ impl IggyShard { name: &str, expiry: IggyExpiry, ) -> Result<(PersonalAccessToken, String), IggyError> { - self.ensure_authenticated(session)?; let user_id = session.get_user_id(); - let identifier = user_id.try_into()?; - { - let user = self.get_user(&identifier).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - })?; - let max_token_per_user = self.config.personal_access_token.max_tokens_per_user; - if user.personal_access_tokens.len() as u32 >= max_token_per_user { - error!( - "User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.", - ); - return Err(IggyError::PersonalAccessTokensLimitReached( - user_id, - max_token_per_user, - )); - } + + let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + })?; + + let max_token_per_user = self.config.personal_access_token.max_tokens_per_user; + let current_count = self.metadata.user_pat_count(user_id); + if current_count as u32 >= max_token_per_user { + error!( + "User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.", + ); + return Err(IggyError::PersonalAccessTokensLimitReached( + user_id, + max_token_per_user, + )); } let (personal_access_token, token) = @@ -82,47 +79,24 @@ impl IggyShard { Ok((personal_access_token, token)) } - pub fn create_personal_access_token_bypass_auth( - &self, - personal_access_token: PersonalAccessToken, - ) -> Result<(), IggyError> { - self.create_personal_access_token_base(personal_access_token) - } - fn create_personal_access_token_base( &self, personal_access_token: PersonalAccessToken, ) -> Result<(), IggyError> { let user_id = personal_access_token.user_id; let name = personal_access_token.name.clone(); - let token_hash = personal_access_token.token.clone(); - let identifier = user_id.try_into()?; - self.users - .with_user_mut(&identifier, |user| { - if user - .personal_access_tokens - .iter() - .any(|pat| pat.name.as_str() == name.as_str()) - { - error!( - "Personal access token: {name} for user with ID: {user_id} already exists." - ); - return Err(IggyError::PersonalAccessTokenAlreadyExists( - name.to_string(), - user_id, - )); - } - - user.personal_access_tokens - .insert(token_hash, personal_access_token); - info!("Created personal access token: {name} for user with ID: {user_id}."); - Ok(()) - }) - .error(|e: &IggyError| { - format!( - "{COMPONENT} create PAT (error: {e}) - failed to access user with id: {user_id}" - ) - })??; + + if self.metadata.user_has_pat_with_name(user_id, &name) { + error!("Personal access token: {name} for user with ID: {user_id} already exists."); + return Err(IggyError::PersonalAccessTokenAlreadyExists( + name.to_string(), + user_id, + )); + } + + self.metadata + .add_personal_access_token(user_id, personal_access_token); + info!("Created personal access token: {name} for user with ID: {user_id}."); Ok(()) } @@ -131,44 +105,24 @@ impl IggyShard { session: &Session, name: &str, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; let user_id = session.get_user_id(); self.delete_personal_access_token_base(user_id, name) } - pub fn delete_personal_access_token_bypass_auth( - &self, - user_id: u32, - name: &str, - ) -> Result<(), IggyError> { - self.delete_personal_access_token_base(user_id, name) - } - fn delete_personal_access_token_base(&self, user_id: u32, name: &str) -> Result<(), IggyError> { - self.users - .with_user_mut(&user_id.try_into()?, |user| { - let token = if let Some(pat) = user - .personal_access_tokens - .iter() - .find(|pat| pat.name.as_str() == name) - { - pat.token.clone() - } else { + let token_hash = + self.metadata + .find_pat_token_hash_by_name(user_id, name) + .ok_or_else(|| { error!( "Personal access token: {name} for user with ID: {user_id} does not exist.", ); - return Err(IggyError::ResourceNotFound(name.to_owned())); - }; + IggyError::ResourceNotFound(name.to_owned()) + })?; - info!("Deleting personal access token: {name} for user with ID: {user_id}..."); - user.personal_access_tokens.remove(&token); - Ok(()) - }) - .error(|e: &IggyError| { - format!( - "{COMPONENT} delete PAT (error: {e}) - failed to access user with id: {user_id}" - ) - })??; + info!("Deleting personal access token: {name} for user with ID: {user_id}..."); + self.metadata + .delete_personal_access_token(user_id, &token_hash); info!("Deleted personal access token: {name} for user with ID: {user_id}."); Ok(()) } @@ -179,33 +133,27 @@ impl IggyShard { session: Option<&Session>, ) -> Result { let token_hash = PersonalAccessToken::hash_token(token); - let users = self.users.values(); - let mut personal_access_token = None; - for user in &users { - if let Some(pat) = user.personal_access_tokens.get(&token_hash) { - personal_access_token = Some(pat); - break; - } - } - if personal_access_token.is_none() { - let redacted_token = if token.len() > 4 { - format!("{}****", &token[..4]) - } else { - "****".to_string() - }; - error!("Personal access token: {redacted_token} does not exist."); - return Err(IggyError::ResourceNotFound(token.to_owned())); - } + let personal_access_token = self + .metadata + .get_personal_access_token_by_hash(&token_hash) + .ok_or_else(|| { + let redacted_token = if token.len() > 4 { + format!("{}****", &token[..4]) + } else { + "****".to_string() + }; + error!("Personal access token: {redacted_token} does not exist."); + IggyError::ResourceNotFound(token.to_owned()) + })?; - let personal_access_token = personal_access_token.unwrap(); if personal_access_token.is_expired(IggyTimestamp::now()) { error!( "Personal access token: {} for user with ID: {} has expired.", personal_access_token.name, personal_access_token.user_id ); return Err(IggyError::PersonalAccessTokenExpired( - personal_access_token.name.as_str().to_owned(), + (*personal_access_token.name).to_owned(), personal_access_token.user_id, )); } diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs index 5dfbbc4e12..2f46b7a85a 100644 --- a/core/server/src/shard/system/segments.rs +++ b/core/server/src/shard/system/segments.rs @@ -1,5 +1,3 @@ -use crate::shard::IggyShard; -use crate::streaming; /* Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,8 +15,10 @@ use crate::streaming; * specific language governing permissions and limitations * under the License. */ -use iggy_common::Identifier; -use iggy_common::IggyError; +use crate::shard::IggyShard; +use crate::shard::namespace::IggyNamespace; +use crate::streaming::segments::Segment; +use iggy_common::{Identifier, IggyError}; impl IggyShard { pub async fn delete_segments_bypass_auth( @@ -28,47 +28,46 @@ impl IggyShard { partition_id: usize, segments_count: u32, ) -> Result<(), IggyError> { - self.delete_segments_base(stream_id, topic_id, partition_id, segments_count) + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + self.delete_segments_base(stream, topic, partition_id, segments_count) .await } - pub async fn delete_segments_base( + pub(crate) async fn delete_segments_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream: usize, + topic: usize, partition_id: usize, segments_count: u32, ) -> Result<(), IggyError> { - let (segments, storages, stats) = self.streams.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - |(_, stats, .., log)| { - let upperbound = log.segments().len(); - let begin = upperbound.saturating_sub(segments_count as usize); - let segments = log - .segments_mut() - .drain(begin..upperbound) - .collect::>(); - let storages = log - .storages_mut() - .drain(begin..upperbound) - .collect::>(); - let _ = log - .indexes_mut() - .drain(begin..upperbound) - .collect::>(); - (segments, storages, stats.clone()) - }, - ); - let numeric_stream_id = self - .streams - .with_stream_by_id(stream_id, streaming::streams::helpers::get_stream_id()); - let numeric_topic_id = self.streams.with_topic_by_id( - stream_id, - topic_id, - streaming::topics::helpers::get_topic_id(), - ); + let namespace = IggyNamespace::new(stream, topic, partition_id); + + // Drain segments from partition_store + let (segments, storages, stats) = { + let mut store = self.partition_store.borrow_mut(); + let partition_data = store + .get_mut(&namespace) + .expect("delete_segments_base: partition must exist in partition_store"); + + let upperbound = partition_data.log.segments().len(); + let begin = upperbound.saturating_sub(segments_count as usize); + let segments = partition_data + .log + .segments_mut() + .drain(begin..upperbound) + .collect::>(); + let storages = partition_data + .log + .storages_mut() + .drain(begin..upperbound) + .collect::>(); + let _ = partition_data + .log + .indexes_mut() + .drain(begin..upperbound) + .collect::>(); + (segments, storages, partition_data.stats.clone()) + }; for (mut storage, segment) in storages.into_iter().zip(segments.into_iter()) { let (msg_writer, index_writer) = storage.shutdown(); @@ -81,35 +80,140 @@ impl IggyShard { let path = msg_writer.path(); drop(msg_writer); drop(index_writer); - compio::fs::remove_file(&path).await.map_err(|e| { - tracing::error!( - "Failed to delete segment file at path: {}, err: {}", - path, - e - ); - IggyError::CannotDeleteFile - })?; + // File might not exist if never actually written to disk (lazy creation) + match compio::fs::remove_file(&path).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::debug!( + "Segment file already gone or never created at path: {}", + path + ); + } + Err(e) => { + tracing::error!( + "Failed to delete segment file at path: {}, err: {}", + path, + e + ); + return Err(IggyError::CannotDeleteFile); + } + } } else { let start_offset = segment.start_offset; let path = self.config.system.get_messages_file_path( - numeric_stream_id, - numeric_topic_id, + stream, + topic, partition_id, start_offset, ); - compio::fs::remove_file(&path).await.map_err(|e| { - tracing::error!( - "Failed to delete segment file at path: {}, err: {}", - path, - e - ); - IggyError::CannotDeleteFile - })?; + // File might not exist if segment was never written to (lazy creation) + match compio::fs::remove_file(&path).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::debug!( + "Segment file already gone or never created at path: {}", + path + ); + } + Err(e) => { + tracing::error!( + "Failed to delete segment file at path: {}, err: {}", + path, + e + ); + return Err(IggyError::CannotDeleteFile); + } + } } } - self.init_log(stream_id, topic_id, partition_id).await?; - // TODO: Tech debt. make the increment seg count be part of init_log. + + // Add segment directly to partition_store + self.init_log_in_partition_store(&namespace).await?; stats.increment_segments_count(1); Ok(()) } + + /// Initialize a new segment in partition_store. + /// Used when partition data is in partition_store (not slabs). + async fn init_log_in_partition_store( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + use crate::streaming::segments::storage::create_segment_storage; + + let start_offset = 0; + let segment = Segment::new( + start_offset, + self.config.system.segment.size, + self.config.system.segment.message_expiry, + ); + + let storage = create_segment_storage( + &self.config.system, + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + 0, // messages_size + 0, // indexes_size + start_offset, + ) + .await?; + + let mut store = self.partition_store.borrow_mut(); + if let Some(partition_data) = store.get_mut(namespace) { + partition_data.log.add_persisted_segment(segment, storage); + // Reset offset when starting fresh with a new segment at offset 0 + partition_data + .offset + .store(start_offset, std::sync::atomic::Ordering::SeqCst); + partition_data.should_increment_offset = false; + } + Ok(()) + } + + /// Rotate to a new segment when the current segment is full. + /// The new segment starts at the next offset after the current segment's end. + pub(crate) async fn rotate_segment_in_partition_store( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + use crate::streaming::segments::storage::create_segment_storage; + + let start_offset = { + let store = self.partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("rotate_segment: partition must exist"); + partition_data.log.active_segment().end_offset + 1 + }; + + let segment = Segment::new( + start_offset, + self.config.system.segment.size, + self.config.system.segment.message_expiry, + ); + + let storage = create_segment_storage( + &self.config.system, + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + 0, // messages_size + 0, // indexes_size + start_offset, + ) + .await?; + + let mut store = self.partition_store.borrow_mut(); + if let Some(partition_data) = store.get_mut(namespace) { + partition_data.log.add_persisted_segment(segment, storage); + partition_data.stats.increment_segments_count(1); + tracing::info!( + "Rotated to new segment at offset {} for partition {:?}", + start_offset, + namespace + ); + } + Ok(()) + } } diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs index 35d94ef007..d157da8ab8 100644 --- a/core/server/src/shard/system/snapshot/mod.rs +++ b/core/server/src/shard/system/snapshot/mod.rs @@ -43,12 +43,10 @@ use std::process::Command; impl IggyShard { pub async fn get_snapshot( &self, - session: &Session, + _session: &Session, compression: SnapshotCompression, snapshot_types: &Vec, ) -> Result { - self.ensure_authenticated(session)?; - let snapshot_types = if snapshot_types.contains(&SystemSnapshotType::All) { if snapshot_types.len() > 1 { error!("When using 'All' snapshot type, no other types can be specified"); diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs index b071d1d8c8..ac36d557b2 100644 --- a/core/server/src/shard/system/stats.rs +++ b/core/server/src/shard/system/stats.rs @@ -17,7 +17,6 @@ */ use crate::shard::IggyShard; -use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents}; use crate::{SEMANTIC_VERSION, VERSION}; use iggy_common::{IggyDuration, IggyError, Stats}; use std::cell::RefCell; @@ -90,42 +89,33 @@ impl IggyShard { stats.written_bytes = disk_usage.total_written_bytes.into(); } - self.streams.with_components(|stream_components| { - let (stream_roots, stream_stats) = stream_components.into_components(); - // Iterate through all streams - for (stream_id, stream_root) in stream_roots.iter() { - stats.streams_count += 1; + let metadata = self.metadata.load(); - // Get stream-level stats - if let Some(stream_stat) = stream_stats.get(stream_id) { - stats.messages_count += stream_stat.messages_count_inconsistent(); - stats.segments_count += stream_stat.segments_count_inconsistent(); - stats.messages_size_bytes += stream_stat.size_bytes_inconsistent().into(); - } + stats.streams_count = metadata.streams.len() as u32; - // Access topics within this stream - stream_root.topics().with_components(|topic_components| { - let (topic_roots, ..) = topic_components.into_components(); - stats.topics_count += topic_roots.len() as u32; - - // Iterate through all topics in this stream - for (_, topic_root) in topic_roots.iter() { - // Count partitions in this topic - topic_root - .partitions() - .with_components(|partition_components| { - let (partition_roots, ..) = - partition_components.into_components(); - stats.partitions_count += partition_roots.len() as u32; - }); - - // Count consumer groups in this topic - stats.consumer_groups_count += - topic_root.consumer_groups().len() as u32; - } - }); + // Count topics, partitions, and consumer groups by iterating the hierarchy + let mut topics_count = 0u32; + let mut partitions_count = 0u32; + let mut consumer_groups_count = 0u32; + for (_, stream) in metadata.streams.iter() { + topics_count += stream.topics.len() as u32; + for (_, topic) in stream.topics.iter() { + partitions_count += topic.partitions.len() as u32; + consumer_groups_count += topic.consumer_groups.len() as u32; } - }); + } + stats.topics_count = topics_count; + stats.partitions_count = partitions_count; + stats.consumer_groups_count = consumer_groups_count; + + // Aggregate stats from SharedMetadata (cross-shard visible) + for stream_id in metadata.streams.keys() { + if let Some(stream_stat) = self.metadata.get_stream_stats(stream_id) { + stats.messages_count += stream_stat.messages_count_inconsistent(); + stats.segments_count += stream_stat.segments_count_inconsistent(); + stats.messages_size_bytes += stream_stat.size_bytes_inconsistent().into(); + } + } Ok(stats) }) diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs index 418532542c..1ad3bd2394 100644 --- a/core/server/src/shard/system/streams.rs +++ b/core/server/src/shard/system/streams.rs @@ -18,43 +18,32 @@ use super::COMPONENT; use crate::shard::IggyShard; -use crate::slab::traits_ext::{DeleteCell, EntityMarker, InsertCell}; +use crate::shard::namespace::IggyNamespace; use crate::streaming::session::Session; -use crate::streaming::streams::storage::{create_stream_file_hierarchy, delete_stream_from_disk}; -use crate::streaming::streams::{self, stream}; +use crate::streaming::streams::storage::{create_stream_file_hierarchy, delete_stream_directory}; use err_trail::ErrContext; use iggy_common::{Identifier, IggyError}; +use std::sync::Arc; + +/// Info returned when a stream is deleted - contains what callers need for logging/events. +pub struct DeletedStreamInfo { + pub id: usize, + pub name: String, +} impl IggyShard { - pub async fn create_stream( - &self, - session: &Session, - name: String, - ) -> Result { - self.ensure_authenticated(session)?; - self.permissioner - .borrow() - .create_stream(session.get_user_id())?; - let exists = self - .streams - .exists(&Identifier::from_str_value(&name).unwrap()); + pub async fn create_stream(&self, session: &Session, name: String) -> Result { + self.permissioner.create_stream(session.get_user_id())?; - if exists { - return Err(IggyError::StreamNameAlreadyExists(name)); - } - let stream = stream::create_and_insert_stream_mem(&self.streams, name); - self.metrics.increment_streams(1); - create_stream_file_hierarchy(stream.id(), &self.config.system).await?; - Ok(stream) - } + let created_at = iggy_common::IggyTimestamp::now(); + let (stream_id, _stats) = self + .metadata + .try_register_stream(Arc::from(name.as_str()), created_at)?; - pub fn create_stream_bypass_auth(&self, stream: stream::Stream) -> usize { - self.streams.insert(stream) - } + self.metrics.increment_streams(1); - pub fn update_stream_bypass_auth(&self, id: &Identifier, name: &str) -> Result<(), IggyError> { - self.update_stream_base(id, name.to_string())?; - Ok(()) + create_stream_file_hierarchy(stream_id, &self.config.system).await?; + Ok(stream_id) } pub fn update_stream( @@ -63,15 +52,10 @@ impl IggyShard { stream_id: &Identifier, name: String, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_stream_exists(stream_id)?; - let id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); + let stream = self.resolve_stream_id(stream_id)?; self.permissioner - .borrow() - .update_stream(session.get_user_id(), id) + .update_stream(session.get_user_id(), stream) .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - failed to update stream, user ID: {}, stream ID: {}", @@ -79,85 +63,111 @@ impl IggyShard { stream_id ) })?; - self.update_stream_base(stream_id, name)?; - Ok(()) + self.update_stream_base(stream, name) } - fn update_stream_base(&self, id: &Identifier, name: String) -> Result<(), IggyError> { - let old_name = self - .streams - .with_stream_by_id(id, streams::helpers::get_stream_name()); - - if old_name == name { - return Ok(()); - } - if self.streams.with_index(|index| index.contains_key(&name)) { - return Err(IggyError::StreamNameAlreadyExists(name.to_string())); - } - - self.streams - .with_stream_by_id_mut(id, streams::helpers::update_stream_name(name.clone())); - self.streams.with_index_mut(|index| { - // Rename the key inside of hashmap - let idx = index.remove(&old_name).expect("Rename key: key not found"); - index.insert(name, idx); - }); - Ok(()) + fn update_stream_base(&self, stream_id: usize, name: String) -> Result<(), IggyError> { + self.metadata + .try_update_stream(stream_id, Arc::from(name.as_str())) } - pub fn delete_stream_bypass_auth(&self, id: &Identifier) -> stream::Stream { - self.delete_stream_base(id) + pub fn delete_stream_bypass_auth( + &self, + id: &Identifier, + ) -> Result { + let stream = self.resolve_stream_id(id)?; + Ok(self.delete_stream_base(stream)) } - fn delete_stream_base(&self, id: &Identifier) -> stream::Stream { - let stream_index = self.streams.get_index(id); - let stream = self.streams.delete(stream_index); - let stats = stream.stats(); + fn delete_stream_base(&self, stream_id: usize) -> DeletedStreamInfo { + let metadata = self.metadata.load(); + let stream_meta = metadata + .streams + .get(stream_id) + .expect("Stream metadata must exist"); + let stream_name = stream_meta.name.to_string(); + let stats = stream_meta.stats.clone(); + + let topics_count = stream_meta.topics.len(); + let partitions_count: usize = stream_meta + .topics + .iter() + .map(|(_, topic)| topic.partitions.len()) + .sum(); + + { + let mut store = self.partition_store.borrow_mut(); + for (topic_id, topic) in stream_meta.topics.iter() { + for (partition_id, _) in topic.partitions.iter() { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + store.remove(&ns); + } + } + } + drop(metadata); self.metrics.decrement_streams(1); - self.metrics.decrement_topics(0); - self.metrics.decrement_partitions(0); + self.metrics.decrement_topics(topics_count as u32); + self.metrics.decrement_partitions(partitions_count as u32); self.metrics .decrement_messages(stats.messages_count_inconsistent()); self.metrics .decrement_segments(stats.segments_count_inconsistent()); - stream + + self.metadata.delete_stream(stream_id); + + DeletedStreamInfo { + id: stream_id, + name: stream_name, + } } pub async fn delete_stream( &self, session: &Session, id: &Identifier, - ) -> Result { - self.ensure_authenticated(session)?; - self.ensure_stream_exists(id)?; - let stream_id = self - .streams - .with_stream_by_id(id, streams::helpers::get_stream_id()); + ) -> Result { + let stream = self.resolve_stream_id(id)?; + self.permissioner - .borrow() - .delete_stream(session.get_user_id(), stream_id) + .delete_stream(session.get_user_id(), stream) .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to delete stream for user {}, stream ID: {}", session.get_user_id(), - stream_id, + stream, ) })?; - let mut stream = self.delete_stream_base(id); - let stream_id_usize = stream.id(); - // Clean up consumer groups from ClientManager for this stream + let topics_with_partitions: Vec<(usize, Vec)> = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .map(|stream_meta| { + stream_meta + .topics + .iter() + .map(|(topic_id, topic)| { + let partition_ids: Vec = topic.partitions.keys().collect(); + (topic_id, partition_ids) + }) + .collect() + }) + .unwrap_or_default() + }; + + let stream_info = self.delete_stream_base(stream); + self.client_manager - .delete_consumer_groups_for_stream(stream_id_usize); + .delete_consumer_groups_for_stream(stream); - // Remove all entries from shards_table for this stream (all topics and partitions) let namespaces_to_remove: Vec<_> = self .shards_table .iter() .filter_map(|entry| { let (ns, _) = entry.pair(); - if ns.stream_id() == stream_id_usize { + if ns.stream_id() == stream { Some(*ns) } else { None @@ -169,8 +179,8 @@ impl IggyShard { self.remove_shard_table_record(&ns); } - delete_stream_from_disk(&mut stream, &self.config.system).await?; - Ok(stream) + delete_stream_directory(stream, &topics_with_partitions, &self.config.system).await?; + Ok(stream_info) } pub async fn purge_stream( @@ -178,44 +188,161 @@ impl IggyShard { session: &Session, stream_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_stream_exists(stream_id)?; - { - let get_stream_id = crate::streaming::streams::helpers::get_stream_id(); - let stream_id = self.streams.with_stream_by_id(stream_id, get_stream_id); - self.permissioner - .borrow() - .purge_stream(session.get_user_id(), stream_id) - .error(|e: &IggyError| { + let stream = self.resolve_stream_id(stream_id)?; + + self.permissioner + .purge_stream(session.get_user_id(), stream) + .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to purge stream for user {}, stream ID: {}", session.get_user_id(), - stream_id, + stream, ) })?; - } - self.purge_stream_base(stream_id).await + self.purge_stream_base(stream).await } pub async fn purge_stream_bypass_auth(&self, stream_id: &Identifier) -> Result<(), IggyError> { - self.purge_stream_base(stream_id).await?; - Ok(()) + let stream = self.resolve_stream_id(stream_id)?; + self.purge_stream_base(stream).await } - async fn purge_stream_base(&self, stream_id: &Identifier) -> Result<(), IggyError> { - // Get all topic IDs in the stream - let topic_ids = self + async fn purge_stream_base(&self, stream_id: usize) -> Result<(), IggyError> { + let metadata = self.metadata.load(); + let topic_ids: Vec = metadata .streams - .with_stream_by_id(stream_id, streams::helpers::get_topic_ids()); + .get(stream_id) + .map(|stream| stream.topics.keys().collect()) + .unwrap_or_default(); + drop(metadata); - // Purge each topic in the stream using bypass auth for topic_id in topic_ids { - let topic_identifier = Identifier::numeric(topic_id as u32).unwrap(); - self.purge_topic_bypass_auth(stream_id, &topic_identifier) - .await?; + self.purge_topic_base(stream_id, topic_id).await?; } Ok(()) } + + pub fn get_stream_from_shared_metadata(&self, stream_id: usize) -> bytes::Bytes { + use crate::shard::namespace::IggyNamespace; + use bytes::{BufMut, BytesMut}; + + let metadata = self.metadata.load(); + + let Some(stream_meta) = metadata.streams.get(stream_id) else { + return bytes::Bytes::new(); + }; + + let mut topic_ids: Vec<_> = stream_meta.topics.keys().collect(); + topic_ids.sort_unstable(); + + let (total_size, total_messages) = { + let mut size = 0u64; + let mut messages = 0u64; + for &topic_id in &topic_ids { + if let Some(topic) = stream_meta.topics.get(topic_id) { + for (partition_id, _) in topic.partitions.iter() { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + if let Some(stats) = self.metadata.get_partition_stats(&ns) { + size += stats.size_bytes_inconsistent(); + messages += stats.messages_count_inconsistent(); + } + } + } + } + (size, messages) + }; + + let mut bytes = BytesMut::new(); + + bytes.put_u32_le(stream_meta.id as u32); + bytes.put_u64_le(stream_meta.created_at.into()); + bytes.put_u32_le(topic_ids.len() as u32); + bytes.put_u64_le(total_size); + bytes.put_u64_le(total_messages); + bytes.put_u8(stream_meta.name.len() as u8); + bytes.put_slice(stream_meta.name.as_bytes()); + + for &topic_id in &topic_ids { + if let Some(topic_meta) = stream_meta.topics.get(topic_id) { + let mut partition_ids: Vec<_> = topic_meta.partitions.keys().collect(); + partition_ids.sort_unstable(); + + let (topic_size, topic_messages) = { + let mut size = 0u64; + let mut messages = 0u64; + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + if let Some(stats) = self.metadata.get_partition_stats(&ns) { + size += stats.size_bytes_inconsistent(); + messages += stats.messages_count_inconsistent(); + } + } + (size, messages) + }; + + bytes.put_u32_le(topic_meta.id as u32); + bytes.put_u64_le(topic_meta.created_at.into()); + bytes.put_u32_le(partition_ids.len() as u32); + bytes.put_u64_le(topic_meta.message_expiry.into()); + bytes.put_u8(topic_meta.compression_algorithm.as_code()); + bytes.put_u64_le(topic_meta.max_topic_size.into()); + bytes.put_u8(topic_meta.replication_factor); + bytes.put_u64_le(topic_size); + bytes.put_u64_le(topic_messages); + bytes.put_u8(topic_meta.name.len() as u8); + bytes.put_slice(topic_meta.name.as_bytes()); + } + } + + bytes.freeze() + } + + pub fn get_streams_from_shared_metadata(&self) -> bytes::Bytes { + use crate::shard::namespace::IggyNamespace; + use bytes::{BufMut, BytesMut}; + + let metadata = self.metadata.load(); + let mut bytes = BytesMut::new(); + + let mut stream_ids: Vec<_> = metadata.streams.keys().collect(); + stream_ids.sort_unstable(); + + for stream_id in stream_ids { + let Some(stream_meta) = metadata.streams.get(stream_id) else { + continue; + }; + + let mut topic_ids: Vec<_> = stream_meta.topics.keys().collect(); + topic_ids.sort_unstable(); + + let (total_size, total_messages) = { + let mut size = 0u64; + let mut messages = 0u64; + for &topic_id in &topic_ids { + if let Some(topic) = stream_meta.topics.get(topic_id) { + for (partition_id, _) in topic.partitions.iter() { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + if let Some(stats) = self.metadata.get_partition_stats(&ns) { + size += stats.size_bytes_inconsistent(); + messages += stats.messages_count_inconsistent(); + } + } + } + } + (size, messages) + }; + + bytes.put_u32_le(stream_meta.id as u32); + bytes.put_u64_le(stream_meta.created_at.into()); + bytes.put_u32_le(topic_ids.len() as u32); + bytes.put_u64_le(total_size); + bytes.put_u64_le(total_messages); + bytes.put_u8(stream_meta.name.len() as u8); + bytes.put_slice(stream_meta.name.as_bytes()); + } + + bytes.freeze() + } } diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs index 042a49821f..ad260cd393 100644 --- a/core/server/src/shard/system/topics.rs +++ b/core/server/src/shard/system/topics.rs @@ -18,16 +18,21 @@ use super::COMPONENT; use crate::shard::IggyShard; -use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, InsertCell, IntoComponents}; +use crate::shard::namespace::IggyNamespace; use crate::streaming::session::Session; -use crate::streaming::topics::storage::{create_topic_file_hierarchy, delete_topic_from_disk}; -use crate::streaming::topics::topic::{self}; -use crate::streaming::{partitions, streams, topics}; +use crate::streaming::topics::storage::{create_topic_file_hierarchy, delete_topic_directory}; use err_trail::ErrContext; use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize}; -use std::str::FromStr; +use std::sync::Arc; use tracing::info; +/// Info returned when a topic is deleted - contains what callers need for logging/events. +pub struct DeletedTopicInfo { + pub id: usize, + pub name: String, + pub stream_id: usize, +} + impl IggyShard { #[allow(clippy::too_many_arguments)] pub async fn create_topic( @@ -39,14 +44,11 @@ impl IggyShard { compression: CompressionAlgorithm, max_topic_size: MaxTopicSize, replication_factor: Option, - ) -> Result { - self.ensure_authenticated(session)?; - self.ensure_stream_exists(stream_id)?; - let numeric_stream_id = self.streams.get_index(stream_id); + ) -> Result { + let stream = self.resolve_stream_id(stream_id)?; { self.permissioner - .borrow() - .create_topic(session.get_user_id(), numeric_stream_id) + .create_topic(session.get_user_id(), stream) .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to create topic with name: {name} in stream with ID: {stream_id} for user with ID: {}", @@ -54,41 +56,56 @@ impl IggyShard { ) })?; } - let exists = self.streams.with_topics( - stream_id, - topics::helpers::exists(&Identifier::from_str(&name).unwrap()), - ); - if exists { - return Err(IggyError::TopicNameAlreadyExists(name, stream_id.clone())); - } let config = &self.config.system; - let parent_stats = self - .streams - .with_stream_by_id(stream_id, |(_, stats)| stats.clone()); let message_expiry = config.resolve_message_expiry(message_expiry); info!("Topic message expiry: {}", message_expiry); let max_topic_size = config.resolve_max_topic_size(max_topic_size)?; - let topic = topic::create_and_insert_topics_mem( - &self.streams, - stream_id, - name, - replication_factor.unwrap_or(1), + + let created_at = iggy_common::IggyTimestamp::now(); + + let (topic_id, _stats) = self.metadata.try_register_topic( + stream, + Arc::from(name.as_str()), + created_at, message_expiry, compression, max_topic_size, - parent_stats, - ); + replication_factor.unwrap_or(1), + 0, // partitions_count starts at 0 + )?; + self.metrics.increment_topics(1); - // Create file hierarchy for the topic. - create_topic_file_hierarchy(numeric_stream_id, topic.id(), &self.config.system).await?; - Ok(topic) + create_topic_file_hierarchy(stream, topic_id, &self.config.system).await?; + Ok(topic_id) } - pub fn create_topic_bypass_auth(&self, stream_id: &Identifier, topic: topic::Topic) -> usize { - self.streams - .with_topics(stream_id, |topics| topics.insert(topic)) + #[allow(clippy::too_many_arguments)] + pub fn create_topic_bypass_auth( + &self, + stream_id: usize, + topic_id: usize, + name: &str, + created_at: iggy_common::IggyTimestamp, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + partitions_count: u32, + ) -> usize { + let _stats = self.metadata.register_topic( + stream_id, + topic_id, + Arc::from(name), + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + partitions_count, + ); + topic_id } #[allow(clippy::too_many_arguments)] @@ -103,47 +120,27 @@ impl IggyShard { max_topic_size: MaxTopicSize, replication_factor: Option, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - { - let topic_id_val = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id_val = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().update_topic( - session.get_user_id(), - stream_id_val, - topic_id_val - ).error(|e: &IggyError| { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + self.permissioner + .update_topic(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to update topic for user with id: {}, stream ID: {}, topic ID: {}", session.get_user_id(), - stream_id_val, - topic_id_val, + stream, + topic, ) })?; - } - - let exists = self.streams.with_topics( - stream_id, - topics::helpers::exists(&Identifier::from_str(&name).unwrap()), - ); - if exists { - return Err(IggyError::TopicNameAlreadyExists(name, stream_id.clone())); - } self.update_topic_base( - stream_id, - topic_id, + stream, + topic, name, message_expiry, compression_algorithm, max_topic_size, replication_factor.unwrap_or(1), - ); - Ok(()) + ) } #[allow(clippy::too_many_arguments)] @@ -157,43 +154,38 @@ impl IggyShard { max_topic_size: MaxTopicSize, replication_factor: Option, ) -> Result<(), IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; self.update_topic_base( - stream_id, - topic_id, + stream, + topic, name, message_expiry, compression_algorithm, max_topic_size, replication_factor.unwrap_or(1), - ); - Ok(()) + ) } #[allow(clippy::too_many_arguments)] - pub fn update_topic_base( + fn update_topic_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream: usize, + topic: usize, name: String, message_expiry: IggyExpiry, compression_algorithm: CompressionAlgorithm, max_topic_size: MaxTopicSize, replication_factor: u8, - ) { - let update_topic_closure = topics::helpers::update_topic( - name.clone(), + ) -> Result<(), IggyError> { + self.metadata.try_update_topic( + stream, + topic, + Arc::from(name.as_str()), message_expiry, compression_algorithm, max_topic_size, replication_factor, - ); - let (old_name, new_name) = - self.streams - .with_topic_by_id_mut(stream_id, topic_id, update_topic_closure); - if old_name != new_name { - let rename_closure = topics::helpers::rename_index(&old_name, new_name); - self.streams.with_topics(stream_id, rename_closure); - } + ) } pub async fn delete_topic( @@ -201,38 +193,48 @@ impl IggyShard { session: &Session, stream_id: &Identifier, topic_id: &Identifier, - ) -> Result { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - let numeric_topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let numeric_stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); + ) -> Result { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; self.permissioner - .borrow() - .delete_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id) + .delete_topic(session.get_user_id(), stream, topic) .error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to delete topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}", session.get_user_id(), ) })?; - let mut topic = self.delete_topic_base(stream_id, topic_id); - let topic_id_numeric = topic.id(); - // Clean up consumer groups from ClientManager for this topic + let (partition_ids, messages_count, size_bytes, segments_count, parent_stats) = { + let metadata = self.metadata.load(); + let stream_meta = metadata + .streams + .get(stream) + .expect("Stream metadata must exist"); + let topic_meta = stream_meta + .topics + .get(topic) + .expect("Topic metadata must exist"); + let pids: Vec = topic_meta.partitions.keys().collect(); + ( + pids, + topic_meta.stats.messages_count_inconsistent(), + topic_meta.stats.size_bytes_inconsistent(), + topic_meta.stats.segments_count_inconsistent(), + topic_meta.stats.parent().clone(), + ) + }; + + let topic_info = self.delete_topic_base(stream, topic); + self.client_manager - .delete_consumer_groups_for_topic(numeric_stream_id, topic_id_numeric); + .delete_consumer_groups_for_topic(stream, topic_info.id); - // Remove all partition entries from shards_table for this topic let namespaces_to_remove: Vec<_> = self .shards_table .iter() .filter_map(|entry| { let (ns, _) = entry.pair(); - if ns.stream_id() == numeric_stream_id && ns.topic_id() == topic_id_numeric { + if ns.stream_id() == stream && ns.topic_id() == topic_info.id { Some(*ns) } else { None @@ -244,28 +246,55 @@ impl IggyShard { self.remove_shard_table_record(&ns); } - let parent = topic.stats().parent().clone(); - // We need to borrow topic as mutable, as we are extracting partitions out of it, in order to close them. - let (messages_count, size_bytes, segments_count) = - delete_topic_from_disk(numeric_stream_id, &mut topic, &self.config.system).await?; - parent.decrement_messages_count(messages_count); - parent.decrement_size_bytes(size_bytes); - parent.decrement_segments_count(segments_count); + delete_topic_directory(stream, topic_info.id, &partition_ids, &self.config.system).await?; + + parent_stats.decrement_messages_count(messages_count); + parent_stats.decrement_size_bytes(size_bytes); + parent_stats.decrement_segments_count(segments_count); self.metrics.decrement_topics(1); - Ok(topic) + Ok(topic_info) } pub fn delete_topic_bypass_auth( &self, stream_id: &Identifier, topic_id: &Identifier, - ) -> topic::Topic { - self.delete_topic_base(stream_id, topic_id) + ) -> Result { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + Ok(self.delete_topic_base(stream, topic)) } - pub fn delete_topic_base(&self, stream_id: &Identifier, topic_id: &Identifier) -> topic::Topic { - self.streams - .with_topics(stream_id, topics::helpers::delete_topic(topic_id)) + fn delete_topic_base(&self, stream: usize, topic: usize) -> DeletedTopicInfo { + let (topic_name, partition_ids) = { + let metadata = self.metadata.load(); + let stream_meta = metadata + .streams + .get(stream) + .expect("Stream metadata must exist"); + let topic_meta = stream_meta + .topics + .get(topic) + .expect("Topic metadata must exist"); + let name = topic_meta.name.to_string(); + let pids: Vec = topic_meta.partitions.keys().collect(); + (name, pids) + }; + + { + let mut store = self.partition_store.borrow_mut(); + for partition_id in partition_ids { + let ns = IggyNamespace::new(stream, topic, partition_id); + store.remove(&ns); + } + } + + self.metadata.delete_topic(stream, topic); + + DeletedTopicInfo { + id: topic, + name: topic_name, + stream_id: stream, + } } pub async fn purge_topic( @@ -274,41 +303,58 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - self.ensure_topic_exists(stream_id, topic_id)?; - { - let topic_id = - self.streams - .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - let stream_id = self - .streams - .with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - self.permissioner.borrow().purge_topic( - session.get_user_id(), - stream_id, - topic_id - ).error(|e: &IggyError| { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + + self.permissioner + .purge_topic(session.get_user_id(), stream, topic) + .error(|e: &IggyError| { format!( - "{COMPONENT} (error: {e}) - permission denied to purge topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}", + "{COMPONENT} (error: {e}) - permission denied to purge topic with ID: {topic} in stream with ID: {stream} for user with ID: {}", session.get_user_id(), ) })?; + + let partition_ids: Vec = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| t.partitions.keys().collect()) + .unwrap_or_default() + }; + + let mut all_consumer_paths = Vec::new(); + let mut all_group_paths = Vec::new(); + + for partition_id in &partition_ids { + let ns = crate::shard::namespace::IggyNamespace::new(stream, topic, *partition_id); + if let Some(partition_data) = self.partition_store.borrow().get(&ns) { + all_consumer_paths.extend( + partition_data + .consumer_offsets + .pin() + .iter() + .map(|item| item.1.path.clone()), + ); + all_group_paths.extend( + partition_data + .consumer_group_offsets + .pin() + .iter() + .map(|item| item.1.path.clone()), + ); + } } - let (consumer_offset_paths, consumer_group_offset_paths) = self.streams.with_partitions( - stream_id, - topic_id, - partitions::helpers::purge_consumer_offsets(), - ); - for path in consumer_offset_paths { + for path in all_consumer_paths { self.delete_consumer_offset_from_disk(&path).await?; } - for path in consumer_group_offset_paths { + for path in all_group_paths { self.delete_consumer_offset_from_disk(&path).await?; } - self.purge_topic_base(stream_id, topic_id).await?; - Ok(()) + self.purge_topic_base(stream, topic).await } pub async fn purge_topic_bypass_auth( @@ -316,41 +362,177 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, ) -> Result<(), IggyError> { - self.purge_topic_base(stream_id, topic_id).await?; - Ok(()) + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + self.purge_topic_base(stream, topic).await } - async fn purge_topic_base( + pub(crate) async fn purge_topic_base( &self, - stream_id: &Identifier, - topic_id: &Identifier, + stream: usize, + topic: usize, ) -> Result<(), IggyError> { - let part_ids = self - .streams - .with_partitions(stream_id, topic_id, |partitions| { - partitions.with_components(|components| { - let (roots, ..) = components.into_components(); - roots.iter().map(|(_, root)| root.id()).collect::>() - }) - }); + use crate::shard::namespace::IggyNamespace; - self.streams.with_partitions( - stream_id, - topic_id, - partitions::helpers::purge_partitions_mem(), - ); + let partition_ids: Vec = { + let metadata = self.metadata.load(); + metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .map(|t| t.partitions.keys().collect()) + .unwrap_or_default() + }; + + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic, partition_id); - for part_id in part_ids { - self.delete_segments_bypass_auth(stream_id, topic_id, part_id, u32::MAX) - .await?; + let has_partition = self.partition_store.borrow().contains(&ns); + if has_partition { + self.delete_segments_base(stream, topic, partition_id, u32::MAX) + .await?; + } } - // Zero out topic stats after purging all partitions - self.streams - .with_topic_by_id(stream_id, topic_id, |(_root, _aux, stats)| { - stats.zero_out_all(); - }); + if let Some(topic_stats) = self.metadata.get_topic_stats(stream, topic) { + topic_stats.zero_out_all(); + } + + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic, partition_id); + if let Some(partition_stats) = self.metadata.get_partition_stats(&ns) { + partition_stats.zero_out_all(); + } + } Ok(()) } + + pub fn get_topic_from_shared_metadata( + &self, + stream_id: usize, + topic_id: usize, + ) -> bytes::Bytes { + use crate::shard::namespace::IggyNamespace; + use bytes::{BufMut, BytesMut}; + + let metadata = self.metadata.load(); + + let Some(stream_meta) = metadata.streams.get(stream_id) else { + return bytes::Bytes::new(); + }; + let Some(topic_meta) = stream_meta.topics.get(topic_id) else { + return bytes::Bytes::new(); + }; + + let mut partition_ids: Vec<_> = topic_meta.partitions.keys().collect(); + partition_ids.sort_unstable(); + + let (total_size, total_messages) = { + let mut size = 0u64; + let mut messages = 0u64; + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + if let Some(stats) = self.metadata.get_partition_stats(&ns) { + size += stats.size_bytes_inconsistent(); + messages += stats.messages_count_inconsistent(); + } + } + (size, messages) + }; + + let mut bytes = BytesMut::new(); + + bytes.put_u32_le(topic_meta.id as u32); + bytes.put_u64_le(topic_meta.created_at.into()); + bytes.put_u32_le(partition_ids.len() as u32); + bytes.put_u64_le(topic_meta.message_expiry.into()); + bytes.put_u8(topic_meta.compression_algorithm.as_code()); + bytes.put_u64_le(topic_meta.max_topic_size.into()); + bytes.put_u8(topic_meta.replication_factor); + bytes.put_u64_le(total_size); + bytes.put_u64_le(total_messages); + bytes.put_u8(topic_meta.name.len() as u8); + bytes.put_slice(topic_meta.name.as_bytes()); + + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + let partition_meta = topic_meta.partitions.get(partition_id); + let created_at = partition_meta + .map(|m| m.created_at) + .unwrap_or_else(iggy_common::IggyTimestamp::now); + + let (segments_count, size_bytes, messages_count, offset) = self + .metadata + .get_partition_stats(&ns) + .map(|stats| { + ( + stats.segments_count_inconsistent(), + stats.size_bytes_inconsistent(), + stats.messages_count_inconsistent(), + stats.current_offset(), + ) + }) + .unwrap_or((0, 0, 0, 0)); + + bytes.put_u32_le(partition_id as u32); + bytes.put_u64_le(created_at.into()); + bytes.put_u32_le(segments_count); + bytes.put_u64_le(offset); + bytes.put_u64_le(size_bytes); + bytes.put_u64_le(messages_count); + } + + bytes.freeze() + } + + pub fn get_topics_from_shared_metadata(&self, stream_id: usize) -> bytes::Bytes { + use crate::shard::namespace::IggyNamespace; + use bytes::{BufMut, BytesMut}; + + let metadata = self.metadata.load(); + let mut bytes = BytesMut::new(); + + let Some(stream_meta) = metadata.streams.get(stream_id) else { + return bytes.freeze(); + }; + + let mut topic_ids: Vec<_> = stream_meta.topics.keys().collect(); + topic_ids.sort_unstable(); + + for topic_id in topic_ids { + let Some(topic_meta) = stream_meta.topics.get(topic_id) else { + continue; + }; + + let mut partition_ids: Vec<_> = topic_meta.partitions.keys().collect(); + partition_ids.sort_unstable(); + + let (total_size, total_messages) = { + let mut size = 0u64; + let mut messages = 0u64; + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + if let Some(stats) = self.metadata.get_partition_stats(&ns) { + size += stats.size_bytes_inconsistent(); + messages += stats.messages_count_inconsistent(); + } + } + (size, messages) + }; + + bytes.put_u32_le(topic_meta.id as u32); + bytes.put_u64_le(topic_meta.created_at.into()); + bytes.put_u32_le(partition_ids.len() as u32); + bytes.put_u64_le(topic_meta.message_expiry.into()); + bytes.put_u8(topic_meta.compression_algorithm.as_code()); + bytes.put_u64_le(topic_meta.max_topic_size.into()); + bytes.put_u8(topic_meta.replication_factor); + bytes.put_u64_le(total_size); + bytes.put_u64_le(total_messages); + bytes.put_u8(topic_meta.name.len() as u8); + bytes.put_slice(topic_meta.name.as_bytes()); + } + + bytes.freeze() + } } diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs index 04a416444a..d9d94c0900 100644 --- a/core/server/src/shard/system/users.rs +++ b/core/server/src/shard/system/users.rs @@ -17,33 +17,64 @@ */ use super::COMPONENT; +use crate::metadata::UserMeta; use crate::shard::IggyShard; use crate::streaming::session::Session; use crate::streaming::users::user::User; use crate::streaming::utils::crypto; +use dashmap::DashMap; use err_trail::ErrContext; use iggy_common::Identifier; use iggy_common::IggyError; use iggy_common::Permissions; use iggy_common::UserStatus; +use std::sync::Arc; use tracing::{error, warn}; const MAX_USERS: usize = u32::MAX as usize; impl IggyShard { + fn user_from_meta(&self, meta: &UserMeta) -> User { + let pats = self.metadata.get_user_personal_access_tokens(meta.id); + let pat_map = DashMap::new(); + for pat in pats { + pat_map.insert(pat.token.clone(), pat); + } + User { + id: meta.id, + status: meta.status, + username: meta.username.to_string(), + password: meta.password_hash.to_string(), + created_at: meta.created_at, + permissions: meta.permissions.as_ref().map(|p| (**p).clone()), + personal_access_tokens: pat_map, + } + } + + fn get_user_from_metadata(&self, identifier: &Identifier) -> Result, IggyError> { + let user_id = match self.metadata.get_user_id(identifier) { + Some(id) => id, + None => return Ok(None), + }; + + Ok(self + .metadata + .get_user(user_id) + .map(|meta| self.user_from_meta(&meta))) + } + pub fn find_user( &self, session: &Session, user_id: &Identifier, ) -> Result, IggyError> { - self.ensure_authenticated(session)?; let Some(user) = self.try_get_user(user_id)? else { return Ok(None); }; let session_user_id = session.get_user_id(); if user.id != session_user_id { - self.permissioner.borrow().get_user(session_user_id).error(|e: &IggyError| { + self.permissioner.get_user(session_user_id).error(|e: &IggyError| { format!( "{COMPONENT} (error: {e}) - permission denied to get user with ID: {user_id} for current user with ID: {session_user_id}" ) @@ -59,13 +90,11 @@ impl IggyShard { } pub fn try_get_user(&self, user_id: &Identifier) -> Result, IggyError> { - self.users.get_by_identifier(user_id) + self.get_user_from_metadata(user_id) } pub async fn get_users(&self, session: &Session) -> Result, IggyError> { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .get_users(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -73,7 +102,14 @@ impl IggyShard { session.get_user_id() ) })?; - Ok(self.users.values()) + + let users = self + .metadata + .get_all_users() + .iter() + .map(|meta| self.user_from_meta(meta)) + .collect(); + Ok(users) } pub fn create_user( @@ -84,9 +120,7 @@ impl IggyShard { status: UserStatus, permissions: Option, ) -> Result { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .create_user(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -95,62 +129,32 @@ impl IggyShard { ) })?; - if self.users.username_exists(username) { - error!("User: {username} already exists."); - return Err(IggyError::UserAlreadyExists); - } - - if self.users.len() >= MAX_USERS { - error!("Available users limit reached."); - return Err(IggyError::UsersLimitReached); - } + let password_hash = crypto::hash_password(password); - let user_id = self.create_user_base(username, password, status, permissions)?; - self.get_user(&(user_id as u32).try_into()?) - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - }) - } - - pub fn create_user_bypass_auth( - &self, - expected_user_id: u32, - username: &str, - password: &str, - status: UserStatus, - permissions: Option, - ) -> Result<(), IggyError> { - let assigned_user_id = self.create_user_base(username, password, status, permissions)?; - - assert_eq!( - assigned_user_id as u32, expected_user_id, - "User ID mismatch: expected {}, got {}. This indicates shards are out of sync.", - expected_user_id, assigned_user_id - ); - - Ok(()) - } + let user_id = self + .metadata + .try_register_user( + Arc::from(username), + Arc::from(password_hash.as_str()), + status, + permissions.map(Arc::new), + MAX_USERS, + ) + .inspect_err(|e| match e { + IggyError::UserAlreadyExists => error!("User: {username} already exists."), + IggyError::UsersLimitReached => error!("Available users limit reached."), + _ => {} + })?; - fn create_user_base( - &self, - username: &str, - password: &str, - status: UserStatus, - permissions: Option, - ) -> Result { - let user = User::new(0, username, password, status, permissions.clone()); - let user_id = self.users.insert(user); - self.permissioner - .borrow_mut() - .init_permissions_for_user(user_id as u32, permissions); self.metrics.increment_users(1); - Ok(user_id) + + self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + }) } pub fn delete_user(&self, session: &Session, user_id: &Identifier) -> Result { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .delete_user(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -162,10 +166,6 @@ impl IggyShard { self.delete_user_base(user_id) } - pub fn delete_user_bypass_auth(&self, user_id: &Identifier) -> Result { - self.delete_user_base(user_id) - } - fn delete_user_base(&self, user_id: &Identifier) -> Result { let user = self.get_user(user_id).error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") @@ -176,16 +176,8 @@ impl IggyShard { return Err(IggyError::CannotDeleteUser(user.id)); } - let user_slab_id = user.id as usize; let user_u32_id = user.id; - let user = self - .users - .remove(user_slab_id) - .ok_or(IggyError::ResourceNotFound(user_id.to_string()))?; - self.permissioner - .borrow_mut() - .delete_permissions_for_user(user_u32_id); self.client_manager .delete_clients_for_user(user_u32_id) .error(|e: &IggyError| { @@ -194,6 +186,9 @@ impl IggyShard { ) })?; self.metrics.decrement_users(1); + + self.metadata.delete_user(user_u32_id); + Ok(user) } @@ -204,9 +199,7 @@ impl IggyShard { username: Option, status: Option, ) -> Result { - self.ensure_authenticated(session)?; self.permissioner - .borrow() .update_user(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -218,15 +211,6 @@ impl IggyShard { self.update_user_base(user_id, username, status) } - pub fn update_user_bypass_auth( - &self, - user_id: &Identifier, - username: Option, - status: Option, - ) -> Result { - self.update_user_base(user_id, username, status) - } - fn update_user_base( &self, user_id: &Identifier, @@ -234,30 +218,15 @@ impl IggyShard { status: Option, ) -> Result { let user = self.get_user(user_id)?; - let numeric_user_id = Identifier::numeric(user.id).unwrap(); - - if let Some(ref new_username) = username { - let existing_user = self.get_user(&new_username.to_owned().try_into()?); - if existing_user.is_ok() && existing_user.unwrap().id != user.id { - error!("User: {new_username} already exists."); - return Err(IggyError::UserAlreadyExists); - } - - self.users.update_username(user_id, new_username.clone())?; - } + let numeric_user_id = user.id; - if let Some(status) = status { - self.users.with_user_mut(&numeric_user_id, |user| { - user.status = status; - }).error(|e: &IggyError| { - format!("{COMPONENT} update user (error: {e}) - failed to update user with id: {user_id}") - })?; - } + let updated_meta = self.metadata.try_update_user( + numeric_user_id, + username.map(|u| Arc::from(u.as_str())), + status, + )?; - self.get_user(&numeric_user_id) - .error(|e: &IggyError| { - format!("{COMPONENT} update user (error: {e}) - failed to get updated user with id: {user_id}") - }) + Ok(self.user_from_meta(&updated_meta)) } pub fn update_permissions( @@ -266,11 +235,8 @@ impl IggyShard { user_id: &Identifier, permissions: Option, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - { self.permissioner - .borrow() .update_permissions(session.get_user_id()) .error(|e: &IggyError| { format!( @@ -289,14 +255,6 @@ impl IggyShard { self.update_permissions_base(user_id, permissions) } - pub fn update_permissions_bypass_auth( - &self, - user_id: &Identifier, - permissions: Option, - ) -> Result<(), IggyError> { - self.update_permissions_base(user_id, permissions) - } - fn update_permissions_base( &self, user_id: &Identifier, @@ -306,17 +264,23 @@ impl IggyShard { format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") })?; - self.permissioner - .borrow_mut() - .update_permissions_for_user(user.id, permissions.clone()); + let current_meta = self + .metadata + .get_user(user.id) + .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; + + let updated_meta = UserMeta { + id: current_meta.id, + username: current_meta.username, + password_hash: current_meta.password_hash, + status: current_meta.status, + permissions: permissions.map(Arc::new), + created_at: current_meta.created_at, + }; - self.users.with_user_mut(user_id, |user| { - user.permissions = permissions; - }).error(|e: &IggyError| { - format!( - "{COMPONENT} update user permissions (error: {e}) - failed to update permissions for user with id: {user_id}" - ) - }) + self.metadata.update_user_meta(user.id, updated_meta); + + Ok(()) } pub fn change_password( @@ -326,51 +290,58 @@ impl IggyShard { current_password: &str, new_password: &str, ) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; - { let user = self.get_user(user_id).error(|e: &IggyError| { format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") })?; let session_user_id = session.get_user_id(); if user.id != session_user_id { - self.permissioner - .borrow() - .change_password(session_user_id)?; + self.permissioner.change_password(session_user_id)?; } } self.change_password_base(user_id, current_password, new_password) } - pub fn change_password_bypass_auth( - &self, - user_id: &Identifier, - current_password: &str, - new_password: &str, - ) -> Result<(), IggyError> { - self.change_password_base(user_id, current_password, new_password) - } - fn change_password_base( &self, user_id: &Identifier, current_password: &str, new_password: &str, ) -> Result<(), IggyError> { - self.users.with_user_mut(user_id, |user| { - if !crypto::verify_password(current_password, &user.password) { - error!( - "Invalid current password for user: {} with ID: {user_id}.", - user.username - ); - return Err(IggyError::InvalidCredentials); - } - user.password = crypto::hash_password(new_password); - Ok(()) - }).error(|e: &IggyError| { - format!("{COMPONENT} change password (error: {e}) - failed to change password for user with id: {user_id}") - })? + let user = self.get_user(user_id).error(|e: &IggyError| { + format!( + "{COMPONENT} change password (error: {e}) - failed to get user with id: {user_id}" + ) + })?; + + // Verify current password + if !crypto::verify_password(current_password, &user.password) { + error!( + "Invalid current password for user: {} with ID: {user_id}.", + user.username + ); + return Err(IggyError::InvalidCredentials); + } + + let current_meta = self + .metadata + .get_user(user.id) + .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; + + let new_password_hash = crypto::hash_password(new_password); + let updated_meta = UserMeta { + id: current_meta.id, + username: current_meta.username, + password_hash: Arc::from(new_password_hash.as_str()), + status: current_meta.status, + permissions: current_meta.permissions, + created_at: current_meta.created_at, + }; + + self.metadata.update_user_meta(user.id, updated_meta); + + Ok(()) } pub fn login_user( @@ -437,7 +408,6 @@ impl IggyShard { } pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> { - self.ensure_authenticated(session)?; let client_id = session.client_id; self.logout_user_base(client_id)?; Ok(()) diff --git a/core/server/src/shard/system/utils.rs b/core/server/src/shard/system/utils.rs index e7ad661b14..db49594a97 100644 --- a/core/server/src/shard/system/utils.rs +++ b/core/server/src/shard/system/utils.rs @@ -17,19 +17,67 @@ use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError}; -use crate::{ - shard::IggyShard, - streaming::{ - polling_consumer::PollingConsumer, - topics::{self}, - }, -}; +use crate::{shard::IggyShard, streaming::polling_consumer::PollingConsumer}; impl IggyShard { - pub fn ensure_stream_exists(&self, stream_id: &Identifier) -> Result<(), IggyError> { - if !self.streams.exists(stream_id) { - return Err(IggyError::StreamIdNotFound(stream_id.clone())); + /// Resolves stream identifier to numeric ID, returning error if not found. + pub fn resolve_stream_id(&self, stream_id: &Identifier) -> Result { + self.metadata + .get_stream_id(stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone())) + } + + /// Resolves topic identifier to (stream_id, topic_id), returning error if not found. + pub fn resolve_topic_id( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result<(usize, usize), IggyError> { + let stream = self.resolve_stream_id(stream_id)?; + let topic = self + .metadata + .get_topic_id(stream, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + Ok((stream, topic)) + } + + /// Resolves partition identifier to (stream_id, topic_id, partition_id), returning error if not found. + pub fn resolve_partition_id( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: usize, + ) -> Result<(usize, usize, usize), IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + if !self.metadata.partition_exists(stream, topic, partition_id) { + return Err(IggyError::PartitionNotFound( + partition_id, + topic_id.clone(), + stream_id.clone(), + )); } + Ok((stream, topic, partition_id)) + } + + /// Resolves consumer group identifier to (stream_id, topic_id, group_id), returning error if not found. + pub fn resolve_consumer_group_id( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + group_id: &Identifier, + ) -> Result<(usize, usize, usize), IggyError> { + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + let group = self + .metadata + .get_consumer_group_id(stream, topic, group_id) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound(group_id.clone(), topic_id.clone()) + })?; + Ok((stream, topic, group)) + } + + pub fn ensure_stream_exists(&self, stream_id: &Identifier) -> Result<(), IggyError> { + self.resolve_stream_id(stream_id)?; Ok(()) } @@ -38,16 +86,7 @@ impl IggyShard { stream_id: &Identifier, topic_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_stream_exists(stream_id)?; - let exists = self - .streams - .with_topics(stream_id, topics::helpers::exists(topic_id)); - if !exists { - return Err(IggyError::TopicIdNotFound( - stream_id.clone(), - topic_id.clone(), - )); - } + self.resolve_topic_id(stream_id, topic_id)?; Ok(()) } @@ -57,19 +96,7 @@ impl IggyShard { topic_id: &Identifier, group_id: &Identifier, ) -> Result<(), IggyError> { - self.ensure_stream_exists(stream_id)?; - self.ensure_topic_exists(stream_id, topic_id)?; - let exists = self.streams.with_topic_by_id( - stream_id, - topic_id, - topics::helpers::cg_exists(group_id), - ); - if !exists { - return Err(IggyError::ConsumerGroupIdNotFound( - group_id.clone(), - topic_id.clone(), - )); - } + self.resolve_consumer_group_id(stream_id, topic_id, group_id)?; Ok(()) } @@ -79,10 +106,8 @@ impl IggyShard { topic_id: &Identifier, partitions_count: u32, ) -> Result<(), IggyError> { - self.ensure_topic_exists(stream_id, topic_id)?; - let actual_partitions_count = - self.streams - .with_partitions(stream_id, topic_id, |partitions| partitions.len()); + let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?; + let actual_partitions_count = self.metadata.partitions_count(stream, topic); if partitions_count > actual_partitions_count as u32 { return Err(IggyError::InvalidPartitionsCount); @@ -97,21 +122,7 @@ impl IggyShard { topic_id: &Identifier, partition_id: usize, ) -> Result<(), IggyError> { - self.ensure_topic_exists(stream_id, topic_id)?; - let partition_exists = self - .streams - .with_topic_by_id(stream_id, topic_id, |(root, ..)| { - root.partitions().exists(partition_id) - }); - - if !partition_exists { - return Err(IggyError::PartitionNotFound( - partition_id, - topic_id.clone(), - stream_id.clone(), - )); - } - + self.resolve_partition_id(stream_id, topic_id, partition_id)?; Ok(()) } @@ -138,29 +149,37 @@ impl IggyShard { return Err(IggyError::StaleClient); } - self.ensure_consumer_group_exists(stream_id, topic_id, &consumer.id)?; - let cg_id = self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - &consumer.id, - topics::helpers::get_consumer_group_id(), - ); - let Some(member_id) = self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - &consumer.id, - topics::helpers::get_consumer_group_member_id(client_id), - ) else { - // Client might have been removed between check above and here - if self.client_manager.try_get_client(client_id).is_none() { - return Err(IggyError::StaleClient); - } - return Err(IggyError::ConsumerGroupMemberNotFound( - client_id, - consumer.id.clone(), - topic_id.clone(), - )); - }; + let (stream, topic, cg_id) = + self.resolve_consumer_group_id(stream_id, topic_id, &consumer.id)?; + + let metadata = self.metadata.load(); + let cg_meta = metadata + .streams + .get(stream) + .and_then(|s| s.topics.get(topic)) + .and_then(|t| t.consumer_groups.get(cg_id)) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound(consumer.id.clone(), topic_id.clone()) + })?; + + let (member_id, _member) = cg_meta + .members + .iter() + .find(|(_, m)| m.client_id == client_id) + .ok_or_else(|| { + // Client might have been removed between check above and here + if self.client_manager.try_get_client(client_id).is_none() { + return IggyError::StaleClient; + } + IggyError::ConsumerGroupMemberNotFound( + client_id, + consumer.id.clone(), + topic_id.clone(), + ) + })?; + + drop(metadata); + if let Some(partition_id) = partition_id { return Ok(Some(( PollingConsumer::consumer_group(cg_id, member_id), @@ -168,29 +187,21 @@ impl IggyShard { ))); } - let partition_id = if calculate_partition_id { - self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - &consumer.id, - topics::helpers::calculate_partition_id_unchecked(member_id), - ) - } else { - self.streams.with_consumer_group_by_id( - stream_id, - topic_id, - &consumer.id, - topics::helpers::get_current_partition_id_unchecked(member_id), - ) - }; - let Some(partition_id) = partition_id else { - return Ok(None); - }; + let partition_id = self.metadata.get_next_member_partition_id( + stream, + topic, + cg_id, + member_id, + calculate_partition_id, + ); - Ok(Some(( - PollingConsumer::consumer_group(cg_id, member_id), - partition_id, - ))) + match partition_id { + Some(partition_id) => Ok(Some(( + PollingConsumer::consumer_group(cg_id, member_id), + partition_id, + ))), + None => Ok(None), + } } } } diff --git a/core/server/src/shard/tasks/periodic/message_cleaner.rs b/core/server/src/shard/tasks/periodic/message_cleaner.rs index 54f6f0f10a..8ee8538207 100644 --- a/core/server/src/shard/tasks/periodic/message_cleaner.rs +++ b/core/server/src/shard/tasks/periodic/message_cleaner.rs @@ -17,8 +17,8 @@ */ use crate::shard::IggyShard; -use crate::streaming::topics::helpers as topics_helpers; -use iggy_common::{Identifier, IggyError, IggyTimestamp}; +use crate::shard::namespace::IggyNamespace; +use iggy_common::{IggyError, IggyTimestamp, MaxTopicSize}; use std::rc::Rc; use tracing::{debug, error, info, trace, warn}; @@ -70,22 +70,13 @@ async fn clean_expired_messages(shard: Rc) -> Result<(), IggyError> { let mut total_deleted_messages = 0u64; for ((stream_id, topic_id), partition_ids) in topics { - let stream_identifier = Identifier::numeric(stream_id as u32).unwrap(); - let topic_identifier = Identifier::numeric(topic_id as u32).unwrap(); - let mut topic_deleted_segments = 0u64; let mut topic_deleted_messages = 0u64; for partition_id in partition_ids { // Handle expired segments - let expired_result = handle_expired_segments( - &shard, - &stream_identifier, - &topic_identifier, - partition_id, - now, - ) - .await; + let expired_result = + handle_expired_segments(&shard, stream_id, topic_id, partition_id, now).await; match expired_result { Ok(deleted) => { @@ -102,13 +93,8 @@ async fn clean_expired_messages(shard: Rc) -> Result<(), IggyError> { // Handle oldest segments if topic size management is enabled if delete_oldest_segments { - let oldest_result = handle_oldest_segments( - &shard, - &stream_identifier, - &topic_identifier, - partition_id, - ) - .await; + let oldest_result = + handle_oldest_segments(&shard, stream_id, topic_id, partition_id).await; match oldest_result { Ok(deleted) => { @@ -164,24 +150,27 @@ struct DeletedSegments { async fn handle_expired_segments( shard: &Rc, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, partition_id: usize, now: IggyTimestamp, ) -> Result { - // Get expired segments - let expired_segment_offsets = - shard - .streams - .with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - let mut expired = Vec::new(); - for segment in log.segments() { - if segment.is_expired(now) { - expired.push(segment.start_offset); - } - } - expired - }); + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + // Scope the borrow to avoid holding across await + let expired_segment_offsets: Vec = { + let store = shard.partition_store.borrow(); + let Some(partition_data) = store.get(&ns) else { + return Ok(DeletedSegments::default()); + }; + partition_data + .log + .segments() + .iter() + .filter(|segment| segment.is_expired(now)) + .map(|segment| segment.start_offset) + .collect() + }; if expired_segment_offsets.is_empty() { return Ok(DeletedSegments::default()); @@ -207,16 +196,21 @@ async fn handle_expired_segments( async fn handle_oldest_segments( shard: &Rc, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, partition_id: usize, ) -> Result { - let topic_info = - shard - .streams - .with_topic_by_id(stream_id, topic_id, topics_helpers::get_topic_size_info()); + let metadata = shard.metadata.load(); + let Some(topic_meta) = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + else { + return Ok(DeletedSegments::default()); + }; - let (is_unlimited, is_almost_full) = topic_info; + let max_size = topic_meta.max_topic_size; + let is_unlimited = matches!(max_size, MaxTopicSize::Unlimited); if is_unlimited { debug!( @@ -226,6 +220,13 @@ async fn handle_oldest_segments( return Ok(DeletedSegments::default()); } + // Get current size from stats + let current_size = topic_meta.stats.size_bytes_inconsistent(); + let max_bytes = max_size.as_bytes_u64(); + // Consider "almost full" as 90% capacity + let is_almost_full = current_size >= (max_bytes * 9 / 10); + drop(metadata); + if !is_almost_full { debug!( "Topic is not almost full, oldest segments will not be deleted for stream ID: {}, topic ID: {}, partition ID: {}", @@ -234,19 +235,22 @@ async fn handle_oldest_segments( return Ok(DeletedSegments::default()); } - let oldest_segment_offset = - shard - .streams - .with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - let segments = log.segments(); - // Find the first closed segment (not the active one) - if segments.len() > 1 { - // The last segment is always active, so we look at earlier ones - segments.first().map(|s| s.start_offset) - } else { - None - } - }); + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + // Scope the borrow to avoid holding across await + let oldest_segment_offset = { + let store = shard.partition_store.borrow(); + store.get(&ns).and_then(|partition_data| { + let segments = partition_data.log.segments(); + // Find the first closed segment (not the active one) + if segments.len() > 1 { + // The last segment is always active, so we look at earlier ones + segments.first().map(|s| s.start_offset) + } else { + None + } + }) + }; if let Some(start_offset) = oldest_segment_offset { info!( @@ -266,8 +270,8 @@ async fn handle_oldest_segments( async fn delete_segments( shard: &Rc, - stream_id: &Identifier, - topic_id: &Identifier, + stream_id: usize, + topic_id: usize, partition_id: usize, segment_offsets: &[u64], ) -> Result { @@ -286,40 +290,46 @@ async fn delete_segments( let mut segments_count = 0u64; let mut messages_count = 0u64; - let to_delete = shard.streams.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - |(_, stats, .., log)| { - let mut segments_to_remove = Vec::new(); - let mut storages_to_remove = Vec::new(); - - let mut indices_to_remove: Vec = Vec::new(); - for &start_offset in segment_offsets { - if let Some(idx) = log - .segments() - .iter() - .position(|s| s.start_offset == start_offset) - { - indices_to_remove.push(idx); - } - } + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - indices_to_remove.sort_by(|a, b| b.cmp(a)); - for idx in indices_to_remove { - let segment = log.segments_mut().remove(idx); - let storage = log.storages_mut().remove(idx); - log.indexes_mut().remove(idx); + // Extract segments and storages to delete from partition_store + let (stats, segments_to_delete, mut storages_to_delete) = { + let mut store = shard.partition_store.borrow_mut(); + let Some(partition_data) = store.get_mut(&ns) else { + return Ok(DeletedSegments::default()); + }; - segments_to_remove.push(segment); - storages_to_remove.push(storage); + let log = &mut partition_data.log; + let mut segments_to_remove = Vec::new(); + let mut storages_to_remove = Vec::new(); + + let mut indices_to_remove: Vec = Vec::new(); + for &start_offset in segment_offsets { + if let Some(idx) = log + .segments() + .iter() + .position(|s| s.start_offset == start_offset) + { + indices_to_remove.push(idx); } + } - (stats.clone(), segments_to_remove, storages_to_remove) - }, - ); + indices_to_remove.sort_by(|a, b| b.cmp(a)); + for idx in indices_to_remove { + let segment = log.segments_mut().remove(idx); + let storage = log.storages_mut().remove(idx); + log.indexes_mut().remove(idx); + + segments_to_remove.push(segment); + storages_to_remove.push(storage); + } - let (stats, segments_to_delete, mut storages_to_delete) = to_delete; + ( + partition_data.stats.clone(), + segments_to_remove, + storages_to_remove, + ) + }; for (segment, storage) in segments_to_delete .into_iter() diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs index c6d0efdcf6..3b4bb11812 100644 --- a/core/server/src/shard/tasks/periodic/message_saver.rs +++ b/core/server/src/shard/tasks/periodic/message_saver.rs @@ -17,7 +17,7 @@ */ use crate::shard::IggyShard; -use iggy_common::{Identifier, IggyError}; +use iggy_common::IggyError; use std::rc::Rc; use tracing::{error, info, trace}; @@ -46,45 +46,35 @@ async fn save_messages(shard: Rc) -> Result<(), IggyError> { trace!("Saving buffered messages..."); let namespaces = shard.get_current_shard_namespaces(); - let mut total_saved_messages = 0u32; - const REASON: &str = "background saver triggered"; + let mut total_saved_messages = 0u64; + let mut partitions_flushed = 0u32; for ns in namespaces { - let stream_id = Identifier::numeric(ns.stream_id() as u32).unwrap(); - let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap(); - let partition_id = ns.partition_id(); - - match shard - .streams - .persist_messages( - &stream_id, - &topic_id, - partition_id, - REASON, - &shard.config.system, - ) - .await - { - Ok(batch_count) => { - total_saved_messages += batch_count; - } - Err(err) => { - error!( - "Failed to save messages for partition {}: {}", - partition_id, err - ); + if shard.partition_store.borrow().get(&ns).is_some() { + match shard + .flush_unsaved_buffer_from_partition_store(&ns, false) + .await + { + Ok(saved) => { + if saved > 0 { + total_saved_messages += saved as u64; + partitions_flushed += 1; + } + } + Err(err) => { + error!("Failed to save messages for partition {:?}: {}", ns, err); + } } } } if total_saved_messages > 0 { - info!("Saved {} buffered messages on disk.", total_saved_messages); + info!("Saved {total_saved_messages} messages from {partitions_flushed} partitions."); } Ok(()) } async fn fsync_all_segments_on_shutdown(shard: Rc, result: Result<(), IggyError>) { - // Only fsync if the last save_messages tick succeeded if result.is_err() { error!( "Last save_messages tick failed, skipping fsync: {:?}", @@ -98,26 +88,20 @@ async fn fsync_all_segments_on_shutdown(shard: Rc, result: Result<(), let namespaces = shard.get_current_shard_namespaces(); for ns in namespaces { - let stream_id = Identifier::numeric(ns.stream_id() as u32).unwrap(); - let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap(); - let partition_id = ns.partition_id(); - - match shard - .streams - .fsync_all_messages(&stream_id, &topic_id, partition_id) - .await - { - Ok(()) => { - trace!( - "Successfully fsynced segment for stream: {}, topic: {}, partition: {} during shutdown", - stream_id, topic_id, partition_id - ); - } - Err(err) => { - error!( - "Failed to fsync segment for stream: {}, topic: {}, partition: {} during shutdown: {}", - stream_id, topic_id, partition_id, err - ); + if shard.partition_store.borrow().get(&ns).is_some() { + match shard.fsync_all_messages_from_partition_store(&ns).await { + Ok(()) => { + trace!( + "Successfully fsynced segment for partition {:?} during shutdown", + ns + ); + } + Err(err) => { + error!( + "Failed to fsync segment for partition {:?} during shutdown: {}", + ns, err + ); + } } } } diff --git a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs index 3c1203a9aa..10d6ca4be7 100644 --- a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs +++ b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs @@ -19,7 +19,6 @@ use crate::shard::IggyShard; use iggy_common::{IggyError, IggyTimestamp}; use std::rc::Rc; -use std::sync::Arc; use tracing::{info, trace}; pub fn spawn_personal_access_token_cleaner(shard: Rc) { @@ -48,23 +47,34 @@ async fn clear_personal_access_tokens(shard: Rc) -> Result<(), IggyEr let now = IggyTimestamp::now(); let mut total_removed = 0; - let users = shard.users.values(); - for user in &users { - let expired_tokens: Vec> = user - .personal_access_tokens + let user_ids: Vec = shard + .metadata + .get_all_users() + .iter() + .map(|u| u.id) + .collect(); + + for user_id in user_ids { + // Get PATs for this user + let pats = shard.metadata.get_user_personal_access_tokens(user_id); + + // Find expired PATs + let expired_tokens: Vec<_> = pats .iter() - .filter(|entry| entry.value().is_expired(now)) - .map(|entry| entry.key().clone()) + .filter(|pat| pat.is_expired(now)) + .map(|pat| (pat.name.clone(), pat.token.clone())) .collect(); - for token_hash in expired_tokens { - if let Some((_, pat)) = user.personal_access_tokens.remove(&token_hash) { - info!( - "Removed expired personal access token '{}' for user ID {}", - pat.name, user.id - ); - total_removed += 1; - } + // Delete expired PATs via SharedMetadata + for (name, token_hash) in expired_tokens { + shard + .metadata + .delete_personal_access_token(user_id, &token_hash); + info!( + "Removed expired personal access token '{}' for user ID {}", + name, user_id + ); + total_removed += 1; } } diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs index 1180d06f99..d212b7f37c 100644 --- a/core/server/src/shard/transmission/event.rs +++ b/core/server/src/shard/transmission/event.rs @@ -15,121 +15,54 @@ // specific language governing permissions and limitations // under the License. -use crate::streaming::{ - partitions::partition, - streams::stream, - topics::{ - consumer_group::{self}, - topic, - }, -}; -use iggy_common::{ - CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, Permissions, PersonalAccessToken, - TransportProtocol, UserStatus, -}; +use iggy_common::{Identifier, IggyTimestamp, TransportProtocol}; use std::net::SocketAddr; use strum::Display; -#[allow(clippy::large_enum_variant)] +/// Minimal partition info for event broadcasting (no slab dependency) +#[derive(Debug, Clone)] +pub struct PartitionInfo { + pub id: usize, + pub created_at: IggyTimestamp, +} + +/// Events that require broadcasting between shards. +/// +/// Note: Metadata events (CreatedStream, DeletedStream, CreatedTopic, DeletedTopic, +/// UpdatedStream, UpdatedTopic, CreatedConsumerGroup, DeletedConsumerGroup) are NOT +/// broadcast because SharedMetadata is already visible to all shards via ArcSwap. +/// Only events that require per-shard local actions are broadcast. #[derive(Debug, Clone, Display)] #[strum(serialize_all = "PascalCase")] pub enum ShardEvent { + /// Flush unsaved buffer to disk for a specific partition FlushUnsavedBuffer { stream_id: Identifier, topic_id: Identifier, partition_id: usize, fsync: bool, }, - CreatedStream { - id: usize, - stream: stream::Stream, - }, - DeletedStream { - id: usize, - stream_id: Identifier, - }, - UpdatedStream { - stream_id: Identifier, - name: String, - }, - PurgedStream { + /// Purge all messages from a stream (requires per-shard log truncation) + PurgedStream { stream_id: Identifier }, + /// Purge all messages from a topic (requires per-shard log truncation) + PurgedTopic { stream_id: Identifier, + topic_id: Identifier, }, + /// New partitions created (requires per-shard log initialization) CreatedPartitions { stream_id: Identifier, topic_id: Identifier, - partitions: Vec, + partitions: Vec, }, + /// Partitions deleted (requires per-shard log cleanup) DeletedPartitions { stream_id: Identifier, topic_id: Identifier, partitions_count: u32, partition_ids: Vec, }, - CreatedTopic { - stream_id: Identifier, - topic: topic::Topic, - }, - CreatedConsumerGroup { - stream_id: Identifier, - topic_id: Identifier, - cg: consumer_group::ConsumerGroup, - }, - DeletedConsumerGroup { - id: usize, - stream_id: Identifier, - topic_id: Identifier, - group_id: Identifier, - }, - UpdatedTopic { - stream_id: Identifier, - topic_id: Identifier, - name: String, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: Option, - }, - PurgedTopic { - stream_id: Identifier, - topic_id: Identifier, - }, - DeletedTopic { - id: usize, - stream_id: Identifier, - topic_id: Identifier, - }, - CreatedUser { - user_id: u32, - username: String, - password: String, - status: UserStatus, - permissions: Option, - }, - UpdatedPermissions { - user_id: Identifier, - permissions: Option, - }, - DeletedUser { - user_id: Identifier, - }, - UpdatedUser { - user_id: Identifier, - username: Option, - status: Option, - }, - ChangedPassword { - user_id: Identifier, - current_password: String, - new_password: String, - }, - CreatedPersonalAccessToken { - personal_access_token: PersonalAccessToken, - }, - DeletedPersonalAccessToken { - user_id: u32, - name: String, - }, + /// Transport address bound (for config file writing) AddressBound { protocol: TransportProtocol, address: SocketAddr, diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs index 7c0fd25784..dc0a23b2ff 100644 --- a/core/server/src/shard/transmission/frame.rs +++ b/core/server/src/shard/transmission/frame.rs @@ -21,12 +21,9 @@ use iggy_common::{IggyError, Stats}; use crate::{ binary::handlers::messages::poll_messages_handler::IggyPollMetadata, shard::transmission::message::ShardMessage, - streaming::{ - segments::IggyMessagesBatchSet, streams::stream, topics::topic, users::user::User, - }, + streaming::{segments::IggyMessagesBatchSet, users::user::User}, }; -#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum ShardResponse { PollMessages((IggyPollMetadata, IggyMessagesBatchSet)), @@ -34,15 +31,25 @@ pub enum ShardResponse { FlushUnsavedBuffer, DeleteSegments, Event, - CreateStreamResponse(stream::Stream), - DeleteStreamResponse(stream::Stream), - CreateTopicResponse(topic::Topic), + /// Stream ID + CreateStreamResponse(usize), + /// Stream ID + DeleteStreamResponse(usize), + /// Topic ID + CreateTopicResponse(usize), UpdateTopicResponse, - DeleteTopicResponse(topic::Topic), + /// Topic ID + DeleteTopicResponse(usize), CreateUserResponse(User), DeletedUser(User), GetStatsResponse(Stats), + /// Partition IDs + CreatePartitionsResponse(Vec), + DeletePartitionsResponse(Vec), + UpdateStreamResponse, SocketTransferResponse, + UpdatePermissionsResponse, + ChangePasswordResponse, ErrorResponse(IggyError), } diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs index fec8bf4735..4eae9562ee 100644 --- a/core/server/src/shard/transmission/message.rs +++ b/core/server/src/shard/transmission/message.rs @@ -20,7 +20,6 @@ use crate::{ system::messages::PollingArgs, transmission::{event::ShardEvent, frame::ShardResponse}, }, - slab::partitions, streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut}, }; use iggy_common::{ @@ -55,7 +54,7 @@ impl ShardRequest { pub fn new( stream_id: Identifier, topic_id: Identifier, - partition_id: partitions::ContainerId, + partition_id: usize, payload: ShardRequestPayload, ) -> Self { Self { @@ -67,6 +66,7 @@ impl ShardRequest { } } +// cleanup this shit #[derive(Debug)] pub enum ShardRequestPayload { SendMessages { @@ -129,6 +129,23 @@ pub enum ShardRequestPayload { DeleteSegments { segments_count: u32, }, + CreatePartitions { + user_id: u32, + stream_id: Identifier, + topic_id: Identifier, + partitions_count: u32, + }, + DeletePartitions { + user_id: u32, + stream_id: Identifier, + topic_id: Identifier, + partitions_count: u32, + }, + UpdateStream { + user_id: u32, + stream_id: Identifier, + name: String, + }, SocketTransfer { fd: OwnedFd, from_shard: u16, @@ -137,6 +154,17 @@ pub enum ShardRequestPayload { address: SocketAddr, initial_data: IggyMessagesBatchMut, }, + UpdatePermissions { + session_user_id: u32, + user_id: Identifier, + permissions: Option, + }, + ChangePassword { + session_user_id: u32, + user_id: Identifier, + current_password: String, + new_password: String, + }, } impl From for ShardMessage { diff --git a/core/server/src/slab/consumer_groups.rs b/core/server/src/slab/consumer_groups.rs deleted file mode 100644 index 618ecb1568..0000000000 --- a/core/server/src/slab/consumer_groups.rs +++ /dev/null @@ -1,200 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - slab::{ - Keyed, consumer_groups, - traits_ext::{ - Borrow, ComponentsById, Delete, EntityComponentSystem, EntityComponentSystemMut, - Insert, IntoComponents, - }, - }, - streaming::topics::consumer_group::{self, ConsumerGroupRef, ConsumerGroupRefMut}, -}; -use ahash::AHashMap; -use iggy_common::Identifier; -use slab::Slab; - -const CAPACITY: usize = 1024; -pub type ContainerId = usize; - -#[derive(Debug, Clone)] -pub struct ConsumerGroups { - index: AHashMap<::Key, usize>, - members: Slab, - root: Slab, -} - -impl Insert for ConsumerGroups { - type Idx = consumer_groups::ContainerId; - type Item = consumer_group::ConsumerGroup; - - fn insert(&mut self, item: Self::Item) -> Self::Idx { - let (root, members) = item.into_components(); - let key = root.key().clone(); - - let entity_id = self.root.insert(root); - let id = self.members.insert(members); - assert_eq!( - entity_id, id, - "consumer_group: id mismatch when inserting members" - ); - self.index.insert(key, entity_id); - let root = self.root.get_mut(entity_id).unwrap(); - root.update_id(entity_id); - entity_id - } -} - -impl Delete for ConsumerGroups { - type Idx = consumer_groups::ContainerId; - type Item = consumer_group::ConsumerGroup; - - fn delete(&mut self, id: Self::Idx) -> Self::Item { - let root = self.root.remove(id); - let members = self.members.remove(id); - self.index - .remove(root.key()) - .expect("consumer_group_delete: key not found"); - consumer_group::ConsumerGroup::new_with_components(root, members) - } -} - -//TODO: those from impls could use a macro aswell. -impl<'a> From<&'a ConsumerGroups> for consumer_group::ConsumerGroupRef<'a> { - fn from(value: &'a ConsumerGroups) -> Self { - consumer_group::ConsumerGroupRef::new(&value.root, &value.members) - } -} - -impl<'a> From<&'a mut ConsumerGroups> for consumer_group::ConsumerGroupRefMut<'a> { - fn from(value: &'a mut ConsumerGroups) -> Self { - consumer_group::ConsumerGroupRefMut::new(&mut value.root, &mut value.members) - } -} - -impl EntityComponentSystem for ConsumerGroups { - type Idx = consumer_groups::ContainerId; - type Entity = consumer_group::ConsumerGroup; - type EntityComponents<'a> = consumer_group::ConsumerGroupRef<'a>; - - fn with_components(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponents<'a>) -> O, - { - f(self.into()) - } -} - -impl EntityComponentSystemMut for ConsumerGroups { - type EntityComponentsMut<'a> = consumer_group::ConsumerGroupRefMut<'a>; - - fn with_components_mut(&mut self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O, - { - f(self.into()) - } -} - -impl ConsumerGroups { - pub fn exists(&self, id: &Identifier) -> bool { - match id.kind { - iggy_common::IdKind::Numeric => { - let id = id.get_u32_value().unwrap() as usize; - self.root.contains(id) - } - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - self.index.contains_key(&key) - } - } - } - - pub fn len(&self) -> usize { - self.root.len() - } - - pub fn is_empty(&self) -> bool { - self.root.is_empty() - } - - pub fn get_index(&self, id: &Identifier) -> usize { - match id.kind { - iggy_common::IdKind::Numeric => id.get_u32_value().unwrap() as usize, - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - *self.index.get(&key).expect("Consumer Group not found") - } - } - } - - pub fn with_consumer_group_by_id( - &self, - identifier: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(identifier); - self.with_components_by_id(id, |components| f(components)) - } - - pub fn with_consumer_group_by_id_mut( - &mut self, - identifier: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(identifier); - self.with_components_by_id_mut(id, |components| f(components)) - } -} - -impl Default for ConsumerGroups { - fn default() -> Self { - Self { - index: AHashMap::with_capacity(CAPACITY), - root: Slab::with_capacity(CAPACITY), - members: Slab::with_capacity(CAPACITY), - } - } -} - -impl ConsumerGroups { - /// Construct from pre-built entries with specific IDs. - pub fn from_entries( - entries: impl IntoIterator, - ) -> Self { - let entries: Vec<_> = entries.into_iter().collect(); - - let mut index = AHashMap::with_capacity(entries.len()); - let mut root_entries = Vec::with_capacity(entries.len()); - let mut members_entries = Vec::with_capacity(entries.len()); - - for (id, cg) in entries { - let (mut root, members) = cg.into_components(); - root.update_id(id); - index.insert(root.key().clone(), id); - root_entries.push((id, root)); - members_entries.push((id, members)); - } - - Self { - index, - root: root_entries.into_iter().collect(), - members: members_entries.into_iter().collect(), - } - } -} diff --git a/core/server/src/slab/helpers.rs b/core/server/src/slab/helpers.rs deleted file mode 100644 index 2d977c7efc..0000000000 --- a/core/server/src/slab/helpers.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - slab::{ - consumer_groups::ConsumerGroups, partitions::Partitions, topics::Topics, - traits_ext::ComponentsById, - }, - streaming::{ - streams::stream::StreamRef, - topics::topic::{TopicRef, TopicRefMut}, - }, -}; - -// Helpers -pub fn topics(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a Topics) -> O, -{ - |(root, ..)| f(root.topics()) -} - -pub fn topics_mut(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a Topics) -> O, -{ - |(root, ..)| f(root.topics()) -} - -pub fn partitions(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a Partitions) -> O, -{ - |(root, ..)| f(root.partitions()) -} - -pub fn partitions_mut(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a mut Partitions) -> O, -{ - |(mut root, ..)| f(root.partitions_mut()) -} - -pub fn consumer_groups(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a ConsumerGroups) -> O, -{ - |(root, ..)| f(root.consumer_groups()) -} - -pub fn consumer_groups_mut(f: F) -> impl FnOnce(ComponentsById) -> O -where - F: for<'a> FnOnce(&'a mut ConsumerGroups) -> O, -{ - |(mut root, ..)| f(root.consumer_groups_mut()) -} diff --git a/core/server/src/slab/mod.rs b/core/server/src/slab/mod.rs deleted file mode 100644 index 68659c90c1..0000000000 --- a/core/server/src/slab/mod.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod consumer_groups; -pub mod helpers; -pub mod partitions; -pub mod streams; -pub mod topics; -pub mod traits_ext; -pub mod users; - -use std::fmt::{Debug, Display}; - -// General rules how to implement `with_*` methods on any slab" -// 1. When implementing method that accepts closure f, make sure that the caller can supply closure only with 1 depth of callbacks. -// for example, observe following code snippet: -// ```rust -// let topic_id = self.streams.with_topic_by_id(stream_id, topic_id, get_topic_id()); -// ``` -// if we would not provide a `with_topic_by_id` method and purely relied only ony `with_topics`, we would have to write: -// ```rust -// let topic_id = self.streams.with_topics(stream_id, get_topic_by_id(topic_id, get_topic_id())); // `get_topic_id` is a closure that retrieves the topic id. -// ``` -// we need to supply a nested closure to `get_topic_by_id`. - -pub trait Keyed { - type Key: Eq + std::hash::Hash + Clone + Debug + Display; - fn key(&self) -> &Self::Key; -} diff --git a/core/server/src/slab/partitions.rs b/core/server/src/slab/partitions.rs deleted file mode 100644 index 214ffc6b16..0000000000 --- a/core/server/src/slab/partitions.rs +++ /dev/null @@ -1,281 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - slab::traits_ext::{ - Borrow, ComponentsById, Delete, EntityComponentSystem, EntityComponentSystemMut, Insert, - IntoComponents, - }, - streaming::{ - deduplication::message_deduplicator::MessageDeduplicator, - partitions::{ - journal::MemoryMessageJournal, - log::SegmentedLog, - partition::{ - self, ConsumerGroupOffsets, ConsumerOffsets, Partition, PartitionRef, - PartitionRefMut, - }, - }, - stats::PartitionStats, - }, -}; -use slab::Slab; -use std::sync::{Arc, atomic::AtomicU64}; - -// TODO: This could be upper limit of partitions per topic, use that value to validate instead of whathever this thing is in `common` crate. -pub const PARTITIONS_CAPACITY: usize = 16384; -pub type ContainerId = usize; - -#[derive(Debug)] -pub struct Partitions { - root: Slab, - stats: Slab>, - message_deduplicator: Slab>>, - offset: Slab>, - - consumer_offset: Slab>, - consumer_group_offset: Slab>, - - log: Slab>, -} - -/// Clone implementation for partitions, does not copy the actual logs. -/// Since those are very expensive to clone and we use `Clone` only during initialization -/// in order to streamline broadcasting entity creation event to other shards. -/// A better strategy would be to have an `Pool` of Streams/Topics/Partitions and during event broadcast, grab a new `Default` instance from the pool aka ZII. -impl Clone for Partitions { - fn clone(&self) -> Self { - Self { - root: self.root.clone(), - stats: self.stats.clone(), - message_deduplicator: self.message_deduplicator.clone(), - offset: self.offset.clone(), - consumer_offset: self.consumer_offset.clone(), - consumer_group_offset: self.consumer_group_offset.clone(), - log: Slab::with_capacity(PARTITIONS_CAPACITY), // Empty log, we don't clone the actual logs. - } - } -} - -impl Insert for Partitions { - type Idx = ContainerId; - type Item = Partition; - - fn insert(&mut self, item: Self::Item) -> Self::Idx { - let (root, stats, deduplicator, offset, consumer_offset, consumer_group_offset, log) = - item.into_components(); - - let entity_id = self.root.insert(root); - let id = self.stats.insert(stats); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating stats" - ); - let id = self.log.insert(log); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating log" - ); - let id = self.message_deduplicator.insert(deduplicator); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating message_deduplicator" - ); - let id = self.offset.insert(offset); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating offset" - ); - let id = self.consumer_offset.insert(consumer_offset); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating consumer_offset" - ); - let id = self.consumer_group_offset.insert(consumer_group_offset); - assert_eq!( - entity_id, id, - "partition_insert: id mismatch when creating consumer_group_offset" - ); - let root = self.root.get_mut(entity_id).unwrap(); - root.update_id(entity_id); - entity_id - } -} - -impl Delete for Partitions { - type Idx = ContainerId; - type Item = Partition; - - fn delete(&mut self, id: Self::Idx) -> Self::Item { - let root = self.root.remove(id); - let stats = self.stats.remove(id); - let message_deduplicator = self.message_deduplicator.remove(id); - let offset = self.offset.remove(id); - let consumer_offset = self.consumer_offset.remove(id); - let consumer_group_offset = self.consumer_group_offset.remove(id); - let log = self.log.remove(id); - - Partition::new_with_components( - root, - stats, - message_deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - ) - } -} - -//TODO: those from impls could use a macro aswell. -impl<'a> From<&'a Partitions> for PartitionRef<'a> { - fn from(value: &'a Partitions) -> Self { - PartitionRef::new( - &value.root, - &value.stats, - &value.message_deduplicator, - &value.offset, - &value.consumer_offset, - &value.consumer_group_offset, - &value.log, - ) - } -} - -impl<'a> From<&'a mut Partitions> for PartitionRefMut<'a> { - fn from(value: &'a mut Partitions) -> Self { - PartitionRefMut::new( - &mut value.root, - &mut value.stats, - &mut value.message_deduplicator, - &mut value.offset, - &mut value.consumer_offset, - &mut value.consumer_group_offset, - &mut value.log, - ) - } -} - -impl EntityComponentSystem for Partitions { - type Idx = ContainerId; - type Entity = Partition; - type EntityComponents<'a> = PartitionRef<'a>; - - fn with_components(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponents<'a>) -> O, - { - f(self.into()) - } -} - -impl EntityComponentSystemMut for Partitions { - type EntityComponentsMut<'a> = PartitionRefMut<'a>; - - fn with_components_mut(&mut self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O, - { - f(self.into()) - } -} - -impl Default for Partitions { - fn default() -> Self { - Self { - root: Slab::with_capacity(PARTITIONS_CAPACITY), - stats: Slab::with_capacity(PARTITIONS_CAPACITY), - log: Slab::with_capacity(PARTITIONS_CAPACITY), - message_deduplicator: Slab::with_capacity(PARTITIONS_CAPACITY), - offset: Slab::with_capacity(PARTITIONS_CAPACITY), - consumer_offset: Slab::with_capacity(PARTITIONS_CAPACITY), - consumer_group_offset: Slab::with_capacity(PARTITIONS_CAPACITY), - } - } -} - -impl Partitions { - /// Construct from pre-built entries with specific IDs. - pub fn from_entries(entries: impl IntoIterator) -> Self { - let entries: Vec<_> = entries.into_iter().collect(); - - let mut root_entries = Vec::with_capacity(entries.len()); - let mut stats_entries = Vec::with_capacity(entries.len()); - let mut dedup_entries = Vec::with_capacity(entries.len()); - let mut offset_entries = Vec::with_capacity(entries.len()); - let mut consumer_offset_entries = Vec::with_capacity(entries.len()); - let mut consumer_group_offset_entries = Vec::with_capacity(entries.len()); - let mut log_entries = Vec::with_capacity(entries.len()); - - for (id, partition) in entries { - let (mut root, stats, dedup, offset, consumer_offset, consumer_group_offset, log) = - partition.into_components(); - root.update_id(id); - root_entries.push((id, root)); - stats_entries.push((id, stats)); - dedup_entries.push((id, dedup)); - offset_entries.push((id, offset)); - consumer_offset_entries.push((id, consumer_offset)); - consumer_group_offset_entries.push((id, consumer_group_offset)); - log_entries.push((id, log)); - } - - Self { - root: root_entries.into_iter().collect(), - stats: stats_entries.into_iter().collect(), - message_deduplicator: dedup_entries.into_iter().collect(), - offset: offset_entries.into_iter().collect(), - consumer_offset: consumer_offset_entries.into_iter().collect(), - consumer_group_offset: consumer_group_offset_entries.into_iter().collect(), - log: log_entries.into_iter().collect(), - } - } -} - -impl Partitions { - pub fn len(&self) -> usize { - self.root.len() - } - - pub fn is_empty(&self) -> bool { - self.root.is_empty() - } - - pub fn insert_default_log(&mut self) -> ContainerId { - self.log.insert(Default::default()) - } - - pub fn with_partition_by_id( - &self, - id: ContainerId, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_components_by_id(id, |components| f(components)) - } - - pub fn exists(&self, id: ContainerId) -> bool { - self.root.contains(id) - } - - pub fn with_partition_by_id_mut( - &mut self, - id: ContainerId, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_components_by_id_mut(id, |components| f(components)) - } -} diff --git a/core/server/src/slab/streams.rs b/core/server/src/slab/streams.rs deleted file mode 100644 index 66e5c7cbb1..0000000000 --- a/core/server/src/slab/streams.rs +++ /dev/null @@ -1,1522 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::task_registry::TaskRegistry; -use crate::streaming::partitions as streaming_partitions; -use crate::streaming::partitions::consumer_offset::ConsumerOffset; -use crate::streaming::stats::StreamStats; -use crate::{ - binary::handlers::messages::poll_messages_handler::IggyPollMetadata, - configs::{cache_indexes::CacheIndexesConfig, system::SystemConfig}, - shard::{namespace::IggyFullNamespace, system::messages::PollingArgs}, - slab::{ - Keyed, - consumer_groups::ConsumerGroups, - helpers, - partitions::{self, Partitions}, - topics::Topics, - traits_ext::{ - ComponentsById, DeleteCell, EntityComponentSystem, EntityComponentSystemMutCell, - InsertCell, InteriorMutability, IntoComponents, - }, - }, - streaming::{ - partitions::{ - journal::Journal, - partition::{PartitionRef, PartitionRefMut}, - }, - polling_consumer::PollingConsumer, - segments::{ - IggyMessagesBatchMut, IggyMessagesBatchSet, Segment, storage::create_segment_storage, - }, - streams::{ - self, - stream::{self, StreamRef, StreamRefMut}, - }, - topics::{ - self, - consumer_group::{ConsumerGroupRef, ConsumerGroupRefMut}, - topic::{TopicRef, TopicRefMut}, - }, - traits::MainOps, - }, -}; -use ahash::AHashMap; -use err_trail::ErrContext; -use iggy_common::{Identifier, IggyError, IggyTimestamp, PollingKind}; -use slab::Slab; -use std::{ - cell::RefCell, - rc::Rc, - sync::{Arc, atomic::Ordering}, -}; -use tracing::error; - -const CAPACITY: usize = 1024; -pub type ContainerId = usize; - -#[derive(Debug, Clone)] -pub struct Streams { - index: RefCell::Key, ContainerId>>, - root: RefCell>, - stats: RefCell>>, -} - -impl Default for Streams { - fn default() -> Self { - Self { - index: RefCell::new(AHashMap::with_capacity(CAPACITY)), - root: RefCell::new(Slab::with_capacity(CAPACITY)), - stats: RefCell::new(Slab::with_capacity(CAPACITY)), - } - } -} - -impl Streams { - /// Construct from pre-built entries with specific IDs. - pub fn from_entries(entries: impl IntoIterator) -> Self { - let entries: Vec<_> = entries.into_iter().collect(); - - let mut index = AHashMap::with_capacity(entries.len()); - let mut root_entries = Vec::with_capacity(entries.len()); - let mut stats_entries = Vec::with_capacity(entries.len()); - - for (id, stream) in entries { - let (mut root, stats) = stream.into_components(); - root.update_id(id); - index.insert(root.key().clone(), id); - root_entries.push((id, root)); - stats_entries.push((id, stats)); - } - - Self { - index: RefCell::new(index), - root: RefCell::new(root_entries.into_iter().collect()), - stats: RefCell::new(stats_entries.into_iter().collect()), - } - } -} - -impl<'a> From<&'a Streams> for stream::StreamRef<'a> { - fn from(value: &'a Streams) -> Self { - let root = value.root.borrow(); - let stats = value.stats.borrow(); - stream::StreamRef::new(root, stats) - } -} - -impl<'a> From<&'a Streams> for stream::StreamRefMut<'a> { - fn from(value: &'a Streams) -> Self { - let root = value.root.borrow_mut(); - let stats = value.stats.borrow_mut(); - stream::StreamRefMut::new(root, stats) - } -} - -impl InsertCell for Streams { - type Idx = ContainerId; - type Item = stream::Stream; - - fn insert(&self, item: Self::Item) -> Self::Idx { - let (root, stats) = item.into_components(); - let mut root_container = self.root.borrow_mut(); - let mut indexes = self.index.borrow_mut(); - let mut stats_container = self.stats.borrow_mut(); - - let key = root.key().clone(); - let entity_id = root_container.insert(root); - let id = stats_container.insert(stats); - assert_eq!( - entity_id, id, - "stream_insert: id mismatch when inserting stats" - ); - let root = root_container.get_mut(entity_id).unwrap(); - root.update_id(entity_id); - indexes.insert(key, entity_id); - entity_id - } -} - -impl DeleteCell for Streams { - type Idx = ContainerId; - type Item = stream::Stream; - - fn delete(&self, id: Self::Idx) -> Self::Item { - let mut root_container = self.root.borrow_mut(); - let mut indexes = self.index.borrow_mut(); - let mut stats_container = self.stats.borrow_mut(); - - let root = root_container.remove(id); - let stats = stats_container.remove(id); - - // Remove from index - let key = root.key(); - indexes - .remove(key) - .expect("stream_delete: key not found in index"); - - stream::Stream::new_with_components(root, stats) - } -} - -impl EntityComponentSystem for Streams { - type Idx = ContainerId; - type Entity = stream::Stream; - type EntityComponents<'a> = stream::StreamRef<'a>; - - fn with_components(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponents<'a>) -> O, - { - f(self.into()) - } -} - -impl EntityComponentSystemMutCell for Streams { - type EntityComponentsMut<'a> = stream::StreamRefMut<'a>; - - fn with_components_mut(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O, - { - f(self.into()) - } -} - -impl MainOps for Streams { - type Namespace = IggyFullNamespace; - type PollingArgs = PollingArgs; - type Consumer = PollingConsumer; - type In = IggyMessagesBatchMut; - type Out = (IggyPollMetadata, IggyMessagesBatchSet); - type Error = IggyError; - - async fn append_messages( - &self, - config: &SystemConfig, - registry: &Rc, - ns: &Self::Namespace, - mut input: Self::In, - ) -> Result<(), Self::Error> { - let stream_id = ns.stream_id(); - let topic_id = ns.topic_id(); - let partition_id = ns.partition_id(); - - let current_offset = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::calculate_current_offset(), - ); - - let current_position = - self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - log.active_segment().current_position - }); - let (segment_start_offset, message_deduplicator) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::get_segment_start_offset_and_deduplicator(), - ); - - input - .prepare_for_persistence( - segment_start_offset, - current_offset, - current_position, - message_deduplicator.as_ref(), - ) - .await; - - let (journal_messages_count, journal_size) = self.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::append_to_journal(current_offset, input), - )?; - - let unsaved_messages_count_exceeded = - journal_messages_count >= config.partition.messages_required_to_save; - let unsaved_messages_size_exceeded = journal_size - >= config - .partition - .size_of_messages_required_to_save - .as_bytes_u64() as u32; - - let is_full = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::is_segment_full(), - ); - - // Try committing the journal - if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded { - let reason = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::persist_reason( - unsaved_messages_count_exceeded, - unsaved_messages_size_exceeded, - journal_messages_count, - journal_size, - config, - ), - ); - - let _batch_count = self - .persist_messages(stream_id, topic_id, partition_id, &reason, config) - .await?; - - if is_full { - self.handle_full_segment(registry, stream_id, topic_id, partition_id, config) - .await?; - } - } - Ok(()) - } - - async fn poll_messages( - &self, - ns: &Self::Namespace, - consumer: Self::Consumer, - args: Self::PollingArgs, - ) -> Result { - let stream_id = ns.stream_id(); - let topic_id = ns.topic_id(); - let partition_id = ns.partition_id(); - let current_offset = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, offset, ..)| offset.load(Ordering::Relaxed), - ); - let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); - let count = args.count; - let strategy = args.strategy; - let value = strategy.value; - let batches = match strategy.kind { - PollingKind::Offset => { - let offset = value; - // We have to remember to keep the invariant from the if that is on line 290. - // Alternatively a better design would be to get rid of that if and move the validations here. - if offset > current_offset { - return Ok((metadata, IggyMessagesBatchSet::default())); - } - - let batches = self - .get_messages_by_offset(stream_id, topic_id, partition_id, offset, count) - .await?; - Ok(batches) - } - PollingKind::Timestamp => { - let timestamp = IggyTimestamp::from(value); - let timestamp_ts = timestamp.as_micros(); - tracing::trace!( - "Getting {count} messages by timestamp: {} for partition: {}...", - timestamp_ts, - partition_id - ); - - let batches = self - .get_messages_by_timestamp( - stream_id, - topic_id, - partition_id, - timestamp_ts, - count, - ) - .await?; - Ok(batches) - } - PollingKind::First => { - let first_offset = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - log.segments() - .first() - .map(|segment| segment.start_offset) - .unwrap_or(0) - }, - ); - - let batches = self - .get_messages_by_offset(stream_id, topic_id, partition_id, first_offset, count) - .await?; - Ok(batches) - } - PollingKind::Last => { - let (start_offset, actual_count) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, offset, _, _, _)| { - let current_offset = offset.load(Ordering::Relaxed); - let mut requested_count = count as u64; - if requested_count > current_offset + 1 { - requested_count = current_offset + 1 - } - let start_offset = 1 + current_offset - requested_count; - (start_offset, requested_count as u32) - }, - ); - - let batches = self - .get_messages_by_offset( - stream_id, - topic_id, - partition_id, - start_offset, - actual_count, - ) - .await?; - Ok(batches) - } - PollingKind::Next => { - let consumer_offset = match consumer { - PollingConsumer::Consumer(consumer_id, _) => self - .with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::get_consumer_offset(consumer_id), - ) - .map(|c_offset| c_offset.stored_offset), - PollingConsumer::ConsumerGroup(consumer_group_id, _) => self - .with_partition_by_id( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::get_consumer_group_offset( - consumer_group_id, - ), - ) - .map(|cg_offset| cg_offset.stored_offset), - }; - - match consumer_offset { - None => { - let batches = self - .get_messages_by_offset(stream_id, topic_id, partition_id, 0, count) - .await?; - Ok(batches) - } - Some(consumer_offset) => { - let offset = consumer_offset + 1; - match consumer { - PollingConsumer::Consumer(consumer_id, _) => { - tracing::trace!( - "Getting next messages for consumer id: {} for partition: {} from offset: {}...", - consumer_id, - partition_id, - offset - ); - } - PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { - tracing::trace!( - "Getting next messages for consumer group: {} member: {} for partition: {} from offset: {}...", - consumer_group_id.0, - member_id.0, - partition_id, - offset - ); - } - } - let batches = self - .get_messages_by_offset( - stream_id, - topic_id, - partition_id, - offset, - count, - ) - .await?; - Ok(batches) - } - } - } - }?; - Ok((metadata, batches)) - } -} - -// A mental note: -// I think we can't expose as an access interface methods such as `get_topic_by_id` or `get_partition_by_id` etc.. -// In a case of a `Stream` module replacement (with a new implementation), the new implementation might not have a notion of `Topic` or `Partition` at all. -// So we should only expose some generic `get_entity_by_id` methods and rely on it's components accessors to get to the nested entities. -impl Streams { - pub fn exists(&self, id: &Identifier) -> bool { - match id.kind { - iggy_common::IdKind::Numeric => { - let id = id.get_u32_value().unwrap() as usize; - self.root.borrow().contains(id) - } - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - self.index.borrow().contains_key(&key) - } - } - } - - pub fn get_index(&self, id: &Identifier) -> usize { - match id.kind { - iggy_common::IdKind::Numeric => id.get_u32_value().unwrap() as usize, - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - *self.index.borrow().get(&key).expect("Stream not found") - } - } - } - - pub fn with_index( - &self, - f: impl FnOnce(&AHashMap<::Key, usize>) -> T, - ) -> T { - let index = self.index.borrow(); - f(&index) - } - - pub fn with_index_mut( - &self, - f: impl FnOnce(&mut AHashMap<::Key, usize>) -> T, - ) -> T { - let mut index = self.index.borrow_mut(); - f(&mut index) - } - - pub fn with_stream_by_id( - &self, - id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(id); - self.with_components_by_id(id, f) - } - - pub fn with_stream_by_id_mut( - &self, - id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(id); - self.with_components_by_id_mut(id, f) - } - - pub fn with_topics(&self, stream_id: &Identifier, f: impl FnOnce(&Topics) -> T) -> T { - self.with_stream_by_id(stream_id, helpers::topics(f)) - } - - pub fn with_topics_mut(&self, stream_id: &Identifier, f: impl FnOnce(&Topics) -> T) -> T { - self.with_stream_by_id(stream_id, helpers::topics_mut(f)) - } - - pub fn with_topic_by_id( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_topics(stream_id, |container| { - container.with_topic_by_id(topic_id, f) - }) - } - - pub fn with_topic_by_id_mut( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_topics_mut(stream_id, |container| { - container.with_topic_by_id_mut(topic_id, f) - }) - } - - pub fn with_consumer_groups( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(&ConsumerGroups) -> T, - ) -> T { - self.with_topics(stream_id, |container| { - container.with_consumer_groups(topic_id, f) - }) - } - - pub fn with_consumer_group_by_id( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - group_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_consumer_groups(stream_id, topic_id, |container| { - container.with_consumer_group_by_id(group_id, f) - }) - } - - pub fn with_consumer_group_by_id_mut( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - group_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_consumer_groups_mut(stream_id, topic_id, |container| { - container.with_consumer_group_by_id_mut(group_id, f) - }) - } - - pub fn with_consumer_groups_mut( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(&mut ConsumerGroups) -> T, - ) -> T { - self.with_topics_mut(stream_id, |container| { - container.with_consumer_groups_mut(topic_id, f) - }) - } - - pub fn with_partitions( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(&Partitions) -> T, - ) -> T { - self.with_topics(stream_id, |container| { - container.with_partitions(topic_id, f) - }) - } - - pub fn with_partitions_mut( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - f: impl FnOnce(&mut Partitions) -> T, - ) -> T { - self.with_topics_mut(stream_id, |container| { - container.with_partitions_mut(topic_id, f) - }) - } - - pub fn with_partition_by_id( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - id: partitions::ContainerId, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_partitions(stream_id, topic_id, |container| { - container.with_partition_by_id(id, f) - }) - } - - pub fn with_partition_by_id_mut( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - id: partitions::ContainerId, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_partitions_mut(stream_id, topic_id, |container| { - container.with_partition_by_id_mut(id, f) - }) - } - - pub async fn get_messages_by_offset( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - offset: u64, - count: u32, - ) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::default()); - } - - use crate::streaming::partitions::helpers; - let range = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - helpers::get_segment_range_by_offset(offset), - ); - - let mut remaining_count = count; - let mut batches = IggyMessagesBatchSet::empty(); - let mut current_offset = offset; - - for idx in range { - if remaining_count == 0 { - break; - } - - let (segment_start_offset, segment_end_offset) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let segment = &log.segments()[idx]; - (segment.start_offset, segment.end_offset) - }, - ); - - let offset = if current_offset < segment_start_offset { - segment_start_offset - } else { - current_offset - }; - - let mut end_offset = offset + (remaining_count - 1).max(1) as u64; - if end_offset > segment_end_offset { - end_offset = segment_end_offset; - } - - let messages = self - .get_messages_by_offset_base( - stream_id, - topic_id, - partition_id, - idx, - offset, - end_offset, - remaining_count, - segment_start_offset, - ) - .await?; - - let messages_count = messages.count(); - if messages_count == 0 { - current_offset = segment_end_offset + 1; - continue; - } - - remaining_count = remaining_count.saturating_sub(messages_count); - - if let Some(last_offset) = messages.last_offset() { - current_offset = last_offset + 1; - } else if messages_count > 0 { - current_offset += messages_count as u64; - } - - batches.add_batch_set(messages); - } - - Ok(batches) - } - - #[allow(clippy::too_many_arguments)] - async fn get_messages_by_offset_base( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - idx: usize, - offset: u64, - end_offset: u64, - count: u32, - segment_start_offset: u64, - ) -> Result { - let (is_journal_empty, journal_first_offset, journal_last_offset) = self - .with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let journal = log.journal(); - ( - journal.is_empty(), - journal.inner().base_offset, - journal.inner().current_offset, - ) - }, - ); - - // Case 0: Accumulator is empty, so all messages have to be on disk - if is_journal_empty { - return self - .load_messages_from_disk_by_offset( - stream_id, - topic_id, - partition_id, - idx, - offset, - count, - segment_start_offset, - ) - .await; - } - - // Case 1: All messages are in accumulator buffer - if offset >= journal_first_offset && end_offset <= journal_last_offset { - let batches = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - log.journal() - .get(|batches| batches.get_by_offset(offset, count)) - }, - ); - return Ok(batches); - } - - // Case 2: All messages are on disk - if end_offset < journal_first_offset { - return self - .load_messages_from_disk_by_offset( - stream_id, - topic_id, - partition_id, - idx, - offset, - count, - segment_start_offset, - ) - .await; - } - - // Case 3: Messages span disk and accumulator buffer boundary - // Calculate how many messages we need from disk - let disk_count = if offset < journal_first_offset { - ((journal_first_offset - offset) as u32).min(count) - } else { - 0 - }; - let mut combined_batch_set = IggyMessagesBatchSet::empty(); - - // Load messages from disk if needed - if disk_count > 0 { - let disk_messages = self - .load_messages_from_disk_by_offset( - stream_id, - topic_id, - partition_id, - idx, - offset, - disk_count, - segment_start_offset, - ) - .await - .error(|e: &IggyError| { - format!("Failed to load messages from disk, start offset: {offset}, count: {disk_count}, error: {e}") - })?; - - if !disk_messages.is_empty() { - combined_batch_set.add_batch_set(disk_messages); - } - } - - // Calculate how many more messages we need from the accumulator - let remaining_count = count - combined_batch_set.count(); - - if remaining_count > 0 { - let accumulator_start_offset = std::cmp::max(offset, journal_first_offset); - let journal_messages = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - log.journal().get(|batches| { - batches.get_by_offset(accumulator_start_offset, remaining_count) - }) - }, - ); - - if !journal_messages.is_empty() { - combined_batch_set.add_batch_set(journal_messages); - } - } - - Ok(combined_batch_set) - } - - #[allow(clippy::too_many_arguments)] - async fn load_messages_from_disk_by_offset( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - idx: usize, - start_offset: u64, - count: u32, - segment_start_offset: u64, - ) -> Result { - let relative_start_offset = (start_offset - segment_start_offset) as u32; - - let (index_reader, messages_reader, indexes) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let index_reader = log.storages()[idx] - .index_reader - .as_ref() - .expect("Index reader not initialized") - .clone(); - let message_reader = log.storages()[idx] - .messages_reader - .as_ref() - .expect("Messages reader not initialized") - .clone(); - let indexes = log.indexes()[idx].as_ref().map(|indexes| { - indexes - .slice_by_offset(relative_start_offset, count) - .unwrap_or_default() - }); - (index_reader, message_reader, indexes) - }, - ); - - let indexes_to_read = if let Some(indexes) = indexes { - if !indexes.is_empty() { - Some(indexes) - } else { - index_reader - .as_ref() - .load_from_disk_by_offset(relative_start_offset, count) - .await? - } - } else { - index_reader - .as_ref() - .load_from_disk_by_offset(relative_start_offset, count) - .await? - }; - - if indexes_to_read.is_none() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let indexes_to_read = indexes_to_read.unwrap(); - let batch = messages_reader - .as_ref() - .load_messages_from_disk(indexes_to_read) - .await - .error(|e: &IggyError| format!("Failed to load messages from disk: {e}"))?; - - batch - .validate_checksums_and_offsets(start_offset) - .error(|e: &IggyError| { - format!("Failed to validate messages read from disk! error: {e}") - })?; - - Ok(IggyMessagesBatchSet::from(batch)) - } - - pub async fn get_messages_by_timestamp( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - timestamp: u64, - count: u32, - ) -> Result { - use crate::streaming::partitions::helpers; - let Ok(range) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - helpers::get_segment_range_by_timestamp(timestamp), - ) else { - return Ok(IggyMessagesBatchSet::default()); - }; - - let mut remaining_count = count; - let mut batches = IggyMessagesBatchSet::empty(); - - for idx in range { - if remaining_count == 0 { - break; - } - - let segment_end_timestamp = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let segment = &log.segments()[idx]; - segment.end_timestamp - }, - ); - - if segment_end_timestamp < timestamp { - continue; - } - - let messages = self - .get_messages_by_timestamp_base( - stream_id, - topic_id, - partition_id, - idx, - timestamp, - remaining_count, - ) - .await?; - - let messages_count = messages.count(); - if messages_count == 0 { - continue; - } - - remaining_count = remaining_count.saturating_sub(messages_count); - batches.add_batch_set(messages); - } - - Ok(batches) - } - - async fn get_messages_by_timestamp_base( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - idx: usize, - timestamp: u64, - count: u32, - ) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::default()); - } - - let (is_journal_empty, journal_first_timestamp, journal_last_timestamp) = self - .with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let journal = log.journal(); - ( - journal.is_empty(), - journal.inner().first_timestamp, - journal.inner().end_timestamp, - ) - }, - ); - - // Case 0: Accumulator is empty, so all messages have to be on disk - if is_journal_empty { - return self - .load_messages_from_disk_by_timestamp( - stream_id, - topic_id, - partition_id, - idx, - timestamp, - count, - ) - .await; - } - - // Case 1: All messages are in accumulator buffer (timestamp is after journal ends) - if timestamp > journal_last_timestamp { - return Ok(IggyMessagesBatchSet::empty()); - } - - // Case 1b: Timestamp is within journal range - if timestamp >= journal_first_timestamp { - let batches = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - log.journal() - .get(|batches| batches.get_by_timestamp(timestamp, count)) - }, - ); - return Ok(batches); - } - - // Case 2: All messages are on disk (timestamp is before journal's first timestamp) - let disk_messages = self - .load_messages_from_disk_by_timestamp( - stream_id, - topic_id, - partition_id, - idx, - timestamp, - count, - ) - .await?; - - if disk_messages.count() >= count { - return Ok(disk_messages); - } - - // Case 3: Messages span disk and accumulator buffer boundary - let remaining_count = count - disk_messages.count(); - let journal_messages = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - log.journal() - .get(|batches| batches.get_by_timestamp(timestamp, remaining_count)) - }, - ); - - let mut combined_batch_set = disk_messages; - if !journal_messages.is_empty() { - combined_batch_set.add_batch_set(journal_messages); - } - Ok(combined_batch_set) - } - - async fn load_messages_from_disk_by_timestamp( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - idx: usize, - timestamp: u64, - count: u32, - ) -> Result { - let (index_reader, messages_reader, indexes) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(_, _, _, _, _, _, log)| { - let index_reader = log.storages()[idx] - .index_reader - .as_ref() - .expect("Index reader not initialized") - .clone(); - let messages_reader = log.storages()[idx] - .messages_reader - .as_ref() - .expect("Messages reader not initialized") - .clone(); - let indexes = log.indexes()[idx].as_ref().map(|indexes| { - indexes - .slice_by_timestamp(timestamp, count) - .unwrap_or_default() - }); - (index_reader, messages_reader, indexes) - }, - ); - - let indexes_to_read = if let Some(indexes) = indexes { - if !indexes.is_empty() { - Some(indexes) - } else { - index_reader - .as_ref() - .load_from_disk_by_timestamp(timestamp, count) - .await? - } - } else { - index_reader - .as_ref() - .load_from_disk_by_timestamp(timestamp, count) - .await? - }; - - if indexes_to_read.is_none() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let indexes_to_read = indexes_to_read.unwrap(); - - let batch = messages_reader - .as_ref() - .load_messages_from_disk(indexes_to_read) - .await - .error(|e: &IggyError| { - format!("Failed to load messages from disk by timestamp: {e}") - })?; - - Ok(IggyMessagesBatchSet::from(batch)) - } - - pub async fn handle_full_segment( - &self, - registry: &Rc, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: partitions::ContainerId, - config: &crate::configs::system::SystemConfig, - ) -> Result<(), IggyError> { - let numeric_stream_id = - self.with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - - if config.segment.cache_indexes == CacheIndexesConfig::OpenSegment - || config.segment.cache_indexes == CacheIndexesConfig::None - { - self.with_partition_by_id_mut(stream_id, topic_id, partition_id, |(.., log)| { - log.clear_active_indexes(); - }); - } - - self.with_partition_by_id_mut(stream_id, topic_id, partition_id, |(.., log)| { - log.active_segment_mut().sealed = true; - }); - let (log_writer, index_writer) = - self.with_partition_by_id_mut(stream_id, topic_id, partition_id, |(.., log)| { - let (msg, index) = log.active_storage_mut().shutdown(); - (msg.unwrap(), index.unwrap()) - }); - - registry - .oneshot("fsync:segment-close-log") - .critical(true) - .run(move |_shutdown| async move { - match log_writer.fsync().await { - Ok(_) => Ok(()), - Err(e) => { - error!("Failed to fsync log writer on segment close: {}", e); - Err(e) - } - } - }) - .spawn(); - - registry - .oneshot("fsync:segment-close-index") - .critical(true) - .run(move |_shutdown| async move { - match index_writer.fsync().await { - Ok(_) => { - drop(index_writer); - Ok(()) - } - Err(e) => { - error!("Failed to fsync index writer on segment close: {}", e); - drop(index_writer); - Err(e) - } - } - }) - .spawn(); - - let (start_offset, size, end_offset) = - self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - ( - log.active_segment().start_offset, - log.active_segment().size, - log.active_segment().end_offset, - ) - }); - - tracing::info!( - "Closed segment for stream: {}, topic: {} with start offset: {}, end offset: {}, size: {} for partition with ID: {}.", - stream_id, - topic_id, - start_offset, - end_offset, - size, - partition_id - ); - - let messages_size = 0; - let indexes_size = 0; - let segment = Segment::new( - end_offset + 1, - config.segment.size, - config.segment.message_expiry, - ); - - let storage = create_segment_storage( - config, - numeric_stream_id, - numeric_topic_id, - partition_id, - messages_size, - indexes_size, - end_offset + 1, - ) - .await?; - self.with_partition_by_id_mut(stream_id, topic_id, partition_id, |(.., log)| { - log.add_persisted_segment(segment, storage); - }); - - Ok(()) - } - - pub async fn persist_messages( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - reason: &str, - config: &SystemConfig, - ) -> Result { - let is_empty = self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - log.journal().is_empty() - }); - if is_empty { - return Ok(0); - } - - let batches = self.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::commit_journal(), - ); - - tracing::trace!( - "Persisting messages on disk for stream ID: {}, topic ID: {}, partition ID: {} because {}...", - stream_id, - topic_id, - partition_id, - reason - ); - - let batch_count = self - .persist_messages_to_disk(stream_id, topic_id, partition_id, batches, config) - .await?; - - Ok(batch_count) - } - - pub async fn persist_messages_to_disk( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - batches: IggyMessagesBatchSet, - config: &SystemConfig, - ) -> Result { - let batch_count = batches.count(); - let batch_size = batches.size(); - - if batch_count == 0 { - return Ok(0); - } - - let has_segments = - self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - log.has_segments() - }); - - if !has_segments { - return Ok(0); - } - - // Extract storage before async operations - let (messages_writer, index_writer) = - self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - ( - log.active_storage() - .messages_writer - .as_ref() - .expect("Messages writer not initialized") - .clone(), - log.active_storage() - .index_writer - .as_ref() - .expect("Index writer not initialized") - .clone(), - ) - }); - let guard = messages_writer.lock.lock().await; - - let saved = messages_writer - .as_ref() - .save_batch_set(batches) - .await - .error(|e: &IggyError| { - format!( - "Failed to save batch of {batch_count} messages \ - ({batch_size} bytes) to stream ID: {stream_id}, topic ID: {topic_id}, partition ID: {partition_id}. {e}", - ) - })?; - - // Extract unsaved indexes before async operation - let unsaved_indexes_slice = - self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - log.active_indexes().unwrap().unsaved_slice() - }); - - let indexes_len = unsaved_indexes_slice.len(); - index_writer - .as_ref() - .save_indexes(unsaved_indexes_slice) - .await - .error(|e: &IggyError| { - format!("Failed to save index of {indexes_len} indexes to stream ID: {stream_id}, topic ID: {topic_id} {partition_id}. {e}",) - })?; - - tracing::trace!( - "Persisted {} messages on disk for stream ID: {}, topic ID: {}, for partition with ID: {}, total bytes written: {}.", - batch_count, - stream_id, - topic_id, - partition_id, - saved - ); - - self.with_partition_by_id_mut( - stream_id, - topic_id, - partition_id, - streaming_partitions::helpers::update_index_and_increment_stats(saved, config), - ); - - drop(guard); - Ok(batch_count) - } - - pub async fn fsync_all_messages( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - ) -> Result<(), IggyError> { - let storage = self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| { - if !log.has_segments() { - return None; - } - Some(log.active_storage().clone()) - }); - - let Some(storage) = storage else { - return Ok(()); - }; - - if storage.messages_writer.is_none() || storage.index_writer.is_none() { - return Ok(()); - } - - if let Some(ref messages_writer) = storage.messages_writer - && let Err(e) = messages_writer.fsync().await - { - tracing::error!( - "Failed to fsync messages writer for partition {}: {}", - partition_id, - e - ); - return Err(e); - } - - if let Some(ref index_writer) = storage.index_writer - && let Err(e) = index_writer.fsync().await - { - tracing::error!( - "Failed to fsync index writer for partition {}: {}", - partition_id, - e - ); - return Err(e); - } - - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - pub async fn auto_commit_consumer_offset( - &self, - config: &SystemConfig, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - consumer: PollingConsumer, - offset: u64, - ) -> Result<(), IggyError> { - let numeric_stream_id = - self.with_stream_by_id(stream_id, streams::helpers::get_stream_id()); - let numeric_topic_id = - self.with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id()); - - tracing::trace!( - "Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}", - offset, - consumer, - numeric_stream_id, - numeric_topic_id, - partition_id - ); - - match consumer { - PollingConsumer::Consumer(consumer_id, _) => { - tracing::trace!( - "Auto-committing offset {} for consumer {} on stream {}, topic {}, partition {}", - offset, - consumer_id, - numeric_stream_id, - numeric_topic_id, - partition_id - ); - let (offset_value, path) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let item = hdl.get_or_insert( - consumer_id, - crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer( - consumer_id as u32, - &config.get_consumer_offsets_path(numeric_stream_id, numeric_topic_id, partition_id), - ), - ); - item.offset.store(offset, Ordering::Relaxed); - let offset_value = item.offset.load(Ordering::Relaxed); - let path = item.path.clone(); - (offset_value, path) - }, - ); - crate::streaming::partitions::storage::persist_offset(&path, offset_value).await?; - } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - tracing::trace!( - "Auto-committing offset {} for consumer group {} on stream {}, topic {}, partition {}", - offset, - consumer_group_id.0, - numeric_stream_id, - numeric_topic_id, - partition_id - ); - let (offset_value, path) = self.with_partition_by_id( - stream_id, - topic_id, - partition_id, - |(.., offsets, _)| { - let hdl = offsets.pin(); - let item = hdl.get_or_insert( - consumer_group_id, - ConsumerOffset::default_for_consumer_group( - consumer_group_id, - &config.get_consumer_group_offsets_path( - numeric_stream_id, - numeric_topic_id, - partition_id, - ), - ), - ); - item.offset.store(offset, Ordering::Relaxed); - let offset_value = item.offset.load(Ordering::Relaxed); - let path = item.path.clone(); - (offset_value, path) - }, - ); - crate::streaming::partitions::storage::persist_offset(&path, offset_value).await?; - } - } - - Ok(()) - } -} diff --git a/core/server/src/slab/topics.rs b/core/server/src/slab/topics.rs deleted file mode 100644 index 5270dbd849..0000000000 --- a/core/server/src/slab/topics.rs +++ /dev/null @@ -1,307 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use ahash::AHashMap; -use iggy_common::Identifier; -use slab::Slab; -use std::{cell::RefCell, sync::Arc}; - -use crate::{ - slab::{ - Keyed, - consumer_groups::ConsumerGroups, - helpers, - partitions::Partitions, - traits_ext::{ - ComponentsById, DeleteCell, EntityComponentSystem, EntityComponentSystemMutCell, - InsertCell, InteriorMutability, IntoComponents, - }, - }, - streaming::{ - stats::TopicStats, - topics::{ - consumer_group::{ConsumerGroupRef, ConsumerGroupRefMut}, - topic::{self, TopicRef, TopicRefMut}, - }, - }, -}; - -const CAPACITY: usize = 1024; -pub type ContainerId = usize; - -#[derive(Debug, Clone)] -pub struct Topics { - index: RefCell::Key, ContainerId>>, - root: RefCell>, - auxilaries: RefCell>, - stats: RefCell>>, -} - -impl InsertCell for Topics { - type Idx = ContainerId; - type Item = topic::Topic; - - fn insert(&self, item: Self::Item) -> Self::Idx { - let (root, auxilary, stats) = item.into_components(); - let mut root_container = self.root.borrow_mut(); - let mut auxilaries = self.auxilaries.borrow_mut(); - let mut indexes = self.index.borrow_mut(); - let mut stats_container = self.stats.borrow_mut(); - - let key = root.key().clone(); - let entity_id = root_container.insert(root); - let id = stats_container.insert(stats); - assert_eq!( - entity_id, id, - "topic_insert: id mismatch when inserting stats component" - ); - let id = auxilaries.insert(auxilary); - assert_eq!( - entity_id, id, - "topic_insert: id mismatch when inserting auxilary component" - ); - let root = root_container.get_mut(entity_id).unwrap(); - root.update_id(entity_id); - indexes.insert(key, entity_id); - entity_id - } -} - -impl DeleteCell for Topics { - type Idx = ContainerId; - type Item = topic::Topic; - - fn delete(&self, id: Self::Idx) -> Self::Item { - let mut root_container = self.root.borrow_mut(); - let mut auxilaries = self.auxilaries.borrow_mut(); - let mut indexes = self.index.borrow_mut(); - let mut stats_container = self.stats.borrow_mut(); - - let root = root_container.remove(id); - let auxilary = auxilaries.remove(id); - let stats = stats_container.remove(id); - - // Remove from index - let key = root.key(); - indexes.remove(key).unwrap_or_else(|| { - panic!( - "topic_delete: key not found with key: {} and id: {}", - key, id - ) - }); - - topic::Topic::new_with_components(root, auxilary, stats) - } -} - -impl<'a> From<&'a Topics> for topic::TopicRef<'a> { - fn from(value: &'a Topics) -> Self { - let root = value.root.borrow(); - let auxilary = value.auxilaries.borrow(); - let stats = value.stats.borrow(); - topic::TopicRef::new(root, auxilary, stats) - } -} - -impl<'a> From<&'a Topics> for topic::TopicRefMut<'a> { - fn from(value: &'a Topics) -> Self { - let root = value.root.borrow_mut(); - let auxilary = value.auxilaries.borrow_mut(); - let stats = value.stats.borrow_mut(); - topic::TopicRefMut::new(root, auxilary, stats) - } -} - -impl Default for Topics { - fn default() -> Self { - Self { - index: RefCell::new(AHashMap::with_capacity(CAPACITY)), - root: RefCell::new(Slab::with_capacity(CAPACITY)), - auxilaries: RefCell::new(Slab::with_capacity(CAPACITY)), - stats: RefCell::new(Slab::with_capacity(CAPACITY)), - } - } -} - -impl Topics { - /// Construct from pre-built entries with specific IDs. - pub fn from_entries(entries: impl IntoIterator) -> Self { - let entries: Vec<_> = entries.into_iter().collect(); - - let mut index = AHashMap::with_capacity(entries.len()); - let mut root_entries = Vec::with_capacity(entries.len()); - let mut auxilary_entries = Vec::with_capacity(entries.len()); - let mut stats_entries = Vec::with_capacity(entries.len()); - - for (id, topic) in entries { - let (mut root, auxilary, stats) = topic.into_components(); - root.update_id(id); - index.insert(root.key().clone(), id); - root_entries.push((id, root)); - auxilary_entries.push((id, auxilary)); - stats_entries.push((id, stats)); - } - - Self { - index: RefCell::new(index), - root: RefCell::new(root_entries.into_iter().collect()), - auxilaries: RefCell::new(auxilary_entries.into_iter().collect()), - stats: RefCell::new(stats_entries.into_iter().collect()), - } - } -} - -impl EntityComponentSystem for Topics { - type Idx = ContainerId; - type Entity = topic::Topic; - type EntityComponents<'a> = topic::TopicRef<'a>; - - fn with_components(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponents<'a>) -> O, - { - f(self.into()) - } -} - -impl EntityComponentSystemMutCell for Topics { - type EntityComponentsMut<'a> = topic::TopicRefMut<'a>; - - fn with_components_mut(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O, - { - f(self.into()) - } -} - -impl Topics { - pub fn len(&self) -> usize { - self.root.borrow().len() - } - - pub fn is_empty(&self) -> bool { - self.root.borrow().is_empty() - } - - pub fn exists(&self, id: &Identifier) -> bool { - match id.kind { - iggy_common::IdKind::Numeric => { - let id = id.get_u32_value().unwrap() as usize; - self.root.borrow().contains(id) - } - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - self.index.borrow().contains_key(&key) - } - } - } - - pub fn get_index(&self, id: &Identifier) -> usize { - match id.kind { - iggy_common::IdKind::Numeric => id.get_u32_value().unwrap() as usize, - iggy_common::IdKind::String => { - let key = id.get_string_value().unwrap(); - *self.index.borrow().get(&key).expect("Topic not found") - } - } - } - - pub fn with_index( - &self, - f: impl FnOnce(&AHashMap<::Key, usize>) -> T, - ) -> T { - let index = self.index.borrow(); - f(&index) - } - - pub fn with_index_mut( - &self, - f: impl FnOnce(&mut AHashMap<::Key, usize>) -> T, - ) -> T { - let mut index = self.index.borrow_mut(); - f(&mut index) - } - - pub fn with_topic_by_id( - &self, - topic_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(topic_id); - self.with_components_by_id(id, f) - } - - pub fn with_topic_by_id_mut( - &self, - topic_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - let id = self.get_index(topic_id); - self.with_components_by_id_mut(id, f) - } - - pub fn with_consumer_groups( - &self, - topic_id: &Identifier, - f: impl FnOnce(&ConsumerGroups) -> T, - ) -> T { - self.with_topic_by_id(topic_id, helpers::consumer_groups(f)) - } - - pub fn with_consumer_groups_mut( - &self, - topic_id: &Identifier, - f: impl FnOnce(&mut ConsumerGroups) -> T, - ) -> T { - self.with_topic_by_id_mut(topic_id, helpers::consumer_groups_mut(f)) - } - - pub fn with_consumer_group_by_id( - &self, - topic_id: &Identifier, - group_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_consumer_groups(topic_id, |container| { - container.with_consumer_group_by_id(group_id, f) - }) - } - - pub fn with_consumer_group_by_id_mut( - &self, - topic_id: &Identifier, - group_id: &Identifier, - f: impl FnOnce(ComponentsById) -> T, - ) -> T { - self.with_consumer_groups_mut(topic_id, |container| { - container.with_consumer_group_by_id_mut(group_id, f) - }) - } - - pub fn with_partitions(&self, topic_id: &Identifier, f: impl FnOnce(&Partitions) -> T) -> T { - self.with_topic_by_id(topic_id, helpers::partitions(f)) - } - - pub fn with_partitions_mut( - &self, - topic_id: &Identifier, - f: impl FnOnce(&mut Partitions) -> T, - ) -> T { - self.with_topic_by_id_mut(topic_id, helpers::partitions_mut(f)) - } -} diff --git a/core/server/src/slab/traits_ext.rs b/core/server/src/slab/traits_ext.rs deleted file mode 100644 index 957232fefa..0000000000 --- a/core/server/src/slab/traits_ext.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub trait IntoComponents { - type Components; - fn into_components(self) -> Self::Components; -} - -/// Marker trait for the `Entity`. -pub trait EntityMarker { - type Idx; - fn id(&self) -> Self::Idx; - fn update_id(&mut self, id: Self::Idx); -} - -/// Insert trait for inserting an `Entity`` into container. -pub trait Insert { - type Idx; - type Item: IntoComponents + EntityMarker; - fn insert(&mut self, item: Self::Item) -> Self::Idx; -} - -pub trait InsertCell { - type Idx; - type Item: IntoComponents + EntityMarker; - fn insert(&self, item: Self::Item) -> Self::Idx; -} - -/// Delete trait for deleting an `Entity` from container. -pub trait Delete { - type Idx; - type Item: IntoComponents + EntityMarker; - fn delete(&mut self, id: Self::Idx) -> Self::Item; -} - -/// Delete trait for deleting an `Entity` from container for container types that use interior mutability. -pub trait DeleteCell { - type Idx; - type Item: IntoComponents + EntityMarker; - fn delete(&self, id: Self::Idx) -> Self::Item; -} - -/// Trait for getting components by EntityId. -pub trait IntoComponentsById { - type Idx; - type Output; - fn into_components_by_id(self, index: Self::Idx) -> Self::Output; -} - -/// Marker type for borrow component containers. -pub struct Borrow; -/// Marker type for component containers that use interior mutability. -pub struct InteriorMutability; - -pub type Components = ::Components; -pub type ComponentsById<'a, T> = ::Output; - -pub trait EntityComponentSystem { - type Idx; - type Entity: IntoComponents + EntityMarker; - type EntityComponents<'a>: IntoComponents + IntoComponentsById; - - fn with_components(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponents<'a>) -> O; - - fn with_components_by_id(&self, id: Self::Idx, f: F) -> O - where - F: for<'a> FnOnce(ComponentsById<'a, Self::EntityComponents<'a>>) -> O, - { - self.with_components(|components| f(components.into_components_by_id(id))) - } -} - -pub trait EntityComponentSystemMut: EntityComponentSystem { - type EntityComponentsMut<'a>: IntoComponents + IntoComponentsById; - - fn with_components_mut(&mut self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O; - - fn with_components_by_id_mut(&mut self, id: Self::Idx, f: F) -> O - where - F: for<'a> FnOnce(ComponentsById<'a, Self::EntityComponentsMut<'a>>) -> O, - { - self.with_components_mut(|components| f(components.into_components_by_id(id))) - } -} - -pub trait EntityComponentSystemMutCell: EntityComponentSystem { - type EntityComponentsMut<'a>: IntoComponents + IntoComponentsById; - - fn with_components_mut(&self, f: F) -> O - where - F: for<'a> FnOnce(Self::EntityComponentsMut<'a>) -> O; - - fn with_components_by_id_mut(&self, id: Self::Idx, f: F) -> O - where - F: for<'a> FnOnce(ComponentsById<'a, Self::EntityComponentsMut<'a>>) -> O, - { - self.with_components_mut(|components| f(components.into_components_by_id(id))) - } -} diff --git a/core/server/src/slab/users.rs b/core/server/src/slab/users.rs deleted file mode 100644 index a0375a9ab2..0000000000 --- a/core/server/src/slab/users.rs +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::users::user::User; -use ahash::AHashMap; -use iggy_common::{Identifier, IggyError}; -use slab::Slab; -use std::cell::RefCell; - -const CAPACITY: usize = 1024; - -#[derive(Debug, Clone, Default)] -pub struct Users { - index: RefCell>, - users: RefCell>, -} - -impl Users { - pub fn new() -> Self { - Self { - index: RefCell::new(AHashMap::with_capacity(CAPACITY)), - users: RefCell::new(Slab::with_capacity(CAPACITY)), - } - } - - /// Insert a user and return the assigned ID (auto-incremented by Slab) - pub fn insert(&self, user: User) -> usize { - let username = user.username.clone(); - let mut users = self.users.borrow_mut(); - let mut index = self.index.borrow_mut(); - - let id = users.insert(user); - users[id].id = id as u32; - index.insert(username, id); - id - } - - /// Get user by ID (numeric identifier) - pub fn get(&self, id: usize) -> Option { - self.users.borrow().get(id).cloned() - } - - /// Get user by username or ID (via Identifier enum) - pub fn get_by_identifier(&self, identifier: &Identifier) -> Result, IggyError> { - match identifier.kind { - iggy_common::IdKind::Numeric => { - let id = identifier.get_u32_value()? as usize; - Ok(self.users.borrow().get(id).cloned()) - } - iggy_common::IdKind::String => { - let username = identifier.get_string_value()?; - let index = self.index.borrow(); - if let Some(&id) = index.get(&username) { - Ok(self.users.borrow().get(id).cloned()) - } else { - Ok(None) - } - } - } - } - - /// Remove user by Slab index - pub fn remove(&self, id: usize) -> Option { - let mut users = self.users.borrow_mut(); - let mut index = self.index.borrow_mut(); - - if !users.contains(id) { - return None; - } - - let user = users.remove(id); - index.remove(&user.username); - Some(user) - } - - /// Check if user exists - pub fn contains(&self, identifier: &Identifier) -> bool { - match identifier.kind { - iggy_common::IdKind::Numeric => { - if let Ok(id) = identifier.get_u32_value() { - self.users.borrow().contains(id as usize) - } else { - false - } - } - iggy_common::IdKind::String => { - if let Ok(username) = identifier.get_string_value() { - self.index.borrow().contains_key(&username) - } else { - false - } - } - } - } - - /// Get all users as a Vec - pub fn values(&self) -> Vec { - self.users.borrow().iter().map(|(_, u)| u.clone()).collect() - } - - /// Get number of users - pub fn len(&self) -> usize { - self.users.borrow().len() - } - - /// Check if empty - pub fn is_empty(&self) -> bool { - self.users.borrow().is_empty() - } - - /// Get mutable access to a user via closure (to avoid RefMut escaping issues) - pub fn with_user_mut(&self, identifier: &Identifier, f: F) -> Result - where - F: FnOnce(&mut User) -> R, - { - let id = match identifier.kind { - iggy_common::IdKind::Numeric => identifier.get_u32_value()? as usize, - iggy_common::IdKind::String => { - let username = identifier.get_string_value()?; - let index = self.index.borrow(); - *index - .get(&username) - .ok_or_else(|| IggyError::ResourceNotFound(username.to_string()))? - } - }; - - let mut users = self.users.borrow_mut(); - let user = users - .get_mut(id) - .ok_or_else(|| IggyError::ResourceNotFound(identifier.to_string()))?; - Ok(f(user)) - } - - /// Check if username already exists (for validation during create/update) - pub fn username_exists(&self, username: &str) -> bool { - self.index.borrow().contains_key(username) - } - - /// Get ID by username (useful for cross-references) - pub fn get_id_by_username(&self, username: &str) -> Option { - self.index.borrow().get(username).copied() - } - - pub fn update_username( - &self, - identifier: &Identifier, - new_username: String, - ) -> Result<(), IggyError> { - let id = match identifier.kind { - iggy_common::IdKind::Numeric => identifier.get_u32_value()? as usize, - iggy_common::IdKind::String => { - let username = identifier.get_string_value()?; - let index = self.index.borrow(); - *index - .get(&username) - .ok_or_else(|| IggyError::ResourceNotFound(username.to_string()))? - } - }; - - let old_username = { - let users = self.users.borrow(); - let user = users - .get(id) - .ok_or_else(|| IggyError::ResourceNotFound(identifier.to_string()))?; - user.username.clone() - }; - - if old_username == new_username { - return Ok(()); - } - - tracing::trace!( - "Updating username: '{}' → '{}' for user ID: {}", - old_username, - new_username, - id - ); - - { - let mut users = self.users.borrow_mut(); - let user = users - .get_mut(id) - .ok_or_else(|| IggyError::ResourceNotFound(identifier.to_string()))?; - user.username = new_username.clone(); - } - - let mut index = self.index.borrow_mut(); - index.remove(&old_username); - index.insert(new_username, id); - - Ok(()) - } -} diff --git a/core/server/src/streaming/auth.rs b/core/server/src/streaming/auth.rs new file mode 100644 index 0000000000..08f21d3db5 --- /dev/null +++ b/core/server/src/streaming/auth.rs @@ -0,0 +1,105 @@ +/* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Type-safe authentication proof system. +//! +//! This module provides compile-time guarantees for authentication by using +//! proof-carrying code patterns. The [`Auth`] type can only be constructed +//! via [`IggyShard::auth()`], ensuring that any code path holding an `Auth` +//! value has passed authentication checks. + +use iggy_common::UserId; + +/// Proof of successful authentication. +/// +/// This type can ONLY be constructed via [`IggyShard::auth()`]. +/// The `_sealed` field prevents external construction, ensuring that +/// possession of an `Auth` value proves authentication has occurred. +/// +/// # Invariants +/// - `user_id` is always a valid, authenticated user ID (never `u32::MAX`) +/// - The user was authenticated at the time of `Auth` construction +#[derive(Clone, Copy, Debug)] +pub struct Auth { + user_id: UserId, + /// Zero-sized private field that prevents external construction. + /// The only way to create an `Auth` is through `Auth::new()` which is `pub(crate)`. + _sealed: (), +} + +impl Auth { + /// Creates a new authentication proof. + /// + /// # Safety Contract (not unsafe, but must be upheld) + /// This MUST only be called after verifying that the user is authenticated. + /// Callers are responsible for ensuring `user_id` corresponds to a valid, + /// authenticated user. + #[inline] + pub(crate) fn new(user_id: UserId) -> Self { + debug_assert!(user_id != u32::MAX, "Auth created with invalid user_id"); + Self { + user_id, + _sealed: (), + } + } + + /// Returns the authenticated user's ID. + #[inline] + pub fn user_id(&self) -> UserId { + self.user_id + } +} + +/// Marker trait for authentication requirements. +/// +/// This trait is sealed and cannot be implemented outside this module. +/// It defines whether a command requires authentication and what token +/// type the handler receives. +pub trait AuthRequirement: private::Sealed { + /// A descriptive name for logging/debugging. + const NAME: &'static str; +} + +/// Marker type indicating a command requires authentication. +/// +/// Handlers with this requirement will receive an [`Auth`] proof token +/// and can pass it to shard methods that require authentication. +pub struct Authenticated; + +impl AuthRequirement for Authenticated { + const NAME: &'static str = "Authenticated"; +} + +impl private::Sealed for Authenticated {} + +/// Marker type indicating a command does NOT require authentication. +/// +/// Used for commands like `Ping`, `LoginUser`, and `LoginWithPersonalAccessToken` +/// that must work before authentication. +pub struct Unauthenticated; + +impl AuthRequirement for Unauthenticated { + const NAME: &'static str = "Unauthenticated"; +} + +impl private::Sealed for Unauthenticated {} + +mod private { + /// Sealed trait pattern - prevents external implementations of AuthRequirement. + pub trait Sealed {} +} diff --git a/core/server/src/streaming/mod.rs b/core/server/src/streaming/mod.rs index a6141bd8b8..18928418e7 100644 --- a/core/server/src/streaming/mod.rs +++ b/core/server/src/streaming/mod.rs @@ -16,9 +16,11 @@ * under the License. */ +pub mod auth; pub mod clients; pub mod deduplication; pub mod diagnostics; +pub mod partition_ops; pub mod partitions; pub mod persistence; pub mod polling_consumer; diff --git a/core/server/src/streaming/partition_ops.rs b/core/server/src/streaming/partition_ops.rs new file mode 100644 index 0000000000..6a878b53e2 --- /dev/null +++ b/core/server/src/streaming/partition_ops.rs @@ -0,0 +1,671 @@ +/* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Shared partition operations that can be used by both production code and tests. +//! +//! This module provides the core logic for polling and loading messages from partitions, +//! avoiding code duplication between `IggyShard` and test harnesses. + +use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata; +use crate::shard::shard_local_partitions::ShardLocalPartitions; +use crate::shard::system::messages::PollingArgs; +use crate::streaming::partitions::journal::Journal; +use crate::streaming::polling_consumer::PollingConsumer; +use crate::streaming::segments::IggyMessagesBatchSet; +use iggy_common::sharding::IggyNamespace; +use iggy_common::{IggyError, PollingKind}; +use std::cell::RefCell; +use std::sync::atomic::Ordering; + +/// Poll messages from a partition store. +/// +/// This is the core polling logic shared between production code and tests. +pub async fn poll_messages( + partition_store: &RefCell, + namespace: &IggyNamespace, + consumer: PollingConsumer, + args: PollingArgs, +) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let partition_id = namespace.partition_id(); + let count = args.count; + let strategy = args.strategy; + let value = strategy.value; + + // Handle timestamp polling separately - it has different logic + if strategy.kind == PollingKind::Timestamp { + return poll_messages_by_timestamp(partition_store, namespace, value, count).await; + } + + // Phase 1: Extract metadata and determine start offset + let (metadata, start_offset) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + + let current_offset = partition_data.offset.load(Ordering::Relaxed); + let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); + + let start_offset = match strategy.kind { + PollingKind::Offset => { + let offset = value; + if offset > current_offset { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + offset + } + PollingKind::First => partition_data + .log + .segments() + .first() + .map(|segment| segment.start_offset) + .unwrap_or(0), + PollingKind::Last => { + let mut requested_count = count as u64; + if requested_count > current_offset + 1 { + requested_count = current_offset + 1; + } + 1 + current_offset - requested_count + } + PollingKind::Next => { + let stored_offset = match consumer { + PollingConsumer::Consumer(id, _) => partition_data + .consumer_offsets + .pin() + .get(&id) + .map(|item| item.offset.load(Ordering::Relaxed)), + PollingConsumer::ConsumerGroup(cg_id, _) => partition_data + .consumer_group_offsets + .pin() + .get(&cg_id) + .map(|item| item.offset.load(Ordering::Relaxed)), + }; + match stored_offset { + Some(offset) => offset + 1, + None => partition_data + .log + .segments() + .first() + .map(|segment| segment.start_offset) + .unwrap_or(0), + } + } + PollingKind::Timestamp => unreachable!("Timestamp handled above"), + }; + + if start_offset > current_offset || count == 0 { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + + (metadata, start_offset) + }; + + // Phase 2: Get messages using hybrid disk+journal logic + let batches = get_messages_by_offset(partition_store, namespace, start_offset, count).await?; + Ok((metadata, batches)) +} + +/// Get messages by offset, handling the hybrid disk+journal case. +pub async fn get_messages_by_offset( + partition_store: &RefCell, + namespace: &IggyNamespace, + start_offset: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Get journal metadata for routing + let (is_journal_empty, journal_first_offset, journal_last_offset) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + + let journal = partition_data.log.journal(); + let journal_inner = journal.inner(); + ( + journal.is_empty(), + journal_inner.base_offset, + journal_inner.current_offset, + ) + }; + + let end_offset = start_offset + (count - 1).max(1) as u64; + + // Case 0: Journal is empty, all messages on disk + if is_journal_empty { + return load_messages_from_disk(partition_store, namespace, start_offset, count).await; + } + + // Case 1: All messages are in journal + if start_offset >= journal_first_offset && end_offset <= journal_last_offset { + let batches = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + partition_data + .log + .journal() + .get(|batches| batches.get_by_offset(start_offset, count)) + }; + return Ok(batches); + } + + // Case 2: All messages on disk (end_offset < journal_first_offset) + if end_offset < journal_first_offset { + return load_messages_from_disk(partition_store, namespace, start_offset, count).await; + } + + // Case 3: Messages span disk and journal boundary + let disk_count = if start_offset < journal_first_offset { + ((journal_first_offset - start_offset) as u32).min(count) + } else { + 0 + }; + + let mut combined_batch_set = IggyMessagesBatchSet::empty(); + + // Load messages from disk if needed + if disk_count > 0 { + let disk_messages = + load_messages_from_disk(partition_store, namespace, start_offset, disk_count).await?; + if !disk_messages.is_empty() { + combined_batch_set.add_batch_set(disk_messages); + } + } + + // Get remaining messages from journal + let remaining_count = count - combined_batch_set.count(); + if remaining_count > 0 { + let journal_start_offset = std::cmp::max(start_offset, journal_first_offset); + let journal_messages = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + partition_data + .log + .journal() + .get(|batches| batches.get_by_offset(journal_start_offset, remaining_count)) + }; + if !journal_messages.is_empty() { + combined_batch_set.add_batch_set(journal_messages); + } + } + + Ok(combined_batch_set) +} + +/// Poll messages by timestamp. +async fn poll_messages_by_timestamp( + partition_store: &RefCell, + namespace: &IggyNamespace, + timestamp: u64, + count: u32, +) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let partition_id = namespace.partition_id(); + + // Get metadata and journal info + let (metadata, is_journal_empty, journal_first_timestamp, journal_last_timestamp) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + + let current_offset = partition_data.offset.load(Ordering::Relaxed); + let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); + + let journal = partition_data.log.journal(); + let journal_inner = journal.inner(); + ( + metadata, + journal.is_empty(), + journal_inner.first_timestamp, + journal_inner.end_timestamp, + ) + }; + + if count == 0 { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + + // Case 0: Journal is empty, all messages on disk + if is_journal_empty { + let batches = + load_messages_from_disk_by_timestamp(partition_store, namespace, timestamp, count) + .await?; + return Ok((metadata, batches)); + } + + // Case 1: Timestamp is after journal's last timestamp - no messages + if timestamp > journal_last_timestamp { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + + // Case 2: Timestamp is within journal range - get from journal + if timestamp >= journal_first_timestamp { + let batches = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + partition_data + .log + .journal() + .get(|batches| batches.get_by_timestamp(timestamp, count)) + }; + return Ok((metadata, batches)); + } + + // Case 3: Timestamp is before journal - need disk + possibly journal + let disk_messages = + load_messages_from_disk_by_timestamp(partition_store, namespace, timestamp, count).await?; + + if disk_messages.count() >= count { + return Ok((metadata, disk_messages)); + } + + // Case 4: Messages span disk and journal + let remaining_count = count - disk_messages.count(); + let journal_messages = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist for poll"); + partition_data + .log + .journal() + .get(|batches| batches.get_by_timestamp(timestamp, remaining_count)) + }; + + let mut combined_batch_set = disk_messages; + if !journal_messages.is_empty() { + combined_batch_set.add_batch_set(journal_messages); + } + Ok((metadata, combined_batch_set)) +} + +/// Load messages from disk by offset. +pub async fn load_messages_from_disk( + partition_store: &RefCell, + namespace: &IggyNamespace, + start_offset: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Get segment range containing the requested offset + let segment_range = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let segments = partition_data.log.segments(); + if segments.is_empty() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let start = segments + .iter() + .rposition(|segment| segment.start_offset <= start_offset) + .unwrap_or(0); + let end = segments.len(); + start..end + }; + + let mut remaining_count = count; + let mut batches = IggyMessagesBatchSet::empty(); + let mut current_offset = start_offset; + + for idx in segment_range { + if remaining_count == 0 { + break; + } + + let (segment_start_offset, segment_end_offset) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let segment = &partition_data.log.segments()[idx]; + (segment.start_offset, segment.end_offset) + }; + + let offset = if current_offset < segment_start_offset { + segment_start_offset + } else { + current_offset + }; + + let mut end_offset = offset + (remaining_count - 1).max(1) as u64; + if end_offset > segment_end_offset { + end_offset = segment_end_offset; + } + + let messages = load_segment_messages( + partition_store, + namespace, + idx, + offset, + end_offset, + remaining_count, + segment_start_offset, + ) + .await?; + + let loaded_count = messages.count(); + if loaded_count > 0 { + batches.add_batch_set(messages); + remaining_count = remaining_count.saturating_sub(loaded_count); + current_offset = end_offset + 1; + } else { + break; + } + } + + Ok(batches) +} + +/// Load messages from a specific segment. +async fn load_segment_messages( + partition_store: &RefCell, + namespace: &IggyNamespace, + idx: usize, + start_offset: u64, + end_offset: u64, + count: u32, + segment_start_offset: u64, +) -> Result { + let relative_start_offset = (start_offset - segment_start_offset) as u32; + + // Check journal first for this segment's data + let journal_data = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let journal = partition_data.log.journal(); + let is_journal_empty = journal.is_empty(); + let journal_inner = journal.inner(); + let journal_first_offset = journal_inner.base_offset; + let journal_last_offset = journal_inner.current_offset; + + if !is_journal_empty + && start_offset >= journal_first_offset + && end_offset <= journal_last_offset + { + Some(journal.get(|batches| batches.get_by_offset(start_offset, count))) + } else { + None + } + }; + + if let Some(batches) = journal_data { + return Ok(batches); + } + + // Load from disk + let (index_reader, messages_reader, indexes) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let storages = partition_data.log.storages(); + if idx >= storages.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let index_reader = storages[idx] + .index_reader + .as_ref() + .expect("Index reader not initialized") + .clone(); + let messages_reader = storages[idx] + .messages_reader + .as_ref() + .expect("Messages reader not initialized") + .clone(); + let indexes_vec = partition_data.log.indexes(); + let indexes = indexes_vec + .get(idx) + .and_then(|opt| opt.as_ref()) + .map(|indexes| { + indexes + .slice_by_offset(relative_start_offset, count) + .unwrap_or_default() + }); + (index_reader, messages_reader, indexes) + }; + + let indexes_to_read = if let Some(indexes) = indexes { + if !indexes.is_empty() { + Some(indexes) + } else { + index_reader + .as_ref() + .load_from_disk_by_offset(relative_start_offset, count) + .await? + } + } else { + index_reader + .as_ref() + .load_from_disk_by_offset(relative_start_offset, count) + .await? + }; + + if indexes_to_read.is_none() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let indexes_to_read = indexes_to_read.unwrap(); + let batch = messages_reader + .as_ref() + .load_messages_from_disk(indexes_to_read) + .await?; + + batch.validate_checksums_and_offsets(start_offset)?; + + Ok(IggyMessagesBatchSet::from(batch)) +} + +/// Load messages from disk by timestamp. +async fn load_messages_from_disk_by_timestamp( + partition_store: &RefCell, + namespace: &IggyNamespace, + timestamp: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Find segment range that might contain messages >= timestamp + let segment_range = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let segments = partition_data.log.segments(); + if segments.is_empty() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let start = segments + .iter() + .position(|segment| segment.end_timestamp >= timestamp) + .unwrap_or(segments.len()); + + if start >= segments.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + start..segments.len() + }; + + let mut remaining_count = count; + let mut batches = IggyMessagesBatchSet::empty(); + + for idx in segment_range { + if remaining_count == 0 { + break; + } + + let segment_end_timestamp = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + partition_data.log.segments()[idx].end_timestamp + }; + + if segment_end_timestamp < timestamp { + continue; + } + + let messages = load_segment_messages_by_timestamp( + partition_store, + namespace, + idx, + timestamp, + remaining_count, + ) + .await?; + + let messages_count = messages.count(); + if messages_count == 0 { + continue; + } + + remaining_count = remaining_count.saturating_sub(messages_count); + batches.add_batch_set(messages); + } + + Ok(batches) +} + +/// Load messages from a specific segment by timestamp. +async fn load_segment_messages_by_timestamp( + partition_store: &RefCell, + namespace: &IggyNamespace, + idx: usize, + timestamp: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Check journal first + let journal_data = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let journal = partition_data.log.journal(); + let is_journal_empty = journal.is_empty(); + let journal_inner = journal.inner(); + let journal_first_timestamp = journal_inner.first_timestamp; + let journal_last_timestamp = journal_inner.end_timestamp; + + if !is_journal_empty + && timestamp >= journal_first_timestamp + && timestamp <= journal_last_timestamp + { + Some(journal.get(|batches| batches.get_by_timestamp(timestamp, count))) + } else { + None + } + }; + + if let Some(batches) = journal_data { + return Ok(batches); + } + + // Load from disk + let (index_reader, messages_reader, indexes) = { + let store = partition_store.borrow(); + let partition_data = store + .get(namespace) + .expect("partition_store: partition must exist"); + + let storages = partition_data.log.storages(); + if idx >= storages.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let index_reader = storages[idx] + .index_reader + .as_ref() + .expect("Index reader not initialized") + .clone(); + let messages_reader = storages[idx] + .messages_reader + .as_ref() + .expect("Messages reader not initialized") + .clone(); + let indexes_vec = partition_data.log.indexes(); + let indexes = indexes_vec + .get(idx) + .and_then(|opt| opt.as_ref()) + .map(|indexes| { + indexes + .slice_by_timestamp(timestamp, count) + .unwrap_or_default() + }); + (index_reader, messages_reader, indexes) + }; + + let indexes_to_read = if let Some(indexes) = indexes { + if !indexes.is_empty() { + Some(indexes) + } else { + index_reader + .as_ref() + .load_from_disk_by_timestamp(timestamp, count) + .await? + } + } else { + index_reader + .as_ref() + .load_from_disk_by_timestamp(timestamp, count) + .await? + }; + + if indexes_to_read.is_none() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let indexes_to_read = indexes_to_read.unwrap(); + let batch = messages_reader + .as_ref() + .load_messages_from_disk(indexes_to_read) + .await?; + + Ok(IggyMessagesBatchSet::from(batch)) +} diff --git a/core/server/src/streaming/partitions/consumer_offset.rs b/core/server/src/streaming/partitions/consumer_offset.rs index c081cc0a20..72d7ecfc17 100644 --- a/core/server/src/streaming/partitions/consumer_offset.rs +++ b/core/server/src/streaming/partitions/consumer_offset.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -use std::sync::atomic::AtomicU64; - use crate::streaming::polling_consumer::ConsumerGroupId; use iggy_common::ConsumerKind; +use std::sync::atomic::AtomicU64; #[derive(Debug)] pub struct ConsumerOffset { diff --git a/core/server/src/streaming/partitions/helpers.rs b/core/server/src/streaming/partitions/helpers.rs index 6d14f89718..a7a0bc29bb 100644 --- a/core/server/src/streaming/partitions/helpers.rs +++ b/core/server/src/streaming/partitions/helpers.rs @@ -15,261 +15,11 @@ // specific language governing permissions and limitations // under the License. -use err_trail::ErrContext; -use iggy_common::{ConsumerOffsetInfo, Identifier, IggyByteSize, IggyError}; -use std::{ - ops::AsyncFnOnce, - sync::{Arc, atomic::Ordering}, -}; - use crate::{ - configs::{cache_indexes::CacheIndexesConfig, system::SystemConfig}, - slab::{ - partitions::{self, Partitions}, - traits_ext::{ - ComponentsById, Delete, EntityComponentSystem, EntityMarker, Insert, IntoComponents, - }, - }, - streaming::{ - deduplication::message_deduplicator::MessageDeduplicator, - partitions::{ - consumer_offset::ConsumerOffset, - journal::Journal, - partition::{self, PartitionRef, PartitionRefMut}, - storage, - }, - polling_consumer::ConsumerGroupId, - segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet, storage::Storage}, - }, + configs::system::SystemConfig, + streaming::deduplication::message_deduplicator::MessageDeduplicator, }; -pub fn get_partition_ids() -> impl FnOnce(&Partitions) -> Vec { - |partitions| { - partitions.with_components(|components| { - let (root, ..) = components.into_components(); - root.iter() - .map(|(_, partition)| partition.id()) - .collect::>() - }) - } -} - -pub fn delete_partitions( - partitions_count: u32, -) -> impl FnOnce(&mut Partitions) -> Vec { - move |partitions| { - let current_count = partitions.len() as u32; - let partitions_to_delete = partitions_count.min(current_count); - let start_idx = (current_count - partitions_to_delete) as usize; - let range = start_idx..current_count as usize; - range - .map(|idx| { - let partition = partitions.delete(idx); - assert_eq!(partition.id(), idx); - partition - }) - .collect() - } -} - -pub fn insert_partition( - partition: partition::Partition, -) -> impl FnOnce(&mut Partitions) -> partitions::ContainerId { - move |partitions| partitions.insert(partition) -} - -pub fn purge_partitions_mem() -> impl FnOnce(&Partitions) { - |partitions| { - partitions.with_components(|components| { - let (.., stats, _, offsets, _, _, _) = components.into_components(); - for (offset, stat) in offsets - .iter() - .map(|(_, o)| o) - .zip(stats.iter().map(|(_, s)| s)) - { - offset.store(0, Ordering::Relaxed); - stat.zero_out_all(); - } - }) - } -} - -pub fn purge_consumer_offsets() -> impl FnOnce(&Partitions) -> (Vec, Vec) { - |partitions| { - partitions.with_components(|components| { - let (.., consumer_offsets, cg_offsets, _) = components.into_components(); - - let mut consumer_offset_paths = Vec::new(); - let mut consumer_group_offset_paths = Vec::new(); - - // Collect paths and clear consumer offsets - for (_, consumer_offset) in consumer_offsets { - let hdl = consumer_offset.pin(); - for item in hdl.values() { - consumer_offset_paths.push(item.path.clone()); - } - hdl.clear(); // Clear the hashmap - } - - // Collect paths and clear consumer group offsets - for (_, cg_offset) in cg_offsets { - let hdl = cg_offset.pin(); - for item in hdl.values() { - consumer_group_offset_paths.push(item.path.clone()); - } - hdl.clear(); // Clear the hashmap - } - - (consumer_offset_paths, consumer_group_offset_paths) - }) - } -} - -pub fn get_consumer_offset( - id: usize, -) -> impl FnOnce(ComponentsById) -> Option { - move |(root, _, _, current_offset, offsets, _, _)| { - offsets.pin().get(&id).map(|item| ConsumerOffsetInfo { - partition_id: root.id() as u32, - current_offset: current_offset.load(Ordering::Relaxed), - stored_offset: item.offset.load(Ordering::Relaxed), - }) - } -} - -pub fn get_consumer_group_offset( - consumer_group_id: ConsumerGroupId, -) -> impl FnOnce(ComponentsById) -> Option { - move |(root, _, _, current_offset, _, offsets, _)| { - offsets - .pin() - .get(&consumer_group_id) - .map(|item| ConsumerOffsetInfo { - partition_id: root.id() as u32, - current_offset: current_offset.load(Ordering::Relaxed), - stored_offset: item.offset.load(Ordering::Relaxed), - }) - } -} - -pub fn store_consumer_offset( - id: usize, - stream_id: usize, - topic_id: usize, - partition_id: usize, - offset: u64, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) { - move |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let item = hdl.get_or_insert( - id, - ConsumerOffset::default_for_consumer( - id as u32, - &config.get_consumer_offsets_path(stream_id, topic_id, partition_id), - ), - ); - item.offset.store(offset, Ordering::Relaxed); - } -} - -pub fn delete_consumer_offset( - id: usize, -) -> impl FnOnce(ComponentsById) -> Result { - move |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let offset = hdl - .remove(&id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(id))?; - Ok(offset.path.clone()) - } -} - -pub fn persist_consumer_offset_to_disk( - id: usize, -) -> impl AsyncFnOnce(ComponentsById) -> Result<(), IggyError> { - async move |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(&id) - .expect("persist_consumer_offset_to_disk: offset not found"); - let offset = item.offset.load(Ordering::Relaxed); - storage::persist_offset(&item.path, offset).await - } -} - -pub fn delete_consumer_offset_from_disk( - id: usize, -) -> impl AsyncFnOnce(ComponentsById) -> Result<(), IggyError> { - async move |(.., offsets, _, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(&id) - .expect("delete_consumer_offset_from_disk: offset not found"); - let path = &item.path; - storage::delete_persisted_offset(path).await - } -} - -pub fn store_consumer_group_offset( - consumer_group_id: ConsumerGroupId, - stream_id: usize, - topic_id: usize, - partition_id: usize, - offset: u64, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) { - move |(.., offsets, _)| { - let hdl = offsets.pin(); - let item = hdl.get_or_insert( - consumer_group_id, - ConsumerOffset::default_for_consumer_group( - consumer_group_id, - &config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id), - ), - ); - item.offset.store(offset, Ordering::Relaxed); - } -} - -pub fn delete_consumer_group_offset( - consumer_group_id: ConsumerGroupId, -) -> impl FnOnce(ComponentsById) -> Result { - move |(.., offsets, _)| { - let hdl = offsets.pin(); - let offset = hdl - .remove(&consumer_group_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(consumer_group_id.0))?; - Ok(offset.path.clone()) - } -} - -pub fn persist_consumer_group_offset_to_disk( - consumer_group_id: ConsumerGroupId, -) -> impl AsyncFnOnce(ComponentsById) -> Result<(), IggyError> { - async move |(.., offsets, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(&consumer_group_id) - .expect("persist_consumer_group_offset_to_disk: offset not found"); - let offset = item.offset.load(Ordering::Relaxed); - storage::persist_offset(&item.path, offset).await - } -} - -pub fn delete_consumer_group_offset_from_disk( - consumer_group_id: ConsumerGroupId, -) -> impl AsyncFnOnce(ComponentsById) -> Result<(), IggyError> { - async move |(.., offsets, _)| { - let hdl = offsets.pin(); - let item = hdl - .get(&consumer_group_id) - .expect("delete_consumer_group_offset_from_disk: offset not found"); - let path = &item.path; - storage::delete_persisted_offset(path).await - } -} - pub fn create_message_deduplicator(config: &SystemConfig) -> Option { if !config.message_deduplication.enabled { return None; @@ -287,261 +37,3 @@ pub fn create_message_deduplicator(config: &SystemConfig) -> Option impl FnOnce(ComponentsById) -> std::ops::Range { - move |(.., log)| { - let segments = log.segments(); - - if segments.is_empty() { - return 0..0; - } - - let start = segments - .iter() - .rposition(|segment| segment.start_offset <= offset) - .unwrap_or(0); - - let end = segments.len(); - start..end - } -} - -pub fn get_segment_range_by_timestamp( - timestamp: u64, -) -> impl FnOnce(ComponentsById) -> Result, IggyError> { - move |(.., log)| -> Result, IggyError> { - let segments = log.segments(); - - if segments.is_empty() { - return Ok(0..0); - } - - let start = segments - .iter() - .enumerate() - .filter(|(_, segment)| segment.end_timestamp >= timestamp) - .map(|(index, _)| index) - .next() - .ok_or(IggyError::TimestampOutOfRange(timestamp))?; - let end = segments.len(); - Ok(start..end) - } -} - -pub async fn load_messages_from_disk_by_timestamp( - storage: &Storage, - index: &Option, - timestamp: u64, - count: u32, -) -> Result { - let indexes_to_read = if let Some(indexes) = index { - if !indexes.is_empty() { - indexes.slice_by_timestamp(timestamp, count) - } else { - storage - .index_reader - .as_ref() - .expect("Index reader not initialized") - .load_from_disk_by_timestamp(timestamp, count) - .await? - } - } else { - storage - .index_reader - .as_ref() - .expect("Index reader not initialized") - .load_from_disk_by_timestamp(timestamp, count) - .await? - }; - - if indexes_to_read.is_none() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let indexes_to_read = indexes_to_read.unwrap(); - - let batch = storage - .messages_reader - .as_ref() - .expect("Messages reader not initialized") - .load_messages_from_disk(indexes_to_read) - .await - .error(|e: &IggyError| format!("Failed to load messages from disk by timestamp: {e}"))?; - - Ok(IggyMessagesBatchSet::from(batch)) -} - -pub fn calculate_current_offset() -> impl FnOnce(ComponentsById) -> u64 { - |(root, _, _, offset, ..)| { - if !root.should_increment_offset() { - 0 - } else { - offset.load(Ordering::Relaxed) + 1 - } - } -} - -pub fn get_segment_start_offset_and_deduplicator() --> impl FnOnce(ComponentsById) -> (u64, Option>) { - move |(.., deduplicator, _, _, _, log)| { - let segment = log.active_segment(); - (segment.start_offset, deduplicator.clone()) - } -} - -pub fn append_to_journal( - current_offset: u64, - batch: IggyMessagesBatchMut, -) -> impl FnOnce(ComponentsById) -> Result<(u32, u32), IggyError> { - move |(root, stats, _, offset, .., log)| { - let segment = log.active_segment_mut(); - - if segment.end_offset == 0 { - segment.start_timestamp = batch.first_timestamp().unwrap(); - } - - let batch_messages_size = batch.size(); - let batch_messages_count = batch.count(); - - stats.increment_size_bytes(batch_messages_size as u64); - stats.increment_messages_count(batch_messages_count as u64); - - segment.end_timestamp = batch.last_timestamp().unwrap(); - segment.end_offset = batch.last_offset().unwrap(); - - let (journal_messages_count, journal_size) = log.journal_mut().append(batch)?; - - let last_offset = if batch_messages_count == 0 { - current_offset - } else { - current_offset + batch_messages_count as u64 - 1 - }; - - if root.should_increment_offset() { - offset.store(last_offset, Ordering::Relaxed); - } else { - root.set_should_increment_offset(true); - offset.store(last_offset, Ordering::Relaxed); - } - log.active_segment_mut().current_position += batch_messages_size; - - Ok((journal_messages_count, journal_size)) - } -} - -pub fn commit_journal() -> impl FnOnce(ComponentsById) -> IggyMessagesBatchSet { - |(.., log)| { - let batches = log.journal_mut().commit(); - log.ensure_indexes(); - batches.append_indexes_to(log.active_indexes_mut().unwrap()); - batches - } -} - -pub fn is_segment_full() -> impl FnOnce(ComponentsById) -> bool { - |(.., log)| log.active_segment().is_full() -} - -pub fn persist_reason( - unsaved_messages_count_exceeded: bool, - unsaved_messages_size_exceeded: bool, - journal_messages_count: u32, - journal_size: u32, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) -> String { - move |(.., log)| { - if unsaved_messages_count_exceeded { - format!( - "unsaved messages count exceeded: {}, max from config: {}", - journal_messages_count, config.partition.messages_required_to_save, - ) - } else if unsaved_messages_size_exceeded { - format!( - "unsaved messages size exceeded: {}, max from config: {}", - journal_size, config.partition.size_of_messages_required_to_save, - ) - } else { - format!( - "segment is full, current size: {}, max from config: {}", - log.active_segment().size, - &config.segment.size, - ) - } - } -} - -pub fn persist_batch( - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - batches: IggyMessagesBatchSet, - reason: String, -) -> impl AsyncFnOnce(ComponentsById) -> Result<(IggyByteSize, u32), IggyError> { - async move |(.., log)| { - tracing::trace!( - "Persisting messages on disk for stream ID: {}, topic ID: {}, partition ID: {} because {}...", - stream_id, - topic_id, - partition_id, - reason - ); - - let batch_count = batches.count(); - let batch_size = batches.size(); - - let storage = log.active_storage(); - let saved = storage - .messages_writer - .as_ref() - .expect("Messages writer not initialized") - .save_batch_set(batches) - .await - .error(|e: &IggyError| { - let segment = log.active_segment(); - format!( - "Failed to save batch of {batch_count} messages \ - ({batch_size} bytes) to {segment}. {e}", - ) - })?; - - let unsaved_indexes_slice = log.active_indexes().unwrap().unsaved_slice(); - let len = unsaved_indexes_slice.len(); - storage - .index_writer - .as_ref() - .expect("Index writer not initialized") - .save_indexes(unsaved_indexes_slice) - .await - .error(|e: &IggyError| { - let segment = log.active_segment(); - format!("Failed to save index of {len} indexes to {segment}. {e}",) - })?; - - tracing::trace!( - "Persisted {} messages on disk for stream ID: {}, topic ID: {}, for partition with ID: {}, total bytes written: {}.", - batch_count, - stream_id, - topic_id, - partition_id, - saved - ); - - Ok((saved, batch_count)) - } -} - -pub fn update_index_and_increment_stats( - saved: IggyByteSize, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) { - move |(.., log)| { - let segment = log.active_segment_mut(); - segment.size = IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64()); - log.active_indexes_mut().unwrap().mark_saved(); - if config.segment.cache_indexes == CacheIndexesConfig::None { - log.active_indexes_mut().unwrap().clear(); - } - } -} diff --git a/core/server/src/streaming/partitions/partition.rs b/core/server/src/streaming/partitions/partition.rs index 7a07b8f331..40e5b4a4fa 100644 --- a/core/server/src/streaming/partitions/partition.rs +++ b/core/server/src/streaming/partitions/partition.rs @@ -15,27 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - configs::system::SystemConfig, - slab::{ - partitions, - streams::Streams, - traits_ext::{EntityMarker, IntoComponents, IntoComponentsById}, - }, - streaming::{ - self, - deduplication::message_deduplicator::MessageDeduplicator, - partitions::{ - consumer_offset, helpers::create_message_deduplicator, journal::MemoryMessageJournal, - log::SegmentedLog, - }, - polling_consumer::ConsumerGroupId, - stats::{PartitionStats, TopicStats}, - }, -}; -use iggy_common::{Identifier, IggyTimestamp}; -use slab::Slab; -use std::sync::{Arc, atomic::AtomicU64}; +use crate::streaming::{partitions::consumer_offset, polling_consumer::ConsumerGroupId}; #[derive(Debug, Clone)] pub struct ConsumerOffsets(papaya::HashMap); @@ -100,355 +80,3 @@ impl std::ops::DerefMut for ConsumerGroupOffsets { &mut self.0 } } - -#[derive(Debug)] -pub struct Partition { - root: PartitionRoot, - stats: Arc, - message_deduplicator: Option>, - offset: Arc, - consumer_offset: Arc, - consumer_group_offset: Arc, - log: SegmentedLog, -} - -impl Partition { - #[allow(clippy::too_many_arguments)] - pub fn new( - created_at: IggyTimestamp, - should_increment_offset: bool, - stats: Arc, - message_deduplicator: Option, - offset: Arc, - consumer_offset: Arc, - consumer_group_offset: Arc, - log: SegmentedLog, - ) -> Self { - let root = PartitionRoot::new(created_at, should_increment_offset); - let message_deduplicator = message_deduplicator.map(Arc::new); - Self { - root, - stats, - message_deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - } - } - - pub fn new_with_components( - root: PartitionRoot, - stats: Arc, - message_deduplicator: Option>, - offset: Arc, - consumer_offset: Arc, - consumer_group_offset: Arc, - log: SegmentedLog, - ) -> Self { - Self { - root, - stats, - message_deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - } - } - - pub fn stats(&self) -> &PartitionStats { - &self.stats - } -} - -impl Clone for Partition { - fn clone(&self) -> Self { - Self { - root: self.root.clone(), - stats: Arc::clone(&self.stats), - message_deduplicator: self.message_deduplicator.clone(), - offset: Arc::clone(&self.offset), - consumer_offset: Arc::clone(&self.consumer_offset), - consumer_group_offset: Arc::clone(&self.consumer_group_offset), - log: Default::default(), - } - } -} - -impl EntityMarker for Partition { - type Idx = partitions::ContainerId; - - fn id(&self) -> Self::Idx { - self.root.id - } - - fn update_id(&mut self, id: Self::Idx) { - self.root.id = id; - } -} - -impl IntoComponents for Partition { - type Components = ( - PartitionRoot, - Arc, - Option>, - Arc, - Arc, - Arc, - SegmentedLog, - ); - - fn into_components(self) -> Self::Components { - ( - self.root, - self.stats, - self.message_deduplicator, - self.offset, - self.consumer_offset, - self.consumer_group_offset, - self.log, - ) - } -} - -#[derive(Default, Debug, Clone)] -pub struct PartitionRoot { - id: usize, - created_at: IggyTimestamp, - should_increment_offset: bool, -} - -impl PartitionRoot { - pub fn new(created_at: IggyTimestamp, should_increment_offset: bool) -> Self { - Self { - id: 0, - created_at, - should_increment_offset, - } - } - - pub fn id(&self) -> usize { - self.id - } - - pub fn update_id(&mut self, id: usize) { - self.id = id; - } - - pub fn created_at(&self) -> IggyTimestamp { - self.created_at - } - - pub fn should_increment_offset(&self) -> bool { - self.should_increment_offset - } - - pub fn set_should_increment_offset(&mut self, value: bool) { - self.should_increment_offset = value; - } -} - -pub struct PartitionRef<'a> { - root: &'a Slab, - stats: &'a Slab>, - message_deduplicator: &'a Slab>>, - offset: &'a Slab>, - consumer_offset: &'a Slab>, - consumer_group_offset: &'a Slab>, - log: &'a Slab>, -} - -impl<'a> PartitionRef<'a> { - pub fn new( - root: &'a Slab, - stats: &'a Slab>, - message_deduplicator: &'a Slab>>, - offset: &'a Slab>, - consumer_offset: &'a Slab>, - consumer_group_offset: &'a Slab>, - log: &'a Slab>, - ) -> Self { - Self { - root, - stats, - message_deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - } - } -} - -impl<'a> IntoComponents for PartitionRef<'a> { - type Components = ( - &'a Slab, - &'a Slab>, - &'a Slab>>, - &'a Slab>, - &'a Slab>, - &'a Slab>, - &'a Slab>, - ); - - fn into_components(self) -> Self::Components { - ( - self.root, - self.stats, - self.message_deduplicator, - self.offset, - self.consumer_offset, - self.consumer_group_offset, - self.log, - ) - } -} - -impl<'a> IntoComponentsById for PartitionRef<'a> { - type Idx = partitions::ContainerId; - type Output = ( - &'a PartitionRoot, - &'a Arc, - &'a Option>, - &'a Arc, - &'a Arc, - &'a Arc, - &'a SegmentedLog, - ); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - ( - &self.root[index], - &self.stats[index], - &self.message_deduplicator[index], - &self.offset[index], - &self.consumer_offset[index], - &self.consumer_group_offset[index], - &self.log[index], - ) - } -} - -pub struct PartitionRefMut<'a> { - root: &'a mut Slab, - stats: &'a mut Slab>, - message_deduplicator: &'a mut Slab>>, - offset: &'a mut Slab>, - consumer_offset: &'a mut Slab>, - consumer_group_offset: &'a mut Slab>, - log: &'a mut Slab>, -} - -impl<'a> PartitionRefMut<'a> { - pub fn new( - root: &'a mut Slab, - stats: &'a mut Slab>, - message_deduplicator: &'a mut Slab>>, - offset: &'a mut Slab>, - consumer_offset: &'a mut Slab>, - consumer_group_offset: &'a mut Slab>, - log: &'a mut Slab>, - ) -> Self { - Self { - root, - stats, - message_deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - } - } -} - -impl<'a> IntoComponents for PartitionRefMut<'a> { - type Components = ( - &'a mut Slab, - &'a mut Slab>, - &'a mut Slab>>, - &'a mut Slab>, - &'a mut Slab>, - &'a mut Slab>, - &'a mut Slab>, - ); - - fn into_components(self) -> Self::Components { - ( - self.root, - self.stats, - self.message_deduplicator, - self.offset, - self.consumer_offset, - self.consumer_group_offset, - self.log, - ) - } -} - -impl<'a> IntoComponentsById for PartitionRefMut<'a> { - type Idx = partitions::ContainerId; - type Output = ( - &'a mut PartitionRoot, - &'a mut Arc, - &'a mut Option>, - &'a mut Arc, - &'a mut Arc, - &'a mut Arc, - &'a mut SegmentedLog, - ); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - ( - &mut self.root[index], - &mut self.stats[index], - &mut self.message_deduplicator[index], - &mut self.offset[index], - &mut self.consumer_offset[index], - &mut self.consumer_group_offset[index], - &mut self.log[index], - ) - } -} - -pub fn create_and_insert_partitions_mem( - streams: &Streams, - stream_id: &Identifier, - topic_id: &Identifier, - parent_stats: Arc, - partitions_count: u32, - config: &SystemConfig, -) -> Vec { - let range = 0..partitions_count as usize; - let created_at = IggyTimestamp::now(); - range - .map(|_| { - // Areczkuuuu. - let stats = Arc::new(PartitionStats::new(parent_stats.clone())); - let should_increment_offset = false; - let deduplicator = create_message_deduplicator(config); - let offset = Arc::new(AtomicU64::new(0)); - let consumer_offset = Arc::new(ConsumerOffsets::with_capacity(2137)); - let consumer_group_offset = Arc::new(ConsumerGroupOffsets::with_capacity(2137)); - let log = Default::default(); - - let mut partition = Partition::new( - created_at, - should_increment_offset, - stats, - deduplicator, - offset, - consumer_offset, - consumer_group_offset, - log, - ); - let id = streams.with_partitions_mut( - stream_id, - topic_id, - streaming::partitions::helpers::insert_partition(partition.clone()), - ); - partition.update_id(id); - partition - }) - .collect() -} diff --git a/core/server/src/streaming/stats/mod.rs b/core/server/src/streaming/stats/mod.rs index 89d5946655..3493aec570 100644 --- a/core/server/src/streaming/stats/mod.rs +++ b/core/server/src/streaming/stats/mod.rs @@ -223,6 +223,7 @@ pub struct PartitionStats { messages_count: AtomicU64, size_bytes: AtomicU64, segments_count: AtomicU32, + current_offset: AtomicU64, } impl PartitionStats { @@ -232,6 +233,7 @@ impl PartitionStats { messages_count: AtomicU64::new(0), size_bytes: AtomicU64::new(0), segments_count: AtomicU32::new(0), + current_offset: AtomicU64::new(0), } } @@ -309,6 +311,14 @@ impl PartitionStats { self.segments_count.load(Ordering::Relaxed) } + pub fn current_offset(&self) -> u64 { + self.current_offset.load(Ordering::Relaxed) + } + + pub fn set_current_offset(&self, offset: u64) { + self.current_offset.store(offset, Ordering::Relaxed); + } + pub fn zero_out_parent_size_bytes(&self) { self.parent.zero_out_size_bytes(); } @@ -340,10 +350,15 @@ impl PartitionStats { self.zero_out_parent_segments_count(); } + pub fn zero_out_current_offset(&self) { + self.current_offset.store(0, Ordering::Relaxed); + } + pub fn zero_out_all(&self) { self.zero_out_size_bytes(); self.zero_out_messages_count(); self.zero_out_segments_count(); + self.zero_out_current_offset(); self.zero_out_parent_all(); } } diff --git a/core/server/src/streaming/streams/helpers.rs b/core/server/src/streaming/streams/helpers.rs deleted file mode 100644 index 1ffad1227d..0000000000 --- a/core/server/src/streaming/streams/helpers.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - configs::system::SystemConfig, - slab::{ - streams, - traits_ext::{ComponentsById, EntityComponentSystem, IntoComponents}, - }, - streaming::{ - partitions, - streams::stream::{StreamRef, StreamRefMut}, - }, -}; -use iggy_common::Identifier; - -pub fn get_stream_id() -> impl FnOnce(ComponentsById) -> streams::ContainerId { - |(root, _)| root.id() -} - -pub fn get_stream_name() -> impl FnOnce(ComponentsById) -> String { - |(root, _)| root.name().clone() -} - -pub fn update_stream_name(name: String) -> impl FnOnce(ComponentsById) { - move |(mut root, _)| { - root.set_name(name); - } -} - -pub fn get_topic_ids() -> impl FnOnce(ComponentsById) -> Vec { - |(root, _)| { - root.topics().with_components(|components| { - let (topic_roots, ..) = components.into_components(); - topic_roots - .iter() - .map(|(_, topic)| topic.id()) - .collect::>() - }) - } -} - -pub fn store_consumer_offset( - consumer_id: usize, - topic_id: &Identifier, - partition_id: usize, - offset: u64, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) { - move |(root, ..)| { - let stream_id = root.id(); - root.topics().with_topic_by_id(topic_id, |(root, ..)| { - let topic_id = root.id(); - root.partitions().with_components_by_id( - partition_id, - partitions::helpers::store_consumer_offset( - consumer_id, - stream_id, - topic_id, - partition_id, - offset, - config, - ), - ) - }) - } -} - -pub fn store_consumer_group_offset( - consumer_group_id: crate::streaming::polling_consumer::ConsumerGroupId, - topic_id: &Identifier, - partition_id: usize, - offset: u64, - config: &SystemConfig, -) -> impl FnOnce(ComponentsById) { - move |(root, ..)| { - let stream_id = root.id(); - root.topics().with_topic_by_id(topic_id, |(root, ..)| { - let topic_id = root.id(); - root.partitions().with_components_by_id( - partition_id, - partitions::helpers::store_consumer_group_offset( - consumer_group_id, - stream_id, - topic_id, - partition_id, - offset, - config, - ), - ) - }) - } -} diff --git a/core/server/src/streaming/streams/mod.rs b/core/server/src/streaming/streams/mod.rs index cf477c8b1b..a41b150e26 100644 --- a/core/server/src/streaming/streams/mod.rs +++ b/core/server/src/streaming/streams/mod.rs @@ -16,8 +16,6 @@ * under the License. */ -pub mod helpers; pub mod storage; -pub mod stream; pub const COMPONENT: &str = "STREAMING_STREAMS"; diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs index c76fbbd039..5068c1d1e2 100644 --- a/core/server/src/streaming/streams/storage.rs +++ b/core/server/src/streaming/streams/storage.rs @@ -15,9 +15,6 @@ // specific language governing permissions and limitations // under the License. -use crate::slab::traits_ext::{DeleteCell, EntityComponentSystem, EntityMarker, IntoComponents}; -use crate::streaming::streams::stream; -use crate::streaming::topics::storage::delete_topic_from_disk; use crate::{configs::system::SystemConfig, io::fs_utils::remove_dir_all}; use compio::fs::create_dir_all; use iggy_common::IggyError; @@ -40,26 +37,24 @@ pub async fn create_stream_file_hierarchy( Ok(()) } -pub async fn delete_stream_from_disk( - stream: &mut stream::Stream, +/// Delete stream directory using only IDs. +/// Does not require slab access - works with SharedMetadata. +/// topics_with_partitions: Vec<(topic_id, Vec)> +pub async fn delete_stream_directory( + stream_id: usize, + topics_with_partitions: &[(usize, Vec)], config: &SystemConfig, ) -> Result<(), IggyError> { - let stream_id = stream.id(); + use crate::streaming::topics::storage::delete_topic_directory; + let stream_path = config.get_stream_path(stream_id); if !Path::new(&stream_path).exists() { return Err(IggyError::StreamDirectoryNotFound(stream_path)); } - // Gather all topic ids. - let ids = stream.root().topics().with_components(|topics| { - let (roots, ..) = topics.into_components(); - roots.iter().map(|(_, root)| root.id()).collect::>() - }); - - // Delete all topics from the stream. - for id in ids { - let mut topic = stream.root_mut().topics_mut().delete(id); - delete_topic_from_disk(stream_id, &mut topic, config).await?; + // Delete all topics + for (topic_id, partition_ids) in topics_with_partitions { + delete_topic_directory(stream_id, *topic_id, partition_ids, config).await?; } remove_dir_all(&stream_path) diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs deleted file mode 100644 index f2e4c18fec..0000000000 --- a/core/server/src/streaming/streams/stream.rs +++ /dev/null @@ -1,225 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::IggyTimestamp; -use slab::Slab; -use std::{ - cell::{Ref, RefMut}, - sync::Arc, -}; - -use crate::{ - slab::{ - Keyed, - streams::{self, Streams}, - topics::Topics, - traits_ext::{EntityMarker, InsertCell, IntoComponents, IntoComponentsById}, - }, - streaming::stats::StreamStats, -}; - -#[derive(Debug, Clone)] -pub struct StreamRoot { - id: usize, - name: String, - created_at: IggyTimestamp, - topics: Topics, -} - -impl Keyed for StreamRoot { - type Key = String; - - fn key(&self) -> &Self::Key { - &self.name - } -} - -impl StreamRoot { - pub fn new(name: String, created_at: IggyTimestamp) -> Self { - Self { - id: 0, - name, - created_at, - topics: Topics::default(), - } - } - - pub fn update_id(&mut self, id: usize) { - self.id = id; - } - - pub fn id(&self) -> usize { - self.id - } - - pub fn name(&self) -> &String { - &self.name - } - - pub fn set_name(&mut self, name: String) { - self.name = name; - } - - pub fn topics_count(&self) -> usize { - self.topics.len() - } - - pub fn remove_topics(&mut self) -> Topics { - std::mem::take(&mut self.topics) - } - - pub fn topics(&self) -> &Topics { - &self.topics - } - - pub fn topics_mut(&mut self) -> &mut Topics { - &mut self.topics - } - - pub fn set_topics(&mut self, topics: Topics) { - self.topics = topics; - } - - pub fn created_at(&self) -> IggyTimestamp { - self.created_at - } -} - -#[derive(Debug, Clone)] -pub struct Stream { - root: StreamRoot, - stats: Arc, -} - -impl IntoComponents for Stream { - type Components = (StreamRoot, Arc); - - fn into_components(self) -> Self::Components { - (self.root, self.stats) - } -} - -impl EntityMarker for Stream { - type Idx = streams::ContainerId; - - fn id(&self) -> Self::Idx { - self.root.id - } - - fn update_id(&mut self, id: Self::Idx) { - self.root.id = id; - } -} - -impl Stream { - pub fn new(name: String, stats: Arc, created_at: IggyTimestamp) -> Self { - let root = StreamRoot::new(name, created_at); - Self { root, stats } - } - - pub fn new_with_components(root: StreamRoot, stats: Arc) -> Self { - Self { root, stats } - } - - pub fn stats(&self) -> &Arc { - &self.stats - } - - pub fn root(&self) -> &StreamRoot { - &self.root - } - - pub fn root_mut(&mut self) -> &mut StreamRoot { - &mut self.root - } -} - -pub struct StreamRef<'a> { - root: Ref<'a, Slab>, - stats: Ref<'a, Slab>>, -} - -impl<'a> StreamRef<'a> { - pub fn new(root: Ref<'a, Slab>, stats: Ref<'a, Slab>>) -> Self { - Self { root, stats } - } -} - -impl<'a> IntoComponents for StreamRef<'a> { - type Components = (Ref<'a, Slab>, Ref<'a, Slab>>); - - fn into_components(self) -> Self::Components { - (self.root, self.stats) - } -} - -impl<'a> IntoComponentsById for StreamRef<'a> { - type Idx = streams::ContainerId; - type Output = (Ref<'a, StreamRoot>, Ref<'a, Arc>); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = Ref::map(self.root, |r| &r[index]); - let stats = Ref::map(self.stats, |s| &s[index]); - (root, stats) - } -} - -pub struct StreamRefMut<'a> { - root: RefMut<'a, Slab>, - stats: RefMut<'a, Slab>>, -} - -impl<'a> StreamRefMut<'a> { - pub fn new( - root: RefMut<'a, Slab>, - stats: RefMut<'a, Slab>>, - ) -> Self { - Self { root, stats } - } -} - -impl<'a> IntoComponents for StreamRefMut<'a> { - type Components = ( - RefMut<'a, Slab>, - RefMut<'a, Slab>>, - ); - - fn into_components(self) -> Self::Components { - (self.root, self.stats) - } -} - -impl<'a> IntoComponentsById for StreamRefMut<'a> { - type Idx = streams::ContainerId; - type Output = (RefMut<'a, StreamRoot>, RefMut<'a, Arc>); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = RefMut::map(self.root, |r| &mut r[index]); - let stats = RefMut::map(self.stats, |s| &mut s[index]); - (root, stats) - } -} - -// TODO: Move this to a dedicated module. -pub fn create_and_insert_stream_mem(streams: &Streams, name: String) -> Stream { - let now = IggyTimestamp::now(); - let stats = Arc::new(Default::default()); - let mut stream = Stream::new(name, stats, now); - let id = streams.insert(stream.clone()); - stream.update_id(id); - stream -} diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs deleted file mode 100644 index c584e033d8..0000000000 --- a/core/server/src/streaming/topics/consumer_group.rs +++ /dev/null @@ -1,243 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::slab::{ - Keyed, consumer_groups, partitions, - traits_ext::{EntityMarker, IntoComponents, IntoComponentsById}, -}; -use arcshift::ArcShift; -use slab::Slab; -use std::sync::atomic::AtomicUsize; - -pub const MEMBERS_CAPACITY: usize = 128; - -#[derive(Debug, Clone)] -pub struct ConsumerGroupMembers { - inner: ArcShift>, -} - -impl ConsumerGroupMembers { - pub fn new(inner: ArcShift>) -> Self { - Self { inner } - } - - pub fn into_inner(self) -> ArcShift> { - self.inner - } - - pub fn inner(&self) -> &ArcShift> { - &self.inner - } - - pub fn inner_mut(&mut self) -> &mut ArcShift> { - &mut self.inner - } -} - -#[derive(Debug, Clone)] -pub struct ConsumerGroup { - root: ConsumerGroupRoot, - members: ConsumerGroupMembers, -} - -#[derive(Default, Debug, Clone)] -pub struct ConsumerGroupRoot { - id: usize, - name: String, - partitions: Vec, -} - -impl ConsumerGroupRoot { - pub fn disarray(self) -> (String, Vec) { - (self.name, self.partitions) - } - - pub fn partitions(&self) -> &Vec { - &self.partitions - } - - pub fn update_id(&mut self, id: usize) { - self.id = id; - } - - pub fn assign_partitions(&mut self, partitions: Vec) { - self.partitions = partitions; - } - - pub fn id(&self) -> consumer_groups::ContainerId { - self.id - } -} - -impl Keyed for ConsumerGroupRoot { - type Key = String; - - fn key(&self) -> &Self::Key { - &self.name - } -} - -impl EntityMarker for ConsumerGroup { - type Idx = consumer_groups::ContainerId; - - fn id(&self) -> Self::Idx { - self.root.id - } - - fn update_id(&mut self, id: Self::Idx) { - self.root.id = id; - } -} - -impl IntoComponents for ConsumerGroup { - type Components = (ConsumerGroupRoot, ConsumerGroupMembers); - - fn into_components(self) -> Self::Components { - (self.root, self.members) - } -} - -pub struct ConsumerGroupRef<'a> { - root: &'a Slab, - members: &'a Slab, -} - -impl<'a> ConsumerGroupRef<'a> { - pub fn new(root: &'a Slab, members: &'a Slab) -> Self { - Self { root, members } - } -} - -impl<'a> IntoComponents for ConsumerGroupRef<'a> { - type Components = (&'a Slab, &'a Slab); - - fn into_components(self) -> Self::Components { - (self.root, self.members) - } -} - -impl<'a> IntoComponentsById for ConsumerGroupRef<'a> { - type Idx = consumer_groups::ContainerId; - type Output = (&'a ConsumerGroupRoot, &'a ConsumerGroupMembers); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = &self.root[index]; - let members = &self.members[index]; - (root, members) - } -} - -pub struct ConsumerGroupRefMut<'a> { - root: &'a mut Slab, - members: &'a mut Slab, -} - -impl<'a> ConsumerGroupRefMut<'a> { - pub fn new( - root: &'a mut Slab, - members: &'a mut Slab, - ) -> Self { - Self { root, members } - } -} - -impl<'a> IntoComponents for ConsumerGroupRefMut<'a> { - type Components = ( - &'a mut Slab, - &'a mut Slab, - ); - - fn into_components(self) -> Self::Components { - (self.root, self.members) - } -} - -impl<'a> IntoComponentsById for ConsumerGroupRefMut<'a> { - type Idx = consumer_groups::ContainerId; - type Output = (&'a mut ConsumerGroupRoot, &'a mut ConsumerGroupMembers); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = &mut self.root[index]; - let members = &mut self.members[index]; - (root, members) - } -} - -impl ConsumerGroup { - pub fn new( - name: String, - members: ArcShift>, - partitions: Vec, - ) -> Self { - let root = ConsumerGroupRoot { - id: 0, - name, - partitions, - }; - let members = ConsumerGroupMembers { inner: members }; - Self { root, members } - } - - pub fn new_with_components(root: ConsumerGroupRoot, members: ConsumerGroupMembers) -> Self { - Self { root, members } - } - - pub fn partitions(&self) -> &Vec { - &self.root.partitions - } - - pub fn members(&self) -> &ConsumerGroupMembers { - &self.members - } -} - -#[derive(Debug)] -pub struct Member { - pub id: usize, - pub client_id: u32, - pub partitions: Vec, - pub current_partition_idx: AtomicUsize, -} - -impl Clone for Member { - fn clone(&self) -> Self { - Self { - id: self.id, - client_id: self.client_id, - partitions: self.partitions.clone(), - current_partition_idx: AtomicUsize::new(0), - } - } -} - -impl Member { - pub fn new(client_id: u32) -> Self { - Member { - id: 0, - client_id, - partitions: Vec::new(), - current_partition_idx: AtomicUsize::new(0), - } - } - - pub fn insert_into(self, container: &mut Slab) -> usize { - let idx = container.insert(self); - let member = &mut container[idx]; - member.id = idx; - idx - } -} diff --git a/core/server/src/streaming/topics/helpers.rs b/core/server/src/streaming/topics/helpers.rs index d5f77de9db..119222b427 100644 --- a/core/server/src/streaming/topics/helpers.rs +++ b/core/server/src/streaming/topics/helpers.rs @@ -15,85 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, atomic::Ordering}; - -use crate::{ - slab::{ - Keyed, - consumer_groups::{self, ConsumerGroups}, - topics::{self, Topics}, - traits_ext::{ - ComponentsById, Delete, DeleteCell, EntityComponentSystem, EntityComponentSystemMut, - EntityMarker, IntoComponents, - }, - }, - streaming::{ - stats::TopicStats, - topics::{ - consumer_group::{ - self, ConsumerGroupMembers, ConsumerGroupRef, ConsumerGroupRefMut, Member, - }, - topic::{Topic, TopicRef, TopicRefMut, TopicRoot}, - }, - }, -}; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, calculate_32}; -use slab::Slab; - -pub fn rename_index( - old_name: &::Key, - new_name: String, -) -> impl FnOnce(&Topics) { - move |topics| { - topics.with_index_mut(|index| { - // Rename the key inside of hashmap - let idx = index.remove(old_name).expect("Rename key: key not found"); - index.insert(new_name, idx); - }) - } -} - -// Topics -pub fn get_stats() -> impl FnOnce(ComponentsById) -> Arc { - |(_, _, stats)| stats.clone() -} - -pub fn get_topic_id() -> impl FnOnce(ComponentsById) -> topics::ContainerId { - |(root, _, _)| root.id() -} - -pub fn get_partition_ids() -> impl FnOnce(ComponentsById) -> Vec { - |(root, ..)| { - root.partitions().with_components(|components| { - let (roots, ..) = components.into_components(); - roots.iter().map(|(id, _)| id).collect() - }) - } -} - -pub fn get_message_expiry() -> impl FnOnce(ComponentsById) -> IggyExpiry { - |(root, _, _)| root.message_expiry() -} - -pub fn get_max_topic_size() -> impl FnOnce(ComponentsById) -> MaxTopicSize { - |(root, _, _)| root.max_topic_size() -} - -pub fn get_topic_size_info() -> impl FnOnce(ComponentsById) -> (bool, bool) { - |(root, _, stats)| { - let max_size = root.max_topic_size(); - let current_size = stats.size_bytes_inconsistent(); - let is_unlimited = matches!(max_size, MaxTopicSize::Unlimited); - let is_almost_full = if !is_unlimited { - let max_bytes = max_size.as_bytes_u64(); - // Consider "almost full" as 90% capacity - current_size >= (max_bytes * 9 / 10) - } else { - false - }; - (is_unlimited, is_almost_full) - } -} +use iggy_common::calculate_32; pub fn calculate_partition_id_by_messages_key_hash( upperbound: usize, @@ -109,228 +31,3 @@ pub fn calculate_partition_id_by_messages_key_hash( ); partition_id } - -pub fn delete_topic(topic_id: &Identifier) -> impl FnOnce(&Topics) -> Topic { - |container| { - let id = container.get_index(topic_id); - let topic = container.delete(id); - assert_eq!(topic.id(), id, "delete_topic: topic ID mismatch"); - topic - } -} - -pub fn exists(identifier: &Identifier) -> impl FnOnce(&Topics) -> bool { - |topics| topics.exists(identifier) -} - -pub fn cg_exists(identifier: &Identifier) -> impl FnOnce(ComponentsById) -> bool { - |(root, ..)| root.consumer_groups().exists(identifier) -} - -pub fn update_topic( - name: String, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, -) -> impl FnOnce(ComponentsById) -> (String, String) { - move |(mut root, _, _)| { - let old_name = root.name().clone(); - root.set_name(name.clone()); - root.set_message_expiry(message_expiry); - root.set_compression(compression_algorithm); - root.set_max_topic_size(max_topic_size); - root.set_replication_factor(replication_factor); - (old_name, name) - // TODO: Set message expiry for all partitions and segments. - } -} - -// Consumer Groups -pub fn get_consumer_group_id() --> impl FnOnce(ComponentsById) -> consumer_groups::ContainerId { - |(root, ..)| root.id() -} - -pub fn delete_consumer_group( - group_id: &Identifier, -) -> impl FnOnce(&mut ConsumerGroups) -> consumer_group::ConsumerGroup { - |container| { - let id = container.get_index(group_id); - let group = container.delete(id); - assert_eq!(group.id(), id, "delete_consumer_group: group ID mismatch"); - group - } -} - -pub fn join_consumer_group(client_id: u32) -> impl FnOnce(ComponentsById) { - move |(root, members)| { - let partitions = root.partitions(); - let id = root.id(); - add_member(id, members, partitions, client_id); - } -} - -pub fn leave_consumer_group( - client_id: u32, -) -> impl FnOnce(ComponentsById) -> Option { - move |(root, members)| { - let partitions = root.partitions(); - let id = root.id(); - delete_member(id, client_id, members, partitions) - } -} - -pub fn rebalance_consumer_group() -> impl FnOnce(ComponentsById) { - move |(root, members)| { - let partitions = root.partitions(); - let id = root.id(); - members.inner_mut().rcu(|existing_members| { - let mut new_members = mimic_members(existing_members); - assign_partitions_to_members(id, &mut new_members, partitions); - new_members - }); - } -} - -pub fn rebalance_consumer_groups( - partition_ids: &[usize], -) -> impl FnOnce(ComponentsById) { - move |(mut root, ..)| { - root.consumer_groups_mut() - .with_components_mut(|components| { - let (all_roots, all_members) = components.into_components(); - for ((_, consumer_group_root), (_, members)) in - all_roots.iter().zip(all_members.iter_mut()) - { - let id = consumer_group_root.id(); - members.inner_mut().rcu(|existing_members| { - let mut new_members = mimic_members(existing_members); - assign_partitions_to_members(id, &mut new_members, partition_ids); - new_members - }); - } - }); - } -} - -pub fn get_consumer_group_member_id( - client_id: u32, -) -> impl FnOnce(ComponentsById) -> Option { - move |(_, members)| { - members - .inner() - .shared_get() - .iter() - .find_map(|(_, member)| (member.client_id == client_id).then_some(member.id)) - } -} - -pub fn calculate_partition_id_unchecked( - member_id: usize, -) -> impl FnOnce(ComponentsById) -> Option { - move |(_, members)| { - let members = members.inner().shared_get(); - let member = &members[member_id]; - if member.partitions.is_empty() { - return None; - } - - let partitions_count = member.partitions.len(); - // It's OK to use `Relaxed` ordering, because we have 1-1 mapping between consumer and member. - // We allow only one consumer to access topic in a given shard - // therefore there is no contention on the member's current partition index. - let current = member - .current_partition_idx - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { - Some((current + 1) % partitions_count) - }) - .expect("fetch_and_update partition id for consumer group member"); - Some(member.partitions[current]) - } -} - -pub fn get_current_partition_id_unchecked( - member_id: usize, -) -> impl FnOnce(ComponentsById) -> Option { - move |(_, members)| { - let members = members.inner().shared_get(); - let member = &members[member_id]; - if member.partitions.is_empty() { - return None; - } - - let partition_idx = member.current_partition_idx.load(Ordering::Relaxed); - Some(member.partitions[partition_idx]) - } -} - -fn add_member(id: usize, members: &mut ConsumerGroupMembers, partitions: &[usize], client_id: u32) { - members.inner_mut().rcu(move |members| { - let mut members = mimic_members(members); - Member::new(client_id).insert_into(&mut members); - assign_partitions_to_members(id, &mut members, partitions); - members - }); -} - -fn delete_member( - id: usize, - client_id: u32, - members: &mut ConsumerGroupMembers, - partitions: &[usize], -) -> Option { - let mut member_id = None; - members.inner_mut().rcu(|inner| { - let member_ids: Vec = inner - .iter() - .filter_map(|(_, member)| (member.client_id == client_id).then_some(member.id)) - .collect(); - - let mut members = mimic_members(inner); - if !member_ids.is_empty() { - member_id = member_ids.first().cloned(); - for member_id in &member_ids { - members.remove(*member_id); - } - members.compact(|entry, _, idx| { - entry.id = idx; - true - }); - assign_partitions_to_members(id, &mut members, partitions); - return members; - } - members - }); - member_id -} - -fn assign_partitions_to_members(id: usize, members: &mut Slab, partitions: &[usize]) { - members - .iter_mut() - .for_each(|(_, member)| member.partitions.clear()); - let count = members.len(); - if count == 0 { - return; - } - - for (idx, partition) in partitions.iter().enumerate() { - let position = idx % count; - let member = &mut members[position]; - member.partitions.push(*partition); - tracing::trace!( - "Assigned partition ID: {} to member with ID: {} in consumer group: {}", - partition, - member.id, - id - ); - } -} - -fn mimic_members(members: &Slab) -> Slab { - let mut container = Slab::with_capacity(members.len()); - for (_, member) in members { - Member::new(member.client_id).insert_into(&mut container); - } - container -} diff --git a/core/server/src/streaming/topics/mod.rs b/core/server/src/streaming/topics/mod.rs index 236e0cb9c7..162e3bc545 100644 --- a/core/server/src/streaming/topics/mod.rs +++ b/core/server/src/streaming/topics/mod.rs @@ -16,9 +16,7 @@ * under the License. */ -pub mod consumer_group; pub mod helpers; pub mod storage; -pub mod topic; pub const COMPONENT: &str = "STREAMING_TOPICS"; diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs index 4abe09c0b6..10774d9a5e 100644 --- a/core/server/src/streaming/topics/storage.rs +++ b/core/server/src/streaming/topics/storage.rs @@ -20,10 +20,8 @@ use iggy_common::IggyError; use std::path::Path; use crate::{ - configs::system::SystemConfig, - io::fs_utils::remove_dir_all, - slab::traits_ext::{Delete, EntityComponentSystem, EntityMarker, IntoComponents}, - streaming::{partitions::storage::delete_partitions_from_disk, topics::topic}, + configs::system::SystemConfig, io::fs_utils::remove_dir_all, + streaming::partitions::storage::delete_partitions_from_disk, }; pub async fn create_topic_file_hierarchy( @@ -52,42 +50,33 @@ pub async fn create_topic_file_hierarchy( Ok(()) } -pub async fn delete_topic_from_disk( +/// Delete topic directory and all partition subdirectories using only IDs. +/// Does not require slab access - works with SharedMetadata. +pub async fn delete_topic_directory( stream_id: usize, - topic: &mut topic::Topic, + topic_id: usize, + partition_ids: &[usize], config: &SystemConfig, -) -> Result<(u64, u64, u32), IggyError> { - let topic_path = config.get_topic_path(stream_id, topic.id()); - let topic_id = topic.id(); +) -> Result<(), IggyError> { + let topic_path = config.get_topic_path(stream_id, topic_id); if !Path::new(&topic_path).exists() { return Err(IggyError::TopicDirectoryNotFound(topic_path)); } - // First lets go over the partitions and it's logs and delete them from disk. - let ids = topic.root().partitions().with_components(|components| { - let (root, ..) = components.into_components(); - root.iter().map(|(_, r)| r.id()).collect::>() - }); - let mut messages_count = 0; - let mut size_bytes = 0; - let mut segments_count = 0; - let partitions = topic.root_mut().partitions_mut(); - for id in ids { - let partition = partitions.delete(id); - let (root, stats, _, _, _, _, _log) = partition.into_components(); - let partition_id = root.id(); + + // Delete partition directories + for &partition_id in partition_ids { delete_partitions_from_disk(stream_id, topic_id, partition_id, config).await?; - messages_count += stats.messages_count_inconsistent(); - size_bytes += stats.size_bytes_inconsistent(); - segments_count += stats.segments_count_inconsistent(); } - // Then delete the topic directory itself. + + // Delete the topic directory itself remove_dir_all(&topic_path).await.map_err(|_| { IggyError::CannotDeleteTopicDirectory(topic_id as u32, stream_id as u32, topic_path) })?; + tracing::info!( "Deleted topic files for topic with ID: {} in stream with ID: {}.", topic_id, stream_id ); - Ok((messages_count, size_bytes, segments_count)) + Ok(()) } diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs deleted file mode 100644 index c87958f685..0000000000 --- a/core/server/src/streaming/topics/topic.rs +++ /dev/null @@ -1,379 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::slab::streams::Streams; -use crate::slab::topics; -use crate::slab::traits_ext::{EntityMarker, InsertCell, IntoComponents, IntoComponentsById}; -use crate::slab::{Keyed, consumer_groups::ConsumerGroups, partitions::Partitions}; -use crate::streaming::stats::{StreamStats, TopicStats}; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, IggyTimestamp, MaxTopicSize}; -use slab::Slab; -use std::cell::{Ref, RefMut}; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -#[derive(Debug, Clone)] -pub struct TopicAuxilary { - current_partition_id: Arc, -} - -impl TopicAuxilary { - pub fn get_next_partition_id(&self, upperbound: usize) -> usize { - let mut partition_id = self.current_partition_id.fetch_add(1, Ordering::AcqRel); - if partition_id >= upperbound { - partition_id = 0; - self.current_partition_id - .swap(partition_id + 1, Ordering::Release); - } - tracing::trace!("Next partition ID: {}", partition_id); - partition_id - } -} - -#[derive(Default, Debug, Clone)] -pub struct TopicRoot { - id: usize, - // TODO: This property should be removed, we won't use it in our clustering impl. - replication_factor: u8, - name: String, - created_at: IggyTimestamp, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - - partitions: Partitions, - consumer_groups: ConsumerGroups, -} - -impl Keyed for TopicRoot { - type Key = String; - - fn key(&self) -> &Self::Key { - &self.name - } -} - -#[derive(Debug, Clone)] -pub struct Topic { - root: TopicRoot, - auxilary: TopicAuxilary, - stats: Arc, -} - -impl Topic { - pub fn new( - name: String, - stats: Arc, - created_at: IggyTimestamp, - replication_factor: u8, - message_expiry: IggyExpiry, - compression: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - ) -> Self { - let root = TopicRoot::new( - name, - created_at, - replication_factor, - message_expiry, - compression, - max_topic_size, - ); - let auxilary = TopicAuxilary { - current_partition_id: Arc::new(AtomicUsize::new(0)), - }; - Self { - root, - auxilary, - stats, - } - } - pub fn new_with_components( - root: TopicRoot, - auxilary: TopicAuxilary, - stats: Arc, - ) -> Self { - Self { - root, - auxilary, - stats, - } - } - - pub fn root(&self) -> &TopicRoot { - &self.root - } - - pub fn root_mut(&mut self) -> &mut TopicRoot { - &mut self.root - } - - pub fn stats(&self) -> &TopicStats { - &self.stats - } -} - -impl IntoComponents for Topic { - type Components = (TopicRoot, TopicAuxilary, Arc); - - fn into_components(self) -> Self::Components { - (self.root, self.auxilary, self.stats) - } -} - -impl EntityMarker for Topic { - type Idx = topics::ContainerId; - fn id(&self) -> Self::Idx { - self.root.id - } - - fn update_id(&mut self, id: Self::Idx) { - self.root.id = id; - } -} - -// TODO: Create a macro to impl those TopicRef/TopicRefMut structs and it's traits. -pub struct TopicRef<'a> { - root: Ref<'a, Slab>, - auxilary: Ref<'a, Slab>, - stats: Ref<'a, Slab>>, -} - -impl<'a> TopicRef<'a> { - pub fn new( - root: Ref<'a, Slab>, - auxilary: Ref<'a, Slab>, - stats: Ref<'a, Slab>>, - ) -> Self { - Self { - root, - auxilary, - stats, - } - } -} - -impl<'a> IntoComponents for TopicRef<'a> { - type Components = ( - Ref<'a, Slab>, - Ref<'a, Slab>, - Ref<'a, Slab>>, - ); - - fn into_components(self) -> Self::Components { - (self.root, self.auxilary, self.stats) - } -} - -impl<'a> IntoComponentsById for TopicRef<'a> { - type Idx = topics::ContainerId; - type Output = ( - Ref<'a, TopicRoot>, - Ref<'a, TopicAuxilary>, - Ref<'a, Arc>, - ); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = Ref::map(self.root, |r| &r[index]); - let auxilary = Ref::map(self.auxilary, |a| &a[index]); - let stats = Ref::map(self.stats, |s| &s[index]); - (root, auxilary, stats) - } -} - -pub struct TopicRefMut<'a> { - root: RefMut<'a, Slab>, - auxilary: RefMut<'a, Slab>, - stats: RefMut<'a, Slab>>, -} - -impl<'a> TopicRefMut<'a> { - pub fn new( - root: RefMut<'a, Slab>, - auxilary: RefMut<'a, Slab>, - stats: RefMut<'a, Slab>>, - ) -> Self { - Self { - root, - auxilary, - stats, - } - } -} - -impl<'a> IntoComponents for TopicRefMut<'a> { - type Components = ( - RefMut<'a, Slab>, - RefMut<'a, Slab>, - RefMut<'a, Slab>>, - ); - - fn into_components(self) -> Self::Components { - (self.root, self.auxilary, self.stats) - } -} - -impl<'a> IntoComponentsById for TopicRefMut<'a> { - type Idx = topics::ContainerId; - type Output = ( - RefMut<'a, TopicRoot>, - RefMut<'a, TopicAuxilary>, - RefMut<'a, Arc>, - ); - - fn into_components_by_id(self, index: Self::Idx) -> Self::Output { - let root = RefMut::map(self.root, |r| &mut r[index]); - let auxilary = RefMut::map(self.auxilary, |a| &mut a[index]); - let stats = RefMut::map(self.stats, |s| &mut s[index]); - (root, auxilary, stats) - } -} - -impl TopicRoot { - pub fn new( - name: String, - created_at: IggyTimestamp, - replication_factor: u8, - message_expiry: IggyExpiry, - compression: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - ) -> Self { - Self { - id: 0, - name, - created_at, - replication_factor, - message_expiry, - compression_algorithm: compression, - max_topic_size, - partitions: Partitions::default(), - consumer_groups: ConsumerGroups::default(), - } - } - - pub fn invoke(&self, f: impl FnOnce(&Self) -> T) -> T { - f(self) - } - - pub fn invoke_mut(&mut self, f: impl FnOnce(&mut Self) -> T) -> T { - f(self) - } - - pub fn update_id(&mut self, id: usize) { - self.id = id; - } - - pub fn id(&self) -> topics::ContainerId { - self.id - } - - pub fn message_expiry(&self) -> IggyExpiry { - self.message_expiry - } - - pub fn max_topic_size(&self) -> MaxTopicSize { - self.max_topic_size - } - - pub fn name(&self) -> &String { - &self.name - } - - pub fn set_name(&mut self, name: String) { - self.name = name; - } - - pub fn set_compression(&mut self, compression: CompressionAlgorithm) { - self.compression_algorithm = compression; - } - - pub fn set_message_expiry(&mut self, message_expiry: IggyExpiry) { - self.message_expiry = message_expiry; - } - - pub fn set_max_topic_size(&mut self, max_topic_size: MaxTopicSize) { - self.max_topic_size = max_topic_size; - } - - pub fn set_replication_factor(&mut self, replication_factor: u8) { - self.replication_factor = replication_factor; - } - - pub fn partitions(&self) -> &Partitions { - &self.partitions - } - - pub fn partitions_mut(&mut self) -> &mut Partitions { - &mut self.partitions - } - - pub fn consumer_groups(&self) -> &ConsumerGroups { - &self.consumer_groups - } - - pub fn consumer_groups_mut(&mut self) -> &mut ConsumerGroups { - &mut self.consumer_groups - } - - pub fn set_partitions(&mut self, partitions: Partitions) { - self.partitions = partitions; - } - - pub fn set_consumer_groups(&mut self, consumer_groups: ConsumerGroups) { - self.consumer_groups = consumer_groups; - } - - pub fn created_at(&self) -> IggyTimestamp { - self.created_at - } - - pub fn compression_algorithm(&self) -> CompressionAlgorithm { - self.compression_algorithm - } - - pub fn replication_factor(&self) -> u8 { - self.replication_factor - } -} - -// TODO: Move to separate module. -#[allow(clippy::too_many_arguments)] -pub fn create_and_insert_topics_mem( - streams: &Streams, - stream_id: &Identifier, - name: String, - replication_factor: u8, - message_expiry: IggyExpiry, - compression: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - parent_stats: Arc, -) -> Topic { - let stats = Arc::new(TopicStats::new(parent_stats)); - let now = IggyTimestamp::now(); - let mut topic = Topic::new( - name, - stats, - now, - replication_factor, - message_expiry, - compression, - max_topic_size, - ); - - let id = streams.with_topics(stream_id, |topics| topics.insert(topic.clone())); - topic.update_id(id); - topic -} diff --git a/core/server/src/streaming/users/permissioner.rs b/core/server/src/streaming/users/permissioner.rs index 807dce4657..bc4b79521f 100644 --- a/core/server/src/streaming/users/permissioner.rs +++ b/core/server/src/streaming/users/permissioner.rs @@ -16,86 +16,61 @@ * under the License. */ -use crate::streaming::users::user::User; -use ahash::{AHashMap, AHashSet}; -use iggy_common::UserId; -use iggy_common::{GlobalPermissions, Permissions, StreamPermissions}; +use crate::metadata::Metadata; +use crate::streaming::utils::ptr::EternalPtr; +use iggy_common::{Permissions, UserId}; -#[derive(Debug, Default)] +/// Permissioner reads permissions directly from SharedMetadata. +/// No caching - SharedMetadata is the single source of truth. pub struct Permissioner { - pub(super) users_permissions: AHashMap, - pub(super) users_streams_permissions: AHashMap<(UserId, usize), StreamPermissions>, - pub(super) users_that_can_poll_messages_from_all_streams: AHashSet, - pub(super) users_that_can_send_messages_to_all_streams: AHashSet, - pub(super) users_that_can_poll_messages_from_specific_streams: AHashSet<(UserId, usize)>, - pub(super) users_that_can_send_messages_to_specific_streams: AHashSet<(UserId, usize)>, + shared_metadata: EternalPtr, } -impl Permissioner { - pub fn init(&mut self, users: &[&User]) { - for user in users { - self.init_permissions_for_user(user.id, user.permissions.clone()); - } +impl std::fmt::Debug for Permissioner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Permissioner") + .field("shared_metadata", &"") + .finish() } +} - pub fn init_permissions_for_user(&mut self, user_id: UserId, permissions: Option) { - if permissions.is_none() { - return; - } - - let permissions = permissions.unwrap(); - if permissions.global.poll_messages { - self.users_that_can_poll_messages_from_all_streams - .insert(user_id); - } - - if permissions.global.send_messages { - self.users_that_can_send_messages_to_all_streams - .insert(user_id); - } - - self.users_permissions.insert(user_id, permissions.global); - if permissions.streams.is_none() { - return; - } - - let streams = permissions.streams.unwrap(); - for (stream_id, stream) in streams { - if stream.poll_messages { - self.users_that_can_poll_messages_from_specific_streams - .insert((user_id, stream_id)); - } +impl Permissioner { + pub fn new(shared_metadata: EternalPtr) -> Self { + Self { shared_metadata } + } - if stream.send_messages { - self.users_that_can_send_messages_to_specific_streams - .insert((user_id, stream_id)); - } + /// Get permissions for a user from SharedMetadata. + /// Returns None if user doesn't exist or has no permissions. + pub(super) fn get_permissions(&self, user_id: UserId) -> Option { + let metadata = self.shared_metadata.load(); + metadata + .users + .get(user_id as usize) + .and_then(|user| user.permissions.as_ref().map(|p| (**p).clone())) + } - self.users_streams_permissions - .insert((user_id, stream_id), stream); - } + /// Check if user has global permission to poll messages from all streams. + pub(super) fn can_poll_messages_globally(&self, user_id: UserId) -> bool { + self.get_permissions(user_id) + .map(|p| p.global.poll_messages) + .unwrap_or(false) } - pub fn update_permissions_for_user( - &mut self, - user_id: UserId, - permissions: Option, - ) { - self.delete_permissions_for_user(user_id); - self.init_permissions_for_user(user_id, permissions); + /// Check if user has global permission to send messages to all streams. + pub(super) fn can_send_messages_globally(&self, user_id: UserId) -> bool { + self.get_permissions(user_id) + .map(|p| p.global.send_messages) + .unwrap_or(false) } - pub fn delete_permissions_for_user(&mut self, user_id: UserId) { - self.users_permissions.remove(&user_id); - self.users_that_can_poll_messages_from_all_streams - .remove(&user_id); - self.users_that_can_send_messages_to_all_streams - .remove(&user_id); - self.users_streams_permissions - .retain(|(id, _), _| *id != user_id); - self.users_that_can_poll_messages_from_specific_streams - .retain(|(id, _)| *id != user_id); - self.users_that_can_send_messages_to_specific_streams - .retain(|(id, _)| *id != user_id); + /// Get stream-specific permissions for a user. + pub(super) fn get_stream_permissions( + &self, + user_id: UserId, + stream_id: usize, + ) -> Option { + self.get_permissions(user_id) + .and_then(|p| p.streams) + .and_then(|streams| streams.get(&stream_id).cloned()) } } diff --git a/core/server/src/streaming/users/permissioner_rules/messages.rs b/core/server/src/streaming/users/permissioner_rules/messages.rs index 7e5a34af24..da7268391e 100644 --- a/core/server/src/streaming/users/permissioner_rules/messages.rs +++ b/core/server/src/streaming/users/permissioner_rules/messages.rs @@ -26,57 +26,37 @@ impl Permissioner { stream_id: usize, topic_id: usize, ) -> Result<(), IggyError> { - if self - .users_that_can_poll_messages_from_all_streams - .contains(&user_id) - { - return Ok(()); - } - - if self - .users_that_can_poll_messages_from_specific_streams - .contains(&(user_id, stream_id)) - { + // Check global permission first (fast path) + if self.can_poll_messages_globally(user_id) { return Ok(()); } - let stream_permissions = self.users_streams_permissions.get(&(user_id, stream_id)); - if stream_permissions.is_none() { + // Check stream-specific permissions + let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) else { return Err(IggyError::Unauthorized); - } - - let stream_permissions = stream_permissions.unwrap(); - if stream_permissions.read_stream { - return Ok(()); - } + }; - if stream_permissions.manage_topics { - return Ok(()); - } - - if stream_permissions.read_topics { + // Check stream-level permissions + if stream_permissions.poll_messages + || stream_permissions.read_stream + || stream_permissions.manage_topics + || stream_permissions.read_topics + { return Ok(()); } - if stream_permissions.poll_messages { + // Check topic-specific permissions + if let Some(topic_permissions) = stream_permissions + .topics + .as_ref() + .and_then(|t| t.get(&topic_id)) + && (topic_permissions.poll_messages + || topic_permissions.read_topic + || topic_permissions.manage_topic) + { return Ok(()); } - if stream_permissions.topics.is_none() { - return Err(IggyError::Unauthorized); - } - - let topic_permissions = stream_permissions.topics.as_ref().unwrap(); - if let Some(topic_permissions) = topic_permissions.get(&topic_id) { - return match topic_permissions.poll_messages - | topic_permissions.read_topic - | topic_permissions.manage_topic - { - true => Ok(()), - false => Err(IggyError::Unauthorized), - }; - } - Err(IggyError::Unauthorized) } @@ -86,50 +66,34 @@ impl Permissioner { stream_id: usize, topic_id: usize, ) -> Result<(), IggyError> { - if self - .users_that_can_send_messages_to_all_streams - .contains(&user_id) - { + // Check global permission first (fast path) + if self.can_send_messages_globally(user_id) { return Ok(()); } - if self - .users_that_can_send_messages_to_specific_streams - .contains(&(user_id, stream_id)) - { - return Ok(()); - } - - let stream_permissions = self.users_streams_permissions.get(&(user_id, stream_id)); - if stream_permissions.is_none() { + // Check stream-specific permissions + let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) else { return Err(IggyError::Unauthorized); - } + }; - let stream_permissions = stream_permissions.unwrap(); - if stream_permissions.manage_stream { - return Ok(()); - } - - if stream_permissions.manage_topics { + // Check stream-level permissions + if stream_permissions.send_messages + || stream_permissions.manage_stream + || stream_permissions.manage_topics + { return Ok(()); } - if stream_permissions.send_messages { + // Check topic-specific permissions + if let Some(topic_permissions) = stream_permissions + .topics + .as_ref() + .and_then(|t| t.get(&topic_id)) + && (topic_permissions.send_messages || topic_permissions.manage_topic) + { return Ok(()); } - if stream_permissions.topics.is_none() { - return Err(IggyError::Unauthorized); - } - - let topic_permissions = stream_permissions.topics.as_ref().unwrap(); - if let Some(topic_permissions) = topic_permissions.get(&topic_id) { - return match topic_permissions.send_messages | topic_permissions.manage_topic { - true => Ok(()), - false => Err(IggyError::Unauthorized), - }; - } - Err(IggyError::Unauthorized) } } diff --git a/core/server/src/streaming/users/permissioner_rules/streams.rs b/core/server/src/streaming/users/permissioner_rules/streams.rs index 0e7545ef32..1147aeb9ab 100644 --- a/core/server/src/streaming/users/permissioner_rules/streams.rs +++ b/core/server/src/streaming/users/permissioner_rules/streams.rs @@ -21,13 +21,17 @@ use iggy_common::IggyError; impl Permissioner { pub fn get_stream(&self, user_id: u32, stream_id: usize) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.read_streams) + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && (p.global.manage_streams || p.global.read_streams) { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) && (stream_permissions.manage_stream || stream_permissions.read_stream) { return Ok(()); @@ -37,8 +41,10 @@ impl Permissioner { } pub fn get_streams(&self, user_id: u32) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.read_streams) + let permissions = self.get_permissions(user_id); + + if let Some(ref p) = permissions + && (p.global.manage_streams || p.global.read_streams) { return Ok(()); } @@ -47,8 +53,10 @@ impl Permissioner { } pub fn create_stream(&self, user_id: u32) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && global_permissions.manage_streams + let permissions = self.get_permissions(user_id); + + if let Some(ref p) = permissions + && p.global.manage_streams { return Ok(()); } @@ -69,13 +77,17 @@ impl Permissioner { } fn manage_stream(&self, user_id: u32, stream_id: usize) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && global_permissions.manage_streams + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && p.global.manage_streams { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) && stream_permissions.manage_stream { return Ok(()); diff --git a/core/server/src/streaming/users/permissioner_rules/system.rs b/core/server/src/streaming/users/permissioner_rules/system.rs index 86f1e0bd03..477860f2b0 100644 --- a/core/server/src/streaming/users/permissioner_rules/system.rs +++ b/core/server/src/streaming/users/permissioner_rules/system.rs @@ -33,8 +33,10 @@ impl Permissioner { } fn get_server_info(&self, user_id: u32) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_servers || global_permissions.read_servers) + let permissions = self.get_permissions(user_id); + + if let Some(ref p) = permissions + && (p.global.manage_servers || p.global.read_servers) { return Ok(()); } diff --git a/core/server/src/streaming/users/permissioner_rules/topics.rs b/core/server/src/streaming/users/permissioner_rules/topics.rs index 28a85f058d..aacd028872 100644 --- a/core/server/src/streaming/users/permissioner_rules/topics.rs +++ b/core/server/src/streaming/users/permissioner_rules/topics.rs @@ -26,23 +26,29 @@ impl Permissioner { stream_id: usize, topic_id: usize, ) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.read_streams - || global_permissions.manage_streams - || global_permissions.manage_topics - || global_permissions.read_topics) + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && (p.global.read_streams + || p.global.manage_streams + || p.global.manage_topics + || p.global.read_topics) { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) - { + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) { if stream_permissions.manage_topics || stream_permissions.read_topics { return Ok(()); } - if let Some(topic_permissions) = - stream_permissions.topics.as_ref().unwrap().get(&topic_id) + // Check topic-specific permissions + if let Some(topic_permissions) = stream_permissions + .topics + .as_ref() + .and_then(|t| t.get(&topic_id)) && (topic_permissions.manage_topic || topic_permissions.read_topic) { return Ok(()); @@ -53,40 +59,40 @@ impl Permissioner { } pub fn get_topics(&self, user_id: u32, stream_id: usize) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.read_streams - || global_permissions.manage_streams - || global_permissions.manage_topics - || global_permissions.read_topics) + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && (p.global.read_streams + || p.global.manage_streams + || p.global.manage_topics + || p.global.read_topics) { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) + && (stream_permissions.manage_topics || stream_permissions.read_topics) { - if stream_permissions.manage_topics || stream_permissions.read_topics { - return Ok(()); - } - - if let Some(topic_permissions) = - stream_permissions.topics.as_ref().unwrap().get(&stream_id) - && (topic_permissions.manage_topic || topic_permissions.read_topic) - { - return Ok(()); - } + return Ok(()); } Err(IggyError::Unauthorized) } pub fn create_topic(&self, user_id: u32, stream_id: usize) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.manage_topics) + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && (p.global.manage_streams || p.global.manage_topics) { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) && stream_permissions.manage_topics { return Ok(()); @@ -128,20 +134,26 @@ impl Permissioner { stream_id: usize, topic_id: usize, ) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.manage_topics) + let permissions = self.get_permissions(user_id); + + // Check global permissions + if let Some(ref p) = permissions + && (p.global.manage_streams || p.global.manage_topics) { return Ok(()); } - if let Some(stream_permissions) = self.users_streams_permissions.get(&(user_id, stream_id)) - { + // Check stream-specific permissions + if let Some(stream_permissions) = self.get_stream_permissions(user_id, stream_id) { if stream_permissions.manage_topics { return Ok(()); } - if let Some(topic_permissions) = - stream_permissions.topics.as_ref().unwrap().get(&topic_id) + // Check topic-specific permissions + if let Some(topic_permissions) = stream_permissions + .topics + .as_ref() + .and_then(|t| t.get(&topic_id)) && topic_permissions.manage_topic { return Ok(()); diff --git a/core/server/src/streaming/users/permissioner_rules/users.rs b/core/server/src/streaming/users/permissioner_rules/users.rs index 8c01d7c7f9..1e2a2c982f 100644 --- a/core/server/src/streaming/users/permissioner_rules/users.rs +++ b/core/server/src/streaming/users/permissioner_rules/users.rs @@ -29,28 +29,30 @@ impl Permissioner { } pub fn create_user(&self, user_id: u32) -> Result<(), IggyError> { - self.manager_users(user_id) + self.manage_users(user_id) } pub fn delete_user(&self, user_id: u32) -> Result<(), IggyError> { - self.manager_users(user_id) + self.manage_users(user_id) } pub fn update_user(&self, user_id: u32) -> Result<(), IggyError> { - self.manager_users(user_id) + self.manage_users(user_id) } pub fn update_permissions(&self, user_id: u32) -> Result<(), IggyError> { - self.manager_users(user_id) + self.manage_users(user_id) } pub fn change_password(&self, user_id: u32) -> Result<(), IggyError> { - self.manager_users(user_id) + self.manage_users(user_id) } - fn manager_users(&self, user_id: u32) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && global_permissions.manage_users + fn manage_users(&self, user_id: u32) -> Result<(), IggyError> { + let permissions = self.get_permissions(user_id); + + if let Some(ref p) = permissions + && p.global.manage_users { return Ok(()); } @@ -59,8 +61,10 @@ impl Permissioner { } fn read_users(&self, user_id: u32) -> Result<(), IggyError> { - if let Some(global_permissions) = self.users_permissions.get(&user_id) - && (global_permissions.manage_users || global_permissions.read_users) + let permissions = self.get_permissions(user_id); + + if let Some(ref p) = permissions + && (p.global.manage_users || p.global.read_users) { return Ok(()); } diff --git a/core/server/src/streaming/users/user.rs b/core/server/src/streaming/users/user.rs index 672662d3e7..4882d4542b 100644 --- a/core/server/src/streaming/users/user.rs +++ b/core/server/src/streaming/users/user.rs @@ -32,7 +32,7 @@ pub struct User { pub password: String, pub created_at: IggyTimestamp, pub permissions: Option, - pub personal_access_tokens: DashMap, PersonalAccessToken>, + pub personal_access_tokens: DashMap, PersonalAccessToken>, } impl Default for User { diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs index d3a58db57f..38819d08c7 100644 --- a/core/server/src/tcp/connection_handler.rs +++ b/core/server/src/tcp/connection_handler.rs @@ -17,7 +17,6 @@ */ use crate::binary::command; -use crate::binary::command::ServerCommandHandler; use crate::server_error::ConnectionError; use crate::shard::IggyShard; use crate::streaming::session::Session; @@ -91,7 +90,7 @@ pub(crate) async fn handle_connection( let command = ServerCommand::from_code_and_reader(code, sender, length - 4).await?; debug!("Received a TCP command: {command}, payload size: {length}"); let cmd_code = command.code(); - match command.handle(sender, length, session, shard).await { + match command.dispatch(sender, length, session, shard).await { Ok(handler_result) => match handler_result { command::HandlerResult::Finished => { debug!( diff --git a/core/server/src/websocket/connection_handler.rs b/core/server/src/websocket/connection_handler.rs index 2ee97196f2..f23e0911a2 100644 --- a/core/server/src/websocket/connection_handler.rs +++ b/core/server/src/websocket/connection_handler.rs @@ -17,7 +17,6 @@ */ use crate::binary::command; -use crate::binary::command::ServerCommandHandler; use crate::server_error::ConnectionError; use crate::shard::IggyShard; use crate::streaming::session::Session; @@ -81,7 +80,7 @@ pub(crate) async fn handle_connection( let command = ServerCommand::from_code_and_reader(code, sender, length - 4).await?; debug!("Received a WebSocket command: {command}, payload size: {length}"); - match command.handle(sender, length, session, shard).await { + match command.dispatch(sender, length, session, shard).await { Ok(_) => { debug!( "Command was handled successfully, session: {session}. WebSocket response was sent."