diff --git a/lib/icinga/CMakeLists.txt b/lib/icinga/CMakeLists.txt index 62077bce74d..8187d48e884 100644 --- a/lib/icinga/CMakeLists.txt +++ b/lib/icinga/CMakeLists.txt @@ -39,7 +39,7 @@ set(icinga_SOURCES comment.cpp comment.hpp comment-ti.hpp compatutility.cpp compatutility.hpp customvarobject.cpp customvarobject.hpp customvarobject-ti.hpp - dependency.cpp dependency.hpp dependency-ti.hpp dependency-apply.cpp + dependency.cpp dependency-group.cpp dependency.hpp dependency-ti.hpp dependency-apply.cpp downtime.cpp downtime.hpp downtime-ti.hpp envresolver.cpp envresolver.hpp eventcommand.cpp eventcommand.hpp eventcommand-ti.hpp diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 6e3b8764b83..31993fc871a 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -154,6 +154,10 @@ Checkable::ProcessingResult Checkable::ProcessCheckResult(const CheckResult::Ptr bool reachable = IsReachable(); bool notification_reachable = IsReachable(DependencyNotification); + // Cache whether the previous state of this Checkable affects its children before overwriting the last check result. + // This will be used to determine whether the on reachability changed event should be triggered. + bool affectsPreviousStateChildren(reachable && AffectsChildren()); + ObjectLock olock(this); CheckResult::Ptr old_cr = GetLastCheckResult(); @@ -533,7 +537,7 @@ Checkable::ProcessingResult Checkable::ProcessCheckResult(const CheckResult::Ptr } /* update reachability for child objects */ - if ((stateChange || hardChange) && !children.empty()) + if ((stateChange || hardChange) && !children.empty() && (affectsPreviousStateChildren || AffectsChildren())) OnReachabilityChanged(this, cr, children, origin); return Result::Ok; diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 58d6b578bb8..65d9dd9022d 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -3,26 +3,169 @@ #include "icinga/service.hpp" #include "icinga/dependency.hpp" #include "base/logger.hpp" -#include using namespace icinga; -void Checkable::AddDependency(const Dependency::Ptr& dep) +/** + * The maximum number of allowed dependency recursion levels. + * + * This is a subjective limit how deep the dependency tree should be allowed to go, as anything beyond this level + * is just madness and will likely result in a stack overflow or other undefined behavior. + */ +static constexpr int l_MaxDependencyRecursionLevel(256); + +/** + * Register all the dependency groups of the current Checkable to the global dependency group registry. + * + * Initially, each Checkable object tracks locally its own dependency groups on Icinga 2 startup, and once the start + * signal of that Checkable is emitted, it pushes all the local tracked dependency groups to the global registry. + * Once the global registry is populated with all the local dependency groups, this Checkable may not necessarily + * contain the exact same dependency groups as it did before, as identical groups are merged together in the registry, + * but it's guaranteed to have the same *number* of dependency groups as before. + */ +void Checkable::PushDependencyGroupsToRegistry() { - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.insert(dep); + std::lock_guard lock(m_DependencyMutex); + if (m_PendingDependencies != nullptr) { + for (const auto& [key, dependencies] : *m_PendingDependencies) { + String redundancyGroup = std::holds_alternative(key) ? std::get(key) : ""; + m_DependencyGroups.emplace(key, DependencyGroup::Register(new DependencyGroup(redundancyGroup, dependencies))); + } + m_PendingDependencies.reset(); + } } -void Checkable::RemoveDependency(const Dependency::Ptr& dep) +std::vector Checkable::GetDependencyGroups() const { - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.erase(dep); + std::lock_guard lock(m_DependencyMutex); + + std::vector dependencyGroups; + for (const auto& [_, dependencyGroup] : m_DependencyGroups) { + dependencyGroups.emplace_back(dependencyGroup); + } + return dependencyGroups; +} + +/** + * Get the key for the provided dependency group. + * + * The key is either the parent Checkable object or the redundancy group name of the dependency object. + * This is used to uniquely identify the dependency group within a given Checkable object. + * + * @param dependency The dependency to get the key for. + * + * @return - Returns the key for the provided dependency group. + */ +static std::variant GetDependencyGroupKey(const Dependency::Ptr& dependency) +{ + if (auto redundancyGroup(dependency->GetRedundancyGroup()); !redundancyGroup.IsEmpty()) { + return std::move(redundancyGroup); + } + + return dependency->GetParent().get(); } -std::vector Checkable::GetDependencies() const +/** + * Add the provided dependency to the current Checkable list of dependencies. + * + * @param dependency The dependency to add. + */ +void Checkable::AddDependency(const Dependency::Ptr& dependency) +{ + std::unique_lock lock(m_DependencyMutex); + + auto dependencyGroupKey(GetDependencyGroupKey(dependency)); + if (m_PendingDependencies != nullptr) { + (*m_PendingDependencies)[dependencyGroupKey].emplace(dependency); + return; + } + + std::set dependencies; + bool removeGroup(false); + + DependencyGroup::Ptr existingGroup; + if (auto it(m_DependencyGroups.find(dependencyGroupKey)); it != m_DependencyGroups.end()) { + existingGroup = it->second; + std::tie(dependencies, removeGroup) = DependencyGroup::Unregister(existingGroup, this); + m_DependencyGroups.erase(it); + } + + dependencies.emplace(dependency); + + auto dependencyGroup(DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies))); + m_DependencyGroups.emplace(dependencyGroupKey, dependencyGroup); + + lock.unlock(); + + if (existingGroup) { + dependencies.erase(dependency); + DependencyGroup::OnChildRemoved(existingGroup, {dependencies.begin(), dependencies.end()}, removeGroup); + } + DependencyGroup::OnChildRegistered(this, dependencyGroup); +} + +/** + * Remove the provided dependency from the current Checkable list of dependencies. + * + * @param dependency The dependency to remove. + * @param runtimeRemoved Whether the given dependency object is being removed at runtime. + */ +void Checkable::RemoveDependency(const Dependency::Ptr& dependency, bool runtimeRemoved) +{ + std::unique_lock lock(m_DependencyMutex); + + auto dependencyGroupKey(GetDependencyGroupKey(dependency)); + auto it = m_DependencyGroups.find(dependencyGroupKey); + if (it == m_DependencyGroups.end()) { + return; + } + + DependencyGroup::Ptr existingGroup(it->second); + auto [dependencies, removeGroup] = DependencyGroup::Unregister(existingGroup, this); + + m_DependencyGroups.erase(it); + dependencies.erase(dependency); + + DependencyGroup::Ptr newDependencyGroup; + if (!dependencies.empty()) { + newDependencyGroup = DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)); + m_DependencyGroups.emplace(dependencyGroupKey, newDependencyGroup); + } + + lock.unlock(); + + if (runtimeRemoved) { + dependencies.emplace(dependency); + DependencyGroup::OnChildRemoved(existingGroup, {dependencies.begin(), dependencies.end()}, removeGroup); + + if (newDependencyGroup) { + DependencyGroup::OnChildRegistered(this, newDependencyGroup); + } + } +} + +std::vector Checkable::GetDependencies(bool includePending) const { std::unique_lock lock(m_DependencyMutex); - return std::vector(m_Dependencies.begin(), m_Dependencies.end()); + std::vector dependencies; + + if (includePending && m_PendingDependencies != nullptr) { + for (const auto& [group, groupDeps] : *m_PendingDependencies) { + dependencies.insert(dependencies.end(), groupDeps.begin(), groupDeps.end()); + } + } + + for (const auto& [_, dependencyGroup] : m_DependencyGroups) { + auto tmpDependencies(dependencyGroup->GetDependenciesForChild(this)); + dependencies.insert(dependencies.end(), tmpDependencies.begin(), tmpDependencies.end()); + } + return dependencies; +} + +bool Checkable::HasAnyDependencies() const +{ + std::unique_lock lock(m_DependencyMutex); + return !m_DependencyGroups.empty() || !m_ReverseDependencies.empty(); } void Checkable::AddReverseDependency(const Dependency::Ptr& dep) @@ -43,89 +186,72 @@ std::vector Checkable::GetReverseDependencies() const return std::vector(m_ReverseDependencies.begin(), m_ReverseDependencies.end()); } -bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency, int rstack) const +bool Checkable::IsReachable(DependencyType dt, int rstack) const { - /* Anything greater than 256 causes recursion bus errors. */ - int limit = 256; - - if (rstack > limit) { + if (rstack > l_MaxDependencyRecursionLevel) { Log(LogWarning, "Checkable") - << "Too many nested dependencies (>" << limit << ") for checkable '" << GetName() << "': Dependency failed."; + << "Too many nested dependencies (>" << l_MaxDependencyRecursionLevel << ") for checkable '" << GetName() << "': Dependency failed."; return false; } - for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, failedDependency, rstack + 1)) - return false; - } - /* implicit dependency on host if this is a service */ const auto *service = dynamic_cast(this); if (service && (dt == DependencyState || dt == DependencyNotification)) { Host::Ptr host = service->GetHost(); if (host && host->GetState() != HostUp && host->GetStateType() == StateTypeHard) { - if (failedDependency) - *failedDependency = nullptr; - return false; } } - auto deps = GetDependencies(); - - std::unordered_map violated; // key: redundancy group, value: nullptr if satisfied, violating dependency otherwise - - for (const Dependency::Ptr& dep : deps) { - std::string redundancy_group = dep->GetRedundancyGroup(); - - if (!dep->IsAvailable(dt)) { - if (redundancy_group.empty()) { - Log(LogDebug, "Checkable") - << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; - - if (failedDependency) - *failedDependency = dep; - - return false; - } + for (auto& dependencyGroup : GetDependencyGroups()) { + if (auto state(dependencyGroup->GetState(this, dt, rstack + 1)); state != DependencyGroup::State::Ok) { + Log(LogDebug, "Checkable") + << "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' have failed for checkable '" + << GetName() << "': Marking as unreachable."; - // tentatively mark this dependency group as failed unless it is already marked; - // so it either passed before (don't overwrite) or already failed (so don't care) - // note that std::unordered_map::insert() will not overwrite an existing entry - violated.insert(std::make_pair(redundancy_group, dep)); - } else if (!redundancy_group.empty()) { - violated[redundancy_group] = nullptr; + return false; } } - auto violator = std::find_if(violated.begin(), violated.end(), [](auto& v) { return v.second != nullptr; }); - if (violator != violated.end()) { - Log(LogDebug, "Checkable") - << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - - if (failedDependency) - *failedDependency = violator->second; + return true; +} +/** + * Checks whether the last check result of this Checkable affects its child dependencies. + * + * A Checkable affects its child dependencies if it runs into a non-OK state and results in any of its child + * Checkables to become unreachable. Though, that unavailable dependency may not necessarily cause the child + * Checkable to be in unreachable state as it might have some other dependencies that are still reachable, instead + * it just indicates whether the edge/connection between this and the child Checkable is broken or not. + * + * @return bool - Returns true if the Checkable affects its child dependencies, otherwise false. + */ +bool Checkable::AffectsChildren() const +{ + if (!GetLastCheckResult() || !IsReachable()) { + // If there is no check result, or the Checkable is not reachable, we can't safely determine whether + // the Checkable affects its child dependencies. return false; } - if (failedDependency) - *failedDependency = nullptr; + for (auto& dep : GetReverseDependencies()) { + if (!dep->IsAvailable(DependencyState)) { + // If one of the child dependency is not available, then it's definitely due to the + // current Checkable state, so we don't need to verify the remaining ones. + return true; + } + } - return true; + return false; } std::set Checkable::GetParents() const { std::set parents; - - for (const Dependency::Ptr& dep : GetDependencies()) { - Checkable::Ptr parent = dep->GetParent(); - - if (parent && parent.get() != this) - parents.insert(parent); + for (auto& dependencyGroup : GetDependencyGroups()) { + dependencyGroup->LoadParents(parents); } return parents; @@ -145,32 +271,51 @@ std::set Checkable::GetChildren() const return parents; } +/** + * Retrieve the total number of all the children of the current Checkable. + * + * Note, due to the max recursion limit of 256, the returned number may not reflect + * the actual total number of children involved in the dependency chain. + * + * @return int - Returns the total number of all the children of the current Checkable. + */ +size_t Checkable::GetAllChildrenCount() const +{ + // Are you thinking in making this more efficient? Please, don't. + // In order not to count the same child multiple times, we need to maintain a separate set of visited children, + // which is basically the same as what GetAllChildren() does. So, we're using it here! + return GetAllChildren().size(); +} + std::set Checkable::GetAllChildren() const { - std::set children = GetChildren(); + std::set children; GetAllChildrenInternal(children, 0); return children; } -void Checkable::GetAllChildrenInternal(std::set& children, int level) const +/** + * Retrieve all direct and indirect children of the current Checkable. + * + * Note, this function performs a recursive call chain traversing all the children of the current Checkable + * up to a certain limit (256). When that limit is reached, it will log a warning message and abort the operation. + * + * @param seenChildren - A container to store all the traversed children into. + * @param level - The current level of recursion. + */ +void Checkable::GetAllChildrenInternal(std::set& seenChildren, int level) const { - if (level > 32) + if (level > l_MaxDependencyRecursionLevel) { + Log(LogWarning, "Checkable") + << "Too many nested dependencies (>" << l_MaxDependencyRecursionLevel << ") for checkable '" << GetName() << "': aborting traversal."; return; + } - std::set localChildren; - - for (const Checkable::Ptr& checkable : children) { - std::set cChildren = checkable->GetChildren(); - - if (!cChildren.empty()) { - GetAllChildrenInternal(cChildren, level + 1); - localChildren.insert(cChildren.begin(), cChildren.end()); + for (const Checkable::Ptr& checkable : GetChildren()) { + if (auto [_, inserted] = seenChildren.insert(checkable); inserted) { + checkable->GetAllChildrenInternal(seenChildren, level + 1); } - - localChildren.insert(checkable); } - - children.insert(localChildren.begin(), localChildren.end()); } diff --git a/lib/icinga/checkable-notification.cpp b/lib/icinga/checkable-notification.cpp index 2a115055672..282b95d3212 100644 --- a/lib/icinga/checkable-notification.cpp +++ b/lib/icinga/checkable-notification.cpp @@ -167,8 +167,7 @@ void Checkable::FireSuppressedNotifications() } } - for (auto& dep : GetDependencies()) { - auto parent (dep->GetParent()); + for (auto& parent : GetParents()) { ObjectLock oLock (parent); if (!parent->GetProblem() && parent->GetLastStateChange() >= threshold) { diff --git a/lib/icinga/checkable.cpp b/lib/icinga/checkable.cpp index ddf84cd1fd2..13fd778a303 100644 --- a/lib/icinga/checkable.cpp +++ b/lib/icinga/checkable.cpp @@ -80,6 +80,8 @@ void Checkable::OnAllConfigLoaded() void Checkable::Start(bool runtimeCreated) { + PushDependencyGroupsToRegistry(); + double now = Utility::GetTime(); { diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index fcfbca9b281..98c015ed60f 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace icinga { @@ -57,6 +58,7 @@ enum FlappingStateFilter class CheckCommand; class EventCommand; class Dependency; +class DependencyGroup; /** * An Icinga service. @@ -77,10 +79,12 @@ class Checkable : public ObjectImpl std::set GetParents() const; std::set GetChildren() const; std::set GetAllChildren() const; + size_t GetAllChildrenCount() const; void AddGroup(const String& name); - bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool IsReachable(DependencyType dt = DependencyState, int rstack = 0) const; + bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); @@ -182,9 +186,12 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ - void AddDependency(const intrusive_ptr& dep); - void RemoveDependency(const intrusive_ptr& dep); - std::vector > GetDependencies() const; + void PushDependencyGroupsToRegistry(); + std::vector> GetDependencyGroups() const; + void AddDependency(const intrusive_ptr& dependency); + void RemoveDependency(const intrusive_ptr& dependency, bool runtimeRemoved = false); + std::vector > GetDependencies(bool includePending = false) const; + bool HasAnyDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); void RemoveReverseDependency(const intrusive_ptr& dep); @@ -244,10 +251,21 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; - std::set > m_Dependencies; + std::map, intrusive_ptr> m_DependencyGroups; std::set > m_ReverseDependencies; - - void GetAllChildrenInternal(std::set& children, int level = 0) const; + /** + * Registering a checkable to its parent DependencyGroups is delayed during config loading until all dependencies + * were registered on the checkable. m_PendingDependencies is used to temporarily store the dependencies until then. + * It is a pointer type for two reasons: + * 1. The field is no longer needed after the DependencyGroups were registered, having it as a pointer reduces the + * overhead from sizeof(std::map<>) to sizeof(std::map<>*). + * 2. It allows the field to also be used as a flag: the delayed group registration is only done until it is reset + * to nullptr. + */ + std::unique_ptr, std::set>>> + m_PendingDependencies {std::make_unique()}; + + void GetAllChildrenInternal(std::set& seenChildren, int level = 0) const; /* Flapping */ static const std::map m_FlappingStateFilterMap; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp new file mode 100644 index 00000000000..aa7d5fc06ac --- /dev/null +++ b/lib/icinga/dependency-group.cpp @@ -0,0 +1,348 @@ +/* Icinga 2 | (c) 2024 Icinga GmbH | GPLv2+ */ + +#include "icinga/dependency.hpp" +#include "base/object-packer.hpp" + +using namespace icinga; + +boost::signals2::signal DependencyGroup::OnChildRegistered; +boost::signals2::signal&, bool)> DependencyGroup::OnChildRemoved; + +std::mutex DependencyGroup::m_RegistryMutex; +DependencyGroup::RegistryType DependencyGroup::m_Registry; + +/** + * Register the provided dependency group to the global dependency group registry. + * + * In case there is already an identical dependency group in the registry, the provided dependency group is merged + * with the existing one, and that group is returned. Otherwise, the provided dependency group is registered as is, + * and it's returned. + * + * @param dependencyGroup The dependency group to register. + */ +DependencyGroup::Ptr DependencyGroup::Register(const DependencyGroup::Ptr& dependencyGroup) +{ + std::lock_guard lock(m_RegistryMutex); + if (auto [it, inserted] = m_Registry.insert(dependencyGroup); !inserted) { + dependencyGroup->CopyDependenciesTo(*it); + return *it; + } + return dependencyGroup; +} + +/** + * Detach the provided child Checkable from the specified dependency group. + * + * Unregisters all the dependency objects the child Checkable depends on from the provided dependency group and + * removes the dependency group from the global registry if it becomes empty afterward. + * + * @param dependencyGroup The dependency group to unregister the child Checkable from. + * @param child The child Checkable to detach from the dependency group. + * + * @return - Returns the dependency objects of the child Checkable that were member of the provided dependency group + * and a boolean indicating whether the dependency group has been erased from the global registry. + */ +std::pair, bool> DependencyGroup::Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child) +{ + std::lock_guard lock(m_RegistryMutex); + if (auto it(m_Registry.find(dependencyGroup)); it != m_Registry.end()) { + auto& existingGroup(*it); + auto dependencies(existingGroup->GetDependenciesForChild(child.get())); + + for (const auto& dependency : dependencies) { + existingGroup->RemoveDependency(dependency); + } + + bool remove = !existingGroup->HasChildren(); + if (remove) { + m_Registry.erase(it); + } + return {{dependencies.begin(), dependencies.end()}, remove}; + } + return {{}, false}; +} + +/** + * Retrieve the size of the global dependency group registry. + * + * @return size_t - Returns the size of the global dependency groups registry. + */ +size_t DependencyGroup::GetRegistrySize() +{ + std::lock_guard lock(m_RegistryMutex); + return m_Registry.size(); +} + +DependencyGroup::DependencyGroup(String name, const std::set& dependencies) + : m_RedundancyGroupName(std::move(name)) +{ + for (const auto& dependency : dependencies) { + m_Members[MakeCompositeKeyFor(dependency)].emplace(dependency->GetChild().get(), dependency.get()); + } +} + +/** + * Create a composite key for the provided dependency. + * + * The composite key consists of all the properties of the provided dependency object that influence its availability. + * + * @param dependency The dependency object to create a composite key for. + * + * @return - Returns the composite key for the provided dependency. + */ +DependencyGroup::CompositeKeyType DependencyGroup::MakeCompositeKeyFor(const Dependency::Ptr& dependency) +{ + return std::make_tuple( + dependency->GetParent().get(), + dependency->GetPeriod().get(), + dependency->GetStateFilter(), + dependency->GetIgnoreSoftStates() + ); +} + +/** + * Check if the current dependency has any children. + * + * @return bool - Returns true if the current dependency group has children, otherwise false. + */ +bool DependencyGroup::HasChildren() const +{ + std::lock_guard lock(m_Mutex); + return std::any_of(m_Members.begin(), m_Members.end(), [](const auto& pair) { return !pair.second.empty(); }); +} + +/** + * Retrieve all dependency objects of the current dependency group the provided child Checkable depend on. + * + * @param child The child Checkable to get the dependencies for. + * + * @return - Returns all the dependencies of the provided child Checkable in the current dependency group. + */ +std::vector DependencyGroup::GetDependenciesForChild(const Checkable* child) const +{ + std::lock_guard lock(m_Mutex); + std::vector dependencies; + for (auto& [_, children] : m_Members) { + auto [begin, end] = children.equal_range(child); + std::transform(begin, end, std::back_inserter(dependencies), [](const auto& pair) { + return pair.second; + }); + } + return dependencies; +} + +/** + * Load all parent Checkables of the current dependency group. + * + * @param parents The set to load the parent Checkables into. + */ +void DependencyGroup::LoadParents(std::set& parents) const +{ + for (auto& [compositeKey, children] : m_Members) { + parents.insert(std::get<0>(compositeKey)); + } +} + +/** + * Retrieve the number of dependency objects in the current dependency group. + * + * This function mainly exists for optimization purposes, i.e. instead of getting a copy of the members and + * counting them, we can directly query the number of dependencies in the group. + * + * @return size_t + */ +size_t DependencyGroup::GetDependenciesCount() const +{ + std::lock_guard lock(m_Mutex); + size_t count(0); + for (auto& [_, dependencies] : m_Members) { + count += dependencies.size(); + } + return count; +} + +/** + * Add a dependency object to the current dependency group. + * + * @param dependency The dependency to add to the dependency group. + */ +void DependencyGroup::AddDependency(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_Mutex); + auto compositeKey(MakeCompositeKeyFor(dependency)); + auto it = m_Members.find(compositeKey); + + // The dependency must be compatible with the group, i.e. its parent config must be known in the group already. + VERIFY(it != m_Members.end()); + + it->second.emplace(dependency->GetChild().get(), dependency.get()); +} + +/** + * Remove a dependency object from the current dependency group. + * + * @param dependency The dependency to remove from the dependency group. + */ +void DependencyGroup::RemoveDependency(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_Mutex); + if (auto it(m_Members.find(MakeCompositeKeyFor(dependency))); it != m_Members.end()) { + auto [begin, end] = it->second.equal_range(dependency->GetChild().get()); + for (auto childrenIt(begin); childrenIt != end; ++childrenIt) { + if (childrenIt->second == dependency) { + // This will also remove the child Checkable from the multimap container + // entirely if this was the last child of it. + it->second.erase(childrenIt); + return; + } + } + } +} + +/** + * Copy the dependency objects of the current dependency group to the provided dependency group (destination). + * + * @param dest The dependency group to copy the dependencies to. + */ +void DependencyGroup::CopyDependenciesTo(const DependencyGroup::Ptr& dest) +{ + VERIFY(this != dest); // Prevent from doing something stupid, i.e. deadlocking ourselves. + + std::lock_guard lock(m_Mutex); + for (auto& [_, children] : m_Members) { + std::for_each(children.begin(), children.end(), [&dest](const auto& pair) { + dest->AddDependency(pair.second); + }); + } +} + +/** + * Set the Icinga DB identifier for the current dependency group. + * + * The only usage of this function is the Icinga DB feature used to cache the unique hash of this dependency groups. + * + * @param identifier The Icinga DB identifier to set. + */ +void DependencyGroup::SetIcingaDBIdentifier(const String& identifier) +{ + std::lock_guard lock(m_Mutex); + m_IcingaDBIdentifier = identifier; +} + +/** + * Retrieve the Icinga DB identifier for the current dependency group. + * + * When the identifier is not already set by Icinga DB via the SetIcingaDBIdentifier method, + * this will just return an empty string. + * + * @return - Returns the Icinga DB identifier for the current dependency group. + */ +String DependencyGroup::GetIcingaDBIdentifier() const +{ + std::lock_guard lock(m_Mutex); + return m_IcingaDBIdentifier; +} + +/** + * Retrieve the redundancy group name of the current dependency group. + * + * If the current dependency group doesn't represent a redundancy group, this will return an empty string. + * + * @return - Returns the name of the current dependency group. + */ +const String& DependencyGroup::GetRedundancyGroupName() const +{ + // We don't need to lock the mutex here, as the name is set once during + // the object construction and never changed afterwards. + return m_RedundancyGroupName; +} + +/** + * Retrieve the unique composite key of the current dependency group. + * + * The composite key consists of some unique data of the group members, and should be used to generate + * a unique deterministic hash for the dependency group. Additionally, for explicitly configured redundancy + * groups, the non-unique dependency group name is also included on top of the composite keys. + * + * @return - Returns the composite key of the current dependency group. + */ +String DependencyGroup::GetCompositeKey() +{ + // This a copy of the CompositeKeyType definition but with the String type instead of Checkable* and TimePeriod*. + // This is because we need to produce a deterministic value from the composite key after each restart and that's + // not achievable using pointers. + using StringTuple = std::tuple; + std::vector compositeKeys; + for (auto& [compositeKey, _] : m_Members) { + auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; + compositeKeys.emplace_back(parent->GetName(), tp ? tp->GetName() : "", stateFilter, ignoreSoftStates); + } + + // IMPORTANT: The order of the composite keys must be sorted to ensure the deterministic hash value. + std::sort(compositeKeys.begin(), compositeKeys.end()); + + Array::Ptr data(new Array{GetRedundancyGroupName()}); + for (auto& compositeKey : compositeKeys) { + // std::apply is used to unpack the composite key tuple and add its elements to the data array. + // It's like manually expanding the tuple into x variables and then adding them one by one to the array. + // See https://en.cppreference.com/w/cpp/language/fold for more information. + std::apply([&data](auto&&... args) { (data->Add(std::move(args)), ...); }, std::move(compositeKey)); + } + + return PackObject(data); +} + +/** + * Retrieve the state of the current dependency group. + * + * The state of the dependency group is determined based on the state of the parent Checkables and dependency objects + * of the group. A dependency group is considered unreachable when none of the parent Checkables is reachable. However, + * a dependency group may still be marked as failed even when it has reachable parent Checkables, but an unreachable + * group has always a failed state. + * + * @param child The child Checkable to evaluate the state for. + * @param dt The dependency type to evaluate the state for, defaults to DependencyState. + * @param rstack The recursion stack level to prevent infinite recursion, defaults to 0. + * + * @return - Returns the state of the current dependency group. + */ +DependencyGroup::State DependencyGroup::GetState(const Checkable* child, DependencyType dt, int rstack) const +{ + auto dependencies(GetDependenciesForChild(child)); + size_t reachable = 0, available = 0; + + for (const auto& dependency : dependencies) { + if (dependency->GetParent()->IsReachable(dt, rstack)) { + reachable++; + + // Only reachable parents are considered for availability. If they are unreachable and checks are + // disabled, they could be incorrectly treated as available otherwise. + if (dependency->IsAvailable(dt)) { + available++; + } + } + } + + if (IsRedundancyGroup()) { + // The state of a redundancy group is determined by the best state of any parent. If any parent ist reachable, + // the redundancy group is reachable, analogously for availability. + if (reachable == 0) { + return State::Unreachable; + } else if (available == 0) { + return State::Failed; + } else { + return State::Ok; + } + } else { + // For dependencies without a redundancy group, dependencies.size() will be 1 in almost all cases. It will only + // contain more elements if there are duplicate dependency config objects between two checkables. In this case, + // all of them have to be reachable/available as they don't provide redundancy. + if (reachable < dependencies.size()) { + return State::Unreachable; + } else if (available < dependencies.size()) { + return State::Failed; + } else { + return State::Ok; + } + } +} diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index a9a7bf3725d..cf4af862710 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -123,7 +123,7 @@ class DependencyCycleChecker } // Explicitly configured dependency objects - for (const auto& dep : checkable->GetDependencies()) { + for (const auto& dep : checkable->GetDependencies(/* includePending = */ true)) { m_Stack.emplace_back(dep); AssertNoCycle(dep->GetParent()); m_Stack.pop_back(); @@ -251,16 +251,24 @@ void Dependency::OnAllConfigLoaded() // InitChildParentReferences() has to be called before. VERIFY(m_Child && m_Parent); - m_Child->AddDependency(this); + // Icinga DB will implicitly send config updates for the parent Checkable to refresh its affects_children and + // affected_children columns when registering the dependency from the child Checkable. So, we need to register + // the dependency from the parent Checkable first, otherwise the config update of the parent Checkable will change + // nothing at all. m_Parent->AddReverseDependency(this); + m_Child->AddDependency(this); } void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this); + // Icinga DB will implicitly send config updates for the parent Checkable to refresh its affects_children and + // affected_children columns when removing the dependency from the child Checkable. So, we need to remove the + // dependency from the parent Checkable first, otherwise the config update of the parent Checkable will change + // nothing at all. GetParent()->RemoveReverseDependency(this); + GetChild()->RemoveDependency(this, runtimeRemoved); } bool Dependency::IsAvailable(DependencyType dt) const diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 32bd8e70dc2..b4e206b7f2c 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -3,9 +3,14 @@ #ifndef DEPENDENCY_H #define DEPENDENCY_H +#include "base/shared-object.hpp" +#include "config/configitem.hpp" #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" -#include "config/configitem.hpp" +#include "icinga/timeperiod.hpp" +#include +#include +#include namespace icinga { @@ -60,6 +65,163 @@ class Dependency final : public ObjectImpl static void BeforeOnAllConfigLoadedHandler(const ConfigItems& items); }; +/** + * A DependencyGroup represents a set of dependencies that are somehow related to each other. + * + * Specifically, a DependencyGroup is a container for Dependency objects of different Checkables that share the same + * child -> parent relationship config, thus forming a group of dependencies. All dependencies of a Checkable that + * have the same "redundancy_group" attribute value set are guaranteed to be part of the same DependencyGroup object, + * and another Checkable will join that group if and only if it has identical set of dependencies, that is, the same + * parent(s), same redundancy group name and all other dependency attributes required to form a composite key. + * + * More specifically, let's say we have a dependency graph like this: + * @verbatim + * PP1 PP2 + * /\ /\ + * || || + * ––––||–––––––––––––––||––––– + * P1 - ( "RG1" ) - P2 + * –––––––––––––––––––––––––––– + * /\ /\ + * || || + * C1 C2 + * @endverbatim + * The arrows represent a dependency relationship from bottom to top, i.e. both "C1" and "C2" depend on + * their "RG1" redundancy group, and "P1" and "P2" depend each on their respective parents (PP1, PP2 - no group). + * Now, as one can see, both "C1" and "C2" have identical dependencies, that is, they both depend on the same + * redundancy group "RG1" (these could e.g. be constructed through some Apply Rules). + * + * So, instead of having to maintain two separate copies of that graph, we can bring that imaginary redundancy group + * into reality by putting both "P1" and "P2" into an actual DependencyGroup object. However, we don't really put "P1" + * and "P2" objects into that group, but rather the actual Dependency objects of both child Checkables. Therefore, the + * group wouldn't just contain 2 dependencies, but 4 in total, i.e. 2 for each child Checkable (C1 -> {P1, P2} and + * C2 -> {P1, P2}). This way, both child Checkables can just refer to that very same DependencyGroup object. + * + * However, since not all dependencies are part of a redundancy group, we also have to consider the case where + * a Checkable has dependencies that are not part of any redundancy group, like P1 -> PP1. In such situations, + * each of the child Checkables (e.g. P1, P2) will have their own (sharable) DependencyGroup object just like for RGs. + * This allows us to keep the implementation simple and treat redundant and non-redundant dependencies in the same + * way, without having to introduce any special cases everywhere. So, in the end, we'd have 3 dependency groups in + * total, i.e. one for the redundancy group "RG1" (shared by C1 and C2), and two distinct groups for P1 and P2. + * + * @ingroup icinga + */ +class DependencyGroup final : public SharedObject +{ +public: + DECLARE_PTR_TYPEDEFS(DependencyGroup); + + /** + * Defines the key type of each dependency group members. + * + * This tuple consists of the dependency parent Checkable, the dependency time period (nullptr if not configured), + * the state filter, and the ignore soft states flag. Each of these values influences the availability of the + * dependency object, and thus used to group similar dependencies from different Checkables together. + */ + using CompositeKeyType = std::tuple; + + /** + * Represents the value type of each dependency group members. + * + * It stores the dependency objects of any given Checkable that produce the same composite key (CompositeKeyType). + * In other words, when looking at the dependency graph from the class description, the two dependency objects + * {C1, C2} -> P1 produce the same composite key, thus they are mapped to the same MemberValueType container with + * "C1" and "C2" as their keys respectively. Since Icinga 2 allows to construct different identical dependencies + * (duplicates), we're using a multimap instead of a simple map here. + */ + using MemberValueType = std::unordered_multimap; + using MembersMap = std::map; + + DependencyGroup(String name, const std::set& dependencies); + + static DependencyGroup::Ptr Register(const DependencyGroup::Ptr& dependencyGroup); + static std::pair, bool> Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child); + static size_t GetRegistrySize(); + + static CompositeKeyType MakeCompositeKeyFor(const Dependency::Ptr& dependency); + + /** + * Check whether the current dependency group represents an explicitly configured redundancy group. + * + * @return bool - Returns true if it's a redundancy group, false otherwise. + */ + inline bool IsRedundancyGroup() const + { + return !m_RedundancyGroupName.IsEmpty(); + } + + bool HasChildren() const; + void AddDependency(const Dependency::Ptr& dependency); + void RemoveDependency(const Dependency::Ptr& dependency); + std::vector GetDependenciesForChild(const Checkable* child) const; + void LoadParents(std::set& parents) const; + size_t GetDependenciesCount() const; + + void SetIcingaDBIdentifier(const String& identifier); + String GetIcingaDBIdentifier() const; + + const String& GetRedundancyGroupName() const; + String GetCompositeKey(); + + enum class State { Ok, Failed, Unreachable }; + State GetState(const Checkable* child, DependencyType dt = DependencyState, int rstack = 0) const; + + static boost::signals2::signal OnChildRegistered; + static boost::signals2::signal&, bool)> OnChildRemoved; + +private: + void CopyDependenciesTo(const DependencyGroup::Ptr& dest); + + struct Hash + { + size_t operator()(const DependencyGroup::Ptr& dependencyGroup) const + { + size_t hash = std::hash{}(dependencyGroup->GetRedundancyGroupName()); + for (const auto& [key, group] : dependencyGroup->m_Members) { + boost::hash_combine(hash, key); + } + return hash; + } + }; + + struct Equal + { + bool operator()(const DependencyGroup::Ptr& lhs, const DependencyGroup::Ptr& rhs) const + { + if (lhs->GetRedundancyGroupName() != rhs->GetRedundancyGroupName()) { + return false; + } + + return std::equal( + lhs->m_Members.begin(), lhs->m_Members.end(), + rhs->m_Members.begin(), rhs->m_Members.end(), + [](const auto& l, const auto& r) { return l.first == r.first; } + ); + } + }; + +private: + mutable std::mutex m_Mutex; + /** + * This identifier is used by Icinga DB to cache the unique hash of this dependency group. + * + * For redundancy groups, once Icinga DB sets this identifier, it will never change again for the lifetime + * of the object. For non-redundant dependency groups, this identifier is (mis)used to cache the shared edge + * state ID of the group. Specifically, non-redundant dependency groups are irrelevant for Icinga DB, so since + * this field isn't going to be used for anything else, we use it to cache the computed shared edge state ID. + * Likewise, if that gets set, it will never change again for the lifetime of the object as well. + */ + String m_IcingaDBIdentifier; + String m_RedundancyGroupName; + MembersMap m_Members; + + using RegistryType = std::unordered_set; + + // The global registry of dependency groups. + static std::mutex m_RegistryMutex; + static RegistryType m_Registry; +}; + } #endif /* DEPENDENCY_H */ diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 41de7ba23cf..a033420ea8d 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -20,6 +20,8 @@ public: class Dependency : CustomVarObject < DependencyNameComposer { + activation_priority -10; + load_after Host; load_after Service; @@ -77,18 +79,18 @@ class Dependency : CustomVarObject < DependencyNameComposer }}} }; - [config] String redundancy_group; + [config, no_user_modify] String redundancy_group; - [config, navigation] name(TimePeriod) period (PeriodRaw) { + [config, no_user_modify, navigation] name(TimePeriod) period (PeriodRaw) { navigate {{{ return TimePeriod::GetByName(GetPeriodRaw()); }}} }; - [config] array(Value) states; + [config, no_user_modify] array(Value) states; [no_user_view, no_user_modify] int state_filter_real (StateFilter); - [config] bool ignore_soft_states { + [config, no_user_modify] bool ignore_soft_states { default {{{ return true; }}} }; diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 920251969f2..40580a35855 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -19,6 +19,7 @@ #include "icinga/command.hpp" #include "icinga/compatutility.hpp" #include "icinga/customvarobject.hpp" +#include "icinga/dependency.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" @@ -94,8 +95,11 @@ void IcingaDB::ConfigStaticInitialize() AcknowledgementClearedHandler(checkable, removedBy, changeTime); }); - Checkable::OnReachabilityChanged.connect([](const Checkable::Ptr&, const CheckResult::Ptr&, std::set children, const MessageOrigin::Ptr&) { - IcingaDB::ReachabilityChangeHandler(children); + Checkable::OnReachabilityChanged.connect([](const Checkable::Ptr& parent, const CheckResult::Ptr&, std::set, const MessageOrigin::Ptr&) { + // Icinga DB Web needs to know about the reachability of all children, not just the direct ones. + // These might get updated with their next check result anyway, but we can't rely on that, since + // they might not be actively checked or have a very high check interval. + IcingaDB::ReachabilityChangeHandler(parent->GetAllChildren()); }); /* triggered on create, update and delete objects */ @@ -106,6 +110,9 @@ void IcingaDB::ConfigStaticInitialize() IcingaDB::VersionChangedHandler(object); }); + DependencyGroup::OnChildRegistered.connect(&IcingaDB::DependencyGroupChildRegisteredHandler); + DependencyGroup::OnChildRemoved.connect(&IcingaDB::DependencyGroupChildRemovedHandler); + /* downtime start */ Downtime::OnDowntimeTriggered.connect(&IcingaDB::DowntimeStartedHandler); /* fixed/flexible downtime end or remove */ @@ -174,7 +181,7 @@ void IcingaDB::ConfigStaticInitialize() void IcingaDB::UpdateAllConfigObjects() { m_Rcon->Sync(); - m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "5"}, Prio::Heartbeat); + m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "6"}, Prio::Heartbeat); Log(LogInformation, "IcingaDB") << "Starting initial config/status dump"; double startTime = Utility::GetTime(); @@ -203,10 +210,19 @@ void IcingaDB::UpdateAllConfigObjects() m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "MAXLEN", "1", "*", "key", "*", "state", "wip"}, Prio::Config); const std::vector globalKeys = { - m_PrefixConfigObject + "customvar", - m_PrefixConfigObject + "action:url", - m_PrefixConfigObject + "notes:url", - m_PrefixConfigObject + "icon:image", + m_PrefixConfigObject + "customvar", + m_PrefixConfigObject + "action:url", + m_PrefixConfigObject + "notes:url", + m_PrefixConfigObject + "icon:image", + + // These keys aren't tied to a specific Checkable type but apply to both "Host" and "Service" types, + // and as such we've to make sure to clear them before we actually start dumping the actual objects. + // This allows us to wait on both types to be dumped before we send a config dump done signal for those keys. + m_PrefixConfigObject + "dependency:node", + m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "dependency:edge:state", + m_PrefixConfigObject + "redundancygroup", + m_PrefixConfigObject + "redundancygroup:state", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -217,6 +233,7 @@ void IcingaDB::UpdateAllConfigObjects() m_DumpedGlobals.ActionUrl.Reset(); m_DumpedGlobals.NotesUrl.Reset(); m_DumpedGlobals.IconImage.Reset(); + m_DumpedGlobals.DependencyGroup.Reset(); }); upq.ParallelFor(types, false, [this](const Type::Ptr& type) { @@ -259,11 +276,6 @@ void IcingaDB::UpdateAllConfigObjects() upqObjectType.ParallelFor(objectChunks, [&](decltype(objectChunks)::const_reference chunk) { std::map> hMSets; - // Two values are appended per object: Object ID (Hash encoded) and Object State (IcingaDB::SerializeState() -> JSON encoded) - std::vector states = {"HMSET", m_PrefixConfigObject + lcType + ":state"}; - // Two values are appended per object: Object ID (Hash encoded) and State Checksum ({ "checksum": checksum } -> JSON encoded) - std::vector statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"}; - std::vector > transaction = {{"MULTI"}}; std::vector hostZAdds = {"ZADD", "icinga:nextupdate:host"}, serviceZAdds = {"ZADD", "icinga:nextupdate:service"}; auto skimObjects ([&]() { @@ -303,9 +315,11 @@ void IcingaDB::UpdateAllConfigObjects() String objectKey = GetObjectIdentifier(object); Dictionary::Ptr state = SerializeState(dynamic_pointer_cast(object)); + auto& states = hMSets[m_PrefixConfigObject + lcType + ":state"]; states.emplace_back(objectKey); states.emplace_back(JsonEncode(state)); + auto& statesChksms = hMSets[m_PrefixConfigCheckSum + lcType + ":state"]; statesChksms.emplace_back(objectKey); statesChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(state)}}))); } @@ -314,27 +328,9 @@ void IcingaDB::UpdateAllConfigObjects() if (!(bulkCounter % 100)) { skimObjects(); - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - if (states.size() > 2) { - transaction.emplace_back(std::move(states)); - transaction.emplace_back(std::move(statesChksms)); - states = {"HMSET", m_PrefixConfigObject + lcType + ":state"}; - statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"}; - } + ExecuteRedisTransaction(rcon, hMSets, {}); hMSets = decltype(hMSets)(); - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); - transaction = {{"MULTI"}}; - } } auto checkable (dynamic_pointer_cast(object)); @@ -357,22 +353,7 @@ void IcingaDB::UpdateAllConfigObjects() skimObjects(); - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - if (states.size() > 2) { - transaction.emplace_back(std::move(states)); - transaction.emplace_back(std::move(statesChksms)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); - } + ExecuteRedisTransaction(rcon, hMSets, {}); for (auto zAdds : {&hostZAdds, &serviceZAdds}) { if (zAdds->size() > 2u) { @@ -788,6 +769,8 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } + InsertCheckableDependencies(checkable, hMSets, runtimeUpdate ? &runtimeUpdates : nullptr); + return; } @@ -1121,6 +1104,195 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } +/** + * Inserts the dependency data for a Checkable object into the given Redis HMSETs and runtime updates. + * + * This function is responsible for serializing the in memory representation Checkable dependencies into + * Redis HMSETs and runtime updates (if any) according to the Icinga DB schema. The serialized data consists + * of the following Redis HMSETs: + * - RedisKey::DependencyNode: Contains dependency node data representing each host, service, and redundancy group + * in the dependency graph. + * - RedisKey::DependencyEdge: Dependency edge information representing all connections between the nodes. + * - RedisKey::RedundancyGroup: Redundancy group data representing all redundancy groups in the graph. + * - RedisKey::RedundancyGroupState: State information for redundancy groups. + * - RedisKey::DependencyEdgeState: State information for (each) dependency edge. Multiple edges may share the + * same state. + * + * If the `onlyDependencyGroup` parameter is set, only dependencies from this group are processed. This is useful + * when only a specific dependency group should be processed, e.g. during runtime updates. For initial config dumps, + * it shouldn't be necessary to set the `runtimeUpdates` and `onlyDependencyGroup` parameters. + * + * @param checkable The checkable object to extract dependencies from. + * @param hMSets The map of Redis HMSETs to insert the dependency data into. + * @param runtimeUpdates If set, runtime updates are additionally added to this vector. + * @param onlyDependencyGroup If set, only process dependency objects from this group. + */ +void IcingaDB::InsertCheckableDependencies( + const Checkable::Ptr& checkable, + std::map& hMSets, + std::vector* runtimeUpdates, + const DependencyGroup::Ptr& onlyDependencyGroup +) +{ + // Only generate a dependency node event if the Checkable is actually part of some dependency graph. + // That's, it either depends on other Checkables or others depend on it, and in both cases, we have + // to at least generate a dependency node entry for it. + if (!checkable->HasAnyDependencies()) { + return; + } + + // First and foremost, generate a dependency node entry for the provided Checkable object and insert it into + // the HMSETs map and if set, the `runtimeUpdates` vector. + auto [host, service] = GetHostService(checkable); + auto checkableId(GetObjectIdentifier(checkable)); + { + Dictionary::Ptr data(new Dictionary{{"environment_id", m_EnvironmentId}, {"host_id", GetObjectIdentifier(host)}}); + if (service) { + data->Set("service_id", checkableId); + } + + AddDataToHmSets(hMSets, RedisKey::DependencyNode, checkableId, data); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, checkableId, m_PrefixConfigObject + "dependency:node", data); + } + } + + // If `onlyDependencyGroup` is provided, process the dependencies only from that group; otherwise, + // retrieve all the dependency groups that the Checkable object is part of. + std::vector dependencyGroups{onlyDependencyGroup}; + if (!onlyDependencyGroup) { + dependencyGroups = checkable->GetDependencyGroups(); + } + + for (auto& dependencyGroup : dependencyGroups) { + String edgeFromNodeId(checkableId); + bool syncSharedEdgeState(false); + + if (!dependencyGroup->IsRedundancyGroup()) { + // Non-redundant dependency groups are just placeholders and never get synced to Redis, thus just figure + // out whether we have to sync the shared edge state. For runtime updates the states are sent via the + // UpdateDependenciesState() method, thus we don't have to sync them here. + syncSharedEdgeState = !runtimeUpdates && m_DumpedGlobals.DependencyGroup.IsNew(dependencyGroup->GetCompositeKey()); + } else { + auto redundancyGroupId(HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()})); + dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); + + edgeFromNodeId = redundancyGroupId; + + // During the initial config sync, multiple children can depend on the same redundancy group, sync it only + // the first time it is encountered. Though, if this is a runtime update, we have to re-serialize and sync + // the redundancy group unconditionally, as we don't know whether it was already synced or the context that + // triggered this update. + if (runtimeUpdates || m_DumpedGlobals.DependencyGroup.IsNew(redundancyGroupId)) { + Dictionary::Ptr groupData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"display_name", dependencyGroup->GetRedundancyGroupName()}, + }); + // Set/refresh the redundancy group data in the Redis HMSETs (redundancy_group database table). + AddDataToHmSets(hMSets, RedisKey::RedundancyGroup, redundancyGroupId, groupData); + + Dictionary::Ptr nodeData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + }); + // Obviously, the redundancy group is part of some dependency chain, thus we have to generate + // dependency node entry for it as well. + AddDataToHmSets(hMSets, RedisKey::DependencyNode, redundancyGroupId, nodeData); + + if (runtimeUpdates) { + // Send the same data sent to the Redis HMSETs to the runtime updates stream as well. + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } else { + syncSharedEdgeState = true; + + // Serialize and sync the redundancy group state information a) to the RedundancyGroupState and b) + // to the DependencyEdgeState HMSETs. The latter is shared by all child Checkables of the current + // redundancy group, and since they all depend on the redundancy group, the state of that group is + // basically the state of the dependency edges between the children and the redundancy group. + auto stateAttrs(SerializeRedundancyGroupState(checkable, dependencyGroup)); + AddDataToHmSets(hMSets, RedisKey::RedundancyGroupState, redundancyGroupId, stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, redundancyGroupId, Dictionary::Ptr(new Dictionary{ + {"id", redundancyGroupId}, + {"environment_id", m_EnvironmentId}, + {"failed", stateAttrs->Get("failed")}, + })); + } + } + + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", checkableId}, + {"to_node_id", redundancyGroupId}, + // All redundancy group members share the same state, thus use the group ID as a reference. + {"dependency_edge_state_id", redundancyGroupId}, + {"display_name", dependencyGroup->GetRedundancyGroupName()}, + }); + + // Generate a dependency edge entry representing the connection between the Checkable and the redundancy + // group. This Checkable dependes on the redundancy group (is a child of it), thus the "dependency_edge_state_id" + // is set to the redundancy group ID. Note that if this group has multiple children, they all will have the + // same "dependency_edge_state_id" value. + auto edgeId(HashValue(new Array{checkableId, redundancyGroupId})); + AddDataToHmSets(hMSets, RedisKey::DependencyEdge, edgeId, data); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } + + auto dependencies(dependencyGroup->GetDependenciesForChild(checkable.get())); + // Sort the dependencies by their parent Checkable object to ensure that all dependency objects that share the + // same parent Checkable are placed next to each other in the container. See the while loop below for more info! + std::sort(dependencies.begin(), dependencies.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + + // Traverse through each dependency objects within the current dependency group the provided Checkable depend + // on and generate a dependency edge entry. The generated dependency edge "from_node_id" may vary depending on + // whether the dependency group is a redundancy group or not. If it's a redundancy group, the "from_node_id" + // will be the redundancy group ID; otherwise, it will be the current Checkable ID. However, the "to_node_id" + // value will always be the parent Checkable ID of the dependency object. + for (auto it(dependencies.begin()); it != dependencies.end(); /* no increment */) { + auto dependency(*it); + auto parent(dependency->GetParent()); + auto displayName(dependency->GetShortName()); + + Dictionary::Ptr edgeStateAttrs(SerializeDependencyEdgeState(dependencyGroup, dependency)); + + // In case there are multiple Dependency objects with the same parent, these are merged into a single edge + // to prevent duplicate edges in the resulting graph. All objects with the same parent were placed next to + // each other by the sort function above. + // + // Additionally, the iterator for the surrounding loop is incremented by this loop: after it has finished, + // "it" will either point to the next dependency with a different parent or to the end of the container. + while (++it != dependencies.end() && (*it)->GetParent() == parent) { + displayName += ", " + (*it)->GetShortName(); + if (syncSharedEdgeState && edgeStateAttrs->Get("failed") == false) { + edgeStateAttrs = SerializeDependencyEdgeState(dependencyGroup, *it); + } + } + + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", edgeFromNodeId}, + {"to_node_id", GetObjectIdentifier(parent)}, + {"dependency_edge_state_id", edgeStateAttrs->Get("id")}, + {"display_name", std::move(displayName)}, + }); + + auto edgeId(HashValue(new Array{data->Get("from_node_id"), data->Get("to_node_id")})); + AddDataToHmSets(hMSets, RedisKey::DependencyEdge, edgeId, data); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } else if (syncSharedEdgeState) { + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, edgeStateAttrs->Get("id"), edgeStateAttrs); + } + } + } +} + /** * Update the state information of a checkable in Redis. * @@ -1177,6 +1349,115 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) } } +/** + * Send dependencies state information of the given Checkable to Redis. + * + * If the dependencyGroup parameter is set, only the dependencies state of that group are sent. Otherwise, all + * dependency groups of the provided Checkable are processed. + * + * @param checkable The Checkable you want to send the dependencies state update for + * @param onlyDependencyGroup If set, send state updates only for this dependency group and its dependencies. + * @param seenGroups A container to track already processed DependencyGroups to avoid duplicate state updates. + */ +void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup, + std::set* seenGroups) const +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + std::vector dependencyGroups{onlyDependencyGroup}; + if (!onlyDependencyGroup) { + dependencyGroups = checkable->GetDependencyGroups(); + if (dependencyGroups.empty()) { + return; + } + } + + RedisConnection::Queries streamStates; + auto addDependencyStateToStream([this, &streamStates](const String& redisKey, const Dictionary::Ptr& stateAttrs) { + RedisConnection::Query xAdd{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", "runtime_type", "upsert", + "redis_key", redisKey + }; + ObjectLock olock(stateAttrs); + for (auto& [key, value] : stateAttrs) { + xAdd.emplace_back(key); + xAdd.emplace_back(IcingaToStreamValue(value)); + } + streamStates.emplace_back(std::move(xAdd)); + }); + + std::map hMSets; + for (auto& dependencyGroup : dependencyGroups) { + bool isRedundancyGroup(dependencyGroup->IsRedundancyGroup()); + if (isRedundancyGroup && dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // Way too soon! The Icinga DB hash will be set during the initial config dump, but this state + // update seems to occur way too early. So, we've to skip it for now and wait for the next one. + // The m_ConfigDumpInProgress flag is probably still set to true at this point! + continue; + } + + if (seenGroups && !seenGroups->insert(dependencyGroup.get()).second) { + // Usually, if the seenGroups set is provided, IcingaDB is triggering a runtime state update for ALL + // children of a given initiator Checkable (parent). In such cases, we may end up with lots of useless + // state updates as all the children of a non-redundant group a) share the same entry in the database b) + // it doesn't matter which child triggers the state update first all the subsequent updates are just useless. + // + // Likewise, for redundancy groups, all children of a redundancy group share the same set of parents + // and thus the resulting state information would be the same from each child Checkable perspective. + // So, serializing the redundancy group state information only once is sufficient. + continue; + } + + auto dependencies(dependencyGroup->GetDependenciesForChild(checkable.get())); + std::sort(dependencies.begin(), dependencies.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + for (auto it(dependencies.begin()); it != dependencies.end(); /* no increment */) { + const auto& dependency(*it); + + Dictionary::Ptr stateAttrs; + // Note: The following loop is intended to cover some possible special cases but may not occur in practice + // that often. That is, having two or more dependency objects that point to the same parent Checkable. + // So, traverse all those duplicates and merge their relevant state information into a single edge. + for (; it != dependencies.end() && (*it)->GetParent() == dependency->GetParent(); ++it) { + if (!stateAttrs || stateAttrs->Get("failed") == false) { + stateAttrs = SerializeDependencyEdgeState(dependencyGroup, *it); + } + } + + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, stateAttrs->Get("id"), stateAttrs); + } + + if (isRedundancyGroup) { + Dictionary::Ptr stateAttrs(SerializeRedundancyGroupState(checkable, dependencyGroup)); + + Dictionary::Ptr sharedGroupState(stateAttrs->ShallowClone()); + sharedGroupState->Remove("redundancy_group_id"); + sharedGroupState->Remove("is_reachable"); + sharedGroupState->Remove("last_state_change"); + + addDependencyStateToStream(m_PrefixConfigObject + "redundancygroup:state", stateAttrs); + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", sharedGroupState); + AddDataToHmSets(hMSets, RedisKey::RedundancyGroupState, dependencyGroup->GetIcingaDBIdentifier(), stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, dependencyGroup->GetIcingaDBIdentifier(), sharedGroupState); + } + } + + if (!streamStates.empty()) { + RedisConnection::Queries queries; + for (auto& [redisKey, query] : hMSets) { + query.insert(query.begin(), {"HSET", redisKey}); + queries.emplace_back(std::move(query)); + } + + m_Rcon->FireAndForgetQueries(std::move(queries), Prio::RuntimeStateSync); + m_Rcon->FireAndForgetQueries(std::move(streamStates), Prio::RuntimeStateStream, {0, 1}); + } +} + // Used to update a single object, used for runtime updates void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { @@ -1194,34 +1475,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile); } - std::vector > transaction = {{"MULTI"}}; - - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - for (auto& objectAttributes : runtimeUpdates) { - std::vector xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); - ObjectLock olock(objectAttributes); - - for (const Dictionary::Pair& kv : objectAttributes) { - String value = IcingaToStreamValue(kv.second); - if (!value.IsEmpty()) { - xAdd.emplace_back(kv.first); - xAdd.emplace_back(value); - } - } - - transaction.emplace_back(std::move(xAdd)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); - } + ExecuteRedisTransaction(m_Rcon, hMSets, runtimeUpdates); if (checkable) { SendNextUpdate(checkable); @@ -1308,6 +1562,11 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("notes", checkable->GetNotes()); attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); + if (size_t totalChildren (checkable->GetAllChildrenCount()); totalChildren > 0) { + // Only set the Redis key if the Checkable has actually some child dependencies. + attributes->Set("total_children", totalChildren); + } + attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); @@ -2580,6 +2839,98 @@ void IcingaDB::SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dict } } +void IcingaDB::SendDependencyGroupChildRegistered(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + std::vector runtimeUpdates; + std::map hMSets; + InsertCheckableDependencies(child, hMSets, &runtimeUpdates, dependencyGroup); + ExecuteRedisTransaction(m_Rcon, hMSets, runtimeUpdates); + + UpdateState(child, StateUpdate::Full); + UpdateDependenciesState(child, dependencyGroup); + + std::set parents; + dependencyGroup->LoadParents(parents); + for (const auto& parent : parents) { + // The total_children and affects_children columns might now have different outcome, so update the parent + // Checkable as well. The grandparent Checkable may still have wrong numbers of total children, though it's not + // worth traversing the whole tree way up and sending config updates for each one of them, as the next Redis + // config dump is going to fix it anyway. + SendConfigUpdate(parent, true); + } +} + +void IcingaDB::SendDependencyGroupChildRemoved( + const DependencyGroup::Ptr& dependencyGroup, + const std::vector& dependencies, + bool removeGroup +) +{ + if (!m_Rcon || !m_Rcon->IsConnected() || dependencies.empty()) { + return; + } + + Checkable::Ptr child; + std::set detachedParents; + for (const auto& dependency : dependencies) { + child = dependency->GetChild(); // All dependencies have the same child. + const auto& parent(dependency->GetParent()); + if (auto [_, inserted] = detachedParents.insert(dependency->GetParent().get()); inserted) { + String edgeId; + if (dependencyGroup->IsRedundancyGroup()) { + // If the redundancy group has no members left, it's going to be removed as well, so we need to + // delete dependency edges from that group to the parent Checkables. + if (removeGroup) { + auto id(HashValue(new Array{dependencyGroup->GetIcingaDBIdentifier(), GetObjectIdentifier(parent)})); + DeleteRelationship(id, RedisKey::DependencyEdge); + DeleteState(id, RedisKey::DependencyEdgeState); + } + + // Remove the connection from the child Checkable to the redundancy group. + edgeId = HashValue(new Array{GetObjectIdentifier(child), dependencyGroup->GetIcingaDBIdentifier()}); + } else { + // Remove the edge between the parent and child Checkable linked through the removed dependency. + edgeId = HashValue(new Array{GetObjectIdentifier(child), GetObjectIdentifier(parent)}); + } + + DeleteRelationship(edgeId, RedisKey::DependencyEdge); + + // The total_children and affects_children columns might now have different outcome, so update the parent + // Checkable as well. The grandparent Checkable may still have wrong numbers of total children, though it's + // not worth traversing the whole tree way up and sending config updates for each one of them, as the next + // Redis config dump is going to fix it anyway. + SendConfigUpdate(parent, true); + + if (!parent->HasAnyDependencies()) { + // If the parent Checkable isn't part of any other dependency chain anymore, drop its dependency node entry. + DeleteRelationship(GetObjectIdentifier(parent), RedisKey::DependencyNode); + } + } + } + + if (removeGroup && dependencyGroup->IsRedundancyGroup()) { + String redundancyGroupId(dependencyGroup->GetIcingaDBIdentifier()); + DeleteRelationship(redundancyGroupId, RedisKey::DependencyNode); + DeleteRelationship(redundancyGroupId, RedisKey::RedundancyGroup); + + DeleteState(redundancyGroupId, RedisKey::RedundancyGroupState); + DeleteState(redundancyGroupId, RedisKey::DependencyEdgeState); + } else if (removeGroup) { + // Note: The Icinga DB identifier of a non-redundant dependency group is used as the edge state ID + // and shared by all of its dependency objects. See also SerializeDependencyEdgeState() for details. + DeleteState(dependencyGroup->GetIcingaDBIdentifier(), RedisKey::DependencyEdgeState); + } + + if (!child->HasAnyDependencies()) { + // If the child Checkable has no parent and reverse dependencies, we can safely remove the dependency node. + DeleteRelationship(GetObjectIdentifier(child), RedisKey::DependencyNode); + } +} + Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -2623,6 +2974,7 @@ Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); + attrs->Set("affects_children", checkable->AffectsChildren()); CheckResult::Ptr cr = checkable->GetLastCheckResult(); @@ -2758,8 +3110,10 @@ void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object, const CheckRe void IcingaDB::ReachabilityChangeHandler(const std::set& children) { for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { + std::set seenGroups; for (auto& checkable : children) { rw->UpdateState(checkable, StateUpdate::Full); + rw->UpdateDependenciesState(checkable, nullptr, &seenGroups); } } } @@ -2856,6 +3210,20 @@ void IcingaDB::NextCheckUpdatedHandler(const Checkable::Ptr& checkable) } } +void IcingaDB::DependencyGroupChildRegisteredHandler(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup) +{ + for (const auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupChildRegistered(child, dependencyGroup); + } +} + +void IcingaDB::DependencyGroupChildRemovedHandler(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup) +{ + for (const auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupChildRemoved(dependencyGroup, dependencies, removeGroup); + } +} + void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) { for (auto& rw : ConfigType::GetObjectsByType()) { /* Host state changes affect is_handled and severity of services. */ @@ -2973,3 +3341,139 @@ void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithou m_Rcon->FireAndForgetQueries(queries, Prio::Config); } + +void IcingaDB::DeleteRelationship(const String& id, RedisKey redisKey, bool hasChecksum) +{ + switch (redisKey) { + case RedisKey::RedundancyGroup: + DeleteRelationship(id, "redundancygroup", hasChecksum); + break; + case RedisKey::DependencyNode: + DeleteRelationship(id, "dependency:node", hasChecksum); + break; + case RedisKey::DependencyEdge: + DeleteRelationship(id, "dependency:edge", hasChecksum); + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid RedisKey provided")); + } +} + +void IcingaDB::DeleteState(const String& id, RedisKey redisKey, bool hasChecksum) const +{ + String redisKeyWithoutPrefix; + switch (redisKey) { + case RedisKey::RedundancyGroupState: + redisKeyWithoutPrefix = "redundancygroup:state"; + break; + case RedisKey::DependencyEdgeState: + redisKeyWithoutPrefix = "dependency:edge:state"; + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid state RedisKey provided")); + } + + Log(LogNotice, "IcingaDB") + << "Deleting state " << std::quoted(redisKeyWithoutPrefix.CStr()) << " -> " << std::quoted(id.CStr()); + + RedisConnection::Queries hdels; + if (hasChecksum) { + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigCheckSum + redisKeyWithoutPrefix, id}); + } + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigObject + redisKeyWithoutPrefix, id}); + + m_Rcon->FireAndForgetQueries(std::move(hdels), Prio::RuntimeStateSync); + // TODO: This is currently purposefully commented out due to how Icinga DB (Go) handles runtime state + // upsert and delete events. See https://github.com/Icinga/icingadb/pull/894 for more details. + /*m_Rcon->FireAndForgetQueries({{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", + "redis_key", m_PrefixConfigObject + redisKeyWithoutPrefix, "id", id, "runtime_type", "delete" + }}, Prio::RuntimeStateStream, {0, 1});*/ +} + +/** + * Add the provided data to the Redis HMSETs map. + * + * Adds the provided data to the Redis HMSETs map for the provided Redis key. The actual Redis key is determined by + * the provided RedisKey enum. The data will be json encoded before being added to the Redis HMSETs map. + * + * @param hMSets The map of RedisConnection::Query you want to add the data to. + * @param redisKey The key of the Redis object you want to add the data to. + * @param id Unique Redis identifier for the provided data. + * @param data The actual data you want to add the Redis HMSETs map. + */ +void IcingaDB::AddDataToHmSets(std::map& hMSets, RedisKey redisKey, const String& id, const Dictionary::Ptr& data) const +{ + RedisConnection::Query* query; + switch (redisKey) { + case RedisKey::RedundancyGroup: + query = &hMSets[m_PrefixConfigObject + "redundancygroup"]; + break; + case RedisKey::DependencyNode: + query = &hMSets[m_PrefixConfigObject + "dependency:node"]; + break; + case RedisKey::DependencyEdge: + query = &hMSets[m_PrefixConfigObject + "dependency:edge"]; + break; + case RedisKey::RedundancyGroupState: + query = &hMSets[m_PrefixConfigObject + "redundancygroup:state"]; + break; + case RedisKey::DependencyEdgeState: + query = &hMSets[m_PrefixConfigObject + "dependency:edge:state"]; + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid RedisKey provided")); + } + + query->emplace_back(id); + query->emplace_back(JsonEncode(data)); +} + +/** + * Execute the provided HMSET values and runtime updates in a single Redis transaction on the provided Redis connection. + * + * The HMSETs should just contain the necessary key value pairs to be set in Redis, i.e, without the HMSET command + * itself. This function will then go through each of the map keys and prepend the HMSET command when transforming the + * map into valid Redis queries. Likewise, the runtime updates should just contain the key value pairs to be streamed + * to the icinga:runtime pipeline, and this function will generate a XADD query for each one of the vector elements. + * + * @param rcon The Redis connection to execute the transaction on. + * @param hMSets A map of Redis keys and their respective HMSET values. + * @param runtimeUpdates A list of dictionaries to be sent to the icinga:runtime stream. + */ +void IcingaDB::ExecuteRedisTransaction(const RedisConnection::Ptr& rcon, std::map& hMSets, + const std::vector& runtimeUpdates) +{ + RedisConnection::Queries transaction{{"MULTI"}}; + for (auto& [redisKey, query] : hMSets) { + if (!query.empty()) { + query.insert(query.begin(), {"HSET", redisKey}); + transaction.emplace_back(std::move(query)); + } + } + + for (auto& attrs : runtimeUpdates) { + RedisConnection::Query xAdd{"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}; + + ObjectLock olock(attrs); + for (auto& [key, value] : attrs) { + if (auto streamVal(IcingaToStreamValue(value)); !streamVal.IsEmpty()) { + xAdd.emplace_back(key); + xAdd.emplace_back(std::move(streamVal)); + } + } + + transaction.emplace_back(std::move(xAdd)); + } + + if (transaction.size() > 1) { + transaction.emplace_back(RedisConnection::Query{"EXEC"}); + if (!runtimeUpdates.empty()) { + rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); + } else { + // This is likely triggered by the initial Redis config dump, so a) we don't need to record the number of + // affected objects and b) we don't really know how many objects are going to be affected by this tx. + rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); + } + } +} diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 35f503ab53d..89e5a5031fd 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -159,6 +159,66 @@ Dictionary::Ptr IcingaDB::SerializeVars(const Dictionary::Ptr& vars) return res; } +/** + * Serialize a dependency edge state for Icinga DB + * + * @param dependencyGroup The state of the group the dependency is part of. + * @param dep The dependency object to serialize. + * + * @return A dictionary with the serialized state. + */ +Dictionary::Ptr IcingaDB::SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep) +{ + String edgeStateId; + // The edge state ID is computed a bit differently depending on whether this is for a redundancy group or not. + // For redundancy groups, the state ID is supposed to represent the connection state between the redundancy group + // and the parent Checkable of the given dependency. Hence, the outcome will always be different for each parent + // Checkable of the redundancy group. + if (dependencyGroup->IsRedundancyGroup()) { + edgeStateId = HashValue(new Array{ + dependencyGroup->GetIcingaDBIdentifier(), + GetObjectIdentifier(dep->GetParent()), + }); + } else if (dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // For non-redundant dependency groups, on the other hand, all dependency objects within that group will + // always have the same parent Checkable. Likewise, the state ID will be always the same as well it doesn't + // matter which dependency object is used to compute it. Therefore, it's sufficient to compute it only once + // and all the other dependency objects can reuse the cached state ID. + edgeStateId = HashValue(new Array{dependencyGroup->GetCompositeKey(), GetObjectIdentifier(dep->GetParent())}); + dependencyGroup->SetIcingaDBIdentifier(edgeStateId); + } else { + // Use the already computed state ID for the dependency group. + edgeStateId = dependencyGroup->GetIcingaDBIdentifier(); + } + + return new Dictionary{ + {"id", std::move(edgeStateId)}, + {"environment_id", m_EnvironmentId}, + {"failed", !dep->IsAvailable(DependencyState) || !dep->GetParent()->IsReachable()} + }; +} + +/** + * Serialize the provided redundancy group state attributes. + * + * @param child The child checkable object to serialize the state for. + * @param redundancyGroup The redundancy group object to serialize the state for. + * + * @return A dictionary with the serialized redundancy group state. + */ +Dictionary::Ptr IcingaDB::SerializeRedundancyGroupState(const Checkable::Ptr& child, const DependencyGroup::Ptr& redundancyGroup) +{ + auto state(redundancyGroup->GetState(child.get())); + return new Dictionary{ + {"id", redundancyGroup->GetIcingaDBIdentifier()}, + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroup->GetIcingaDBIdentifier()}, + {"failed", state != DependencyGroup::State::Ok}, + {"is_reachable", state != DependencyGroup::State::Unreachable}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}, + }; +} + const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type) { switch (type) { diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index b943d7f4b1f..af58a977db7 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -90,6 +90,15 @@ class IcingaDB : public ObjectImpl Full = Volatile | RuntimeOnly, }; + enum class RedisKey : uint8_t + { + RedundancyGroup, + DependencyNode, + DependencyEdge, + RedundancyGroupState, + DependencyEdgeState, + }; + void OnConnectedHandler(); void PublishStatsTimerHandler(); @@ -101,8 +110,12 @@ class IcingaDB : public ObjectImpl void DeleteKeys(const RedisConnection::Ptr& conn, const std::vector& keys, RedisConnection::QueryPriority priority); std::vector GetTypeOverwriteKeys(const String& type); std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); + void InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr, + std::set* seenGroups = nullptr) const; void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -112,6 +125,9 @@ class IcingaDB : public ObjectImpl void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false); + void DeleteRelationship(const String& id, RedisKey redisKey, bool hasChecksum = false); + void DeleteState(const String& id, RedisKey redisKey, bool hasChecksum = false) const; + void AddDataToHmSets(std::map& hMSets, RedisKey redisKey, const String& id, const Dictionary::Ptr& data) const; void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, @@ -136,6 +152,8 @@ class IcingaDB : public ObjectImpl void SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + void SendDependencyGroupChildRegistered(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup); + void SendDependencyGroupChildRemoved(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup); void ForwardHistoryEntries(); @@ -157,6 +175,8 @@ class IcingaDB : public ObjectImpl static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0)); static const char* GetNotificationTypeByEnum(NotificationType type); static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars); + static Dictionary::Ptr SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep); + static Dictionary::Ptr SerializeRedundancyGroupState(const Checkable::Ptr& child, const DependencyGroup::Ptr& redundancyGroup); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); @@ -180,6 +200,8 @@ class IcingaDB : public ObjectImpl static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime); static void NewCheckResultHandler(const Checkable::Ptr& checkable); static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable); + static void DependencyGroupChildRegisteredHandler(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup); + static void DependencyGroupChildRemovedHandler(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup); static void HostProblemChangedHandler(const Service::Ptr& service); static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry); static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime); @@ -195,6 +217,9 @@ class IcingaDB : public ObjectImpl static void CommandArgumentsChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); static void CustomVarsChangedHandler(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + static void ExecuteRedisTransaction(const RedisConnection::Ptr& rcon, std::map& hMSets, + const std::vector& runtimeUpdates); + void AssertOnWorkQueue(); void ExceptionHandler(boost::exception_ptr exp); @@ -225,7 +250,7 @@ class IcingaDB : public ObjectImpl std::atomic_size_t m_PendingRcons; struct { - DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; + DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage, DependencyGroup; } m_DumpedGlobals; // m_EnvironmentId is shared across all IcingaDB objects (typically there is at most one, but it is perfectly fine diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c4b1041ddb5..810b35befd7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -232,6 +232,10 @@ add_boost_test(base icinga_checkresult/service_flapping_notification icinga_checkresult/suppressed_notification icinga_dependencies/multi_parent + icinga_dependencies/push_dependency_groups_to_registry + icinga_dependencies/default_redundancy_group_registration_unregistration + icinga_dependencies/simple_redundancy_group_registration_unregistration + icinga_dependencies/mixed_redundancy_group_registration_unregsitration icinga_notification/strings icinga_notification/state_filter icinga_notification/type_filter diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 929b6ca0de5..b1fbf7b4c1c 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -9,6 +9,67 @@ using namespace icinga; BOOST_AUTO_TEST_SUITE(icinga_dependencies) +static Host::Ptr CreateHost(const std::string& name, bool pushDependencyGroupsToRegistry = true) +{ + Host::Ptr host = new Host(); + host->SetName(name); + if (pushDependencyGroupsToRegistry) { + host->PushDependencyGroupsToRegistry(); + } + return host; +} + +static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const String& name) +{ + Dependency::Ptr dep = new Dependency(); + dep->SetParent(parent); + dep->SetChild(child); + dep->SetName(name + "!" + child->GetName()); + return dep; +} + +static void RegisterDependency(Dependency::Ptr dep, const String& redundancyGroup) +{ + dep->SetRedundancyGroup(redundancyGroup); + dep->GetChild()->AddDependency(dep); + dep->GetParent()->AddReverseDependency(dep); +} + +static void AssertCheckableRedundancyGroup(Checkable::Ptr checkable, int dependencyCount, int groupCount, int totalDependenciesCount) +{ + BOOST_CHECK_MESSAGE( + dependencyCount == checkable->GetDependencies().size(), + "Dependency count mismatch for '" << checkable->GetName() << "' - expected=" << dependencyCount << "; got=" + << checkable->GetDependencies().size() + ); + auto dependencyGroups(checkable->GetDependencyGroups()); + BOOST_CHECK_MESSAGE( + groupCount == dependencyGroups.size(), + "Dependency group count mismatch for '" << checkable->GetName() << "' - expected=" << groupCount + << "; got=" << dependencyGroups.size() + ); + + for (auto& dependencyGroup : dependencyGroups) { + BOOST_CHECK_MESSAGE( + totalDependenciesCount == dependencyGroup->GetDependenciesCount(), + "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' and Checkable '" << checkable->GetName() + << "' total dependencies count mismatch - expected=" << totalDependenciesCount << "; got=" + << dependencyGroup->GetDependenciesCount() + ); + } + + if (groupCount > 0) { + BOOST_REQUIRE_MESSAGE(!dependencyGroups.empty(), "Checkable '" << checkable->GetName() << "' should have at least one dependency group."); + } +} + +static std::vector ExtractGroups(const Checkable::Ptr& checkable) +{ + auto dependencyGroups(checkable->GetDependencyGroups()); + std::sort(dependencyGroups.begin(), dependencyGroups.end()); + return dependencyGroups; +} + BOOST_AUTO_TEST_CASE(multi_parent) { /* One child host, two parent hosts. Simulate multi-parent dependencies. */ @@ -20,53 +81,28 @@ BOOST_AUTO_TEST_CASE(multi_parent) * - Parent objects need a CheckResult object * - Dependencies need a StateFilter */ - Host::Ptr parentHost1 = new Host(); - parentHost1->SetActive(true); - parentHost1->SetMaxCheckAttempts(1); - parentHost1->Activate(); - parentHost1->SetAuthority(true); + Host::Ptr parentHost1 = CreateHost("parentHost1"); parentHost1->SetStateRaw(ServiceCritical); parentHost1->SetStateType(StateTypeHard); parentHost1->SetLastCheckResult(new CheckResult()); - Host::Ptr parentHost2 = new Host(); - parentHost2->SetActive(true); - parentHost2->SetMaxCheckAttempts(1); - parentHost2->Activate(); - parentHost2->SetAuthority(true); + Host::Ptr parentHost2 = CreateHost("parentHost2"); parentHost2->SetStateRaw(ServiceOK); parentHost2->SetStateType(StateTypeHard); parentHost2->SetLastCheckResult(new CheckResult()); - Host::Ptr childHost = new Host(); - childHost->SetActive(true); - childHost->SetMaxCheckAttempts(1); - childHost->Activate(); - childHost->SetAuthority(true); + Host::Ptr childHost = CreateHost("childHost"); childHost->SetStateRaw(ServiceOK); childHost->SetStateType(StateTypeHard); /* Build the dependency tree. */ - Dependency::Ptr dep1 = new Dependency(); - - dep1->SetParent(parentHost1); - dep1->SetChild(childHost); + Dependency::Ptr dep1 (CreateDependency(parentHost1, childHost, "dep1")); dep1->SetStateFilter(StateFilterUp); + RegisterDependency(dep1, ""); - // Reverse dependencies - childHost->AddDependency(dep1); - parentHost1->AddReverseDependency(dep1); - - Dependency::Ptr dep2 = new Dependency(); - - dep2->SetParent(parentHost2); - dep2->SetChild(childHost); + Dependency::Ptr dep2 (CreateDependency(parentHost2, childHost, "dep2")); dep2->SetStateFilter(StateFilterUp); - - // Reverse dependencies - childHost->AddDependency(dep2); - parentHost2->AddReverseDependency(dep2); - + RegisterDependency(dep2, ""); /* Test the reachability from this point. * parentHost1 is DOWN, parentHost2 is UP. @@ -77,18 +113,42 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); + Dependency::Ptr duplicateDep (CreateDependency(parentHost1, childHost, "dep4")); + duplicateDep->SetIgnoreSoftStates(false, true); + RegisterDependency(duplicateDep, ""); + parentHost1->SetStateType(StateTypeSoft); + + // It should still be unreachable, due to the duplicated dependency object above with ignore_soft_states set to false. + BOOST_CHECK(childHost->IsReachable() == false); + parentHost1->SetStateType(StateTypeHard); + childHost->RemoveDependency(duplicateDep); + /* The only DNS server is DOWN. * Expected result: childHost is unreachable. */ - dep1->SetRedundancyGroup("DNS"); + childHost->RemoveDependency(dep1); // Remove the dep and re-add it with a configured redundancy group. + RegisterDependency(dep1, "DNS"); BOOST_CHECK(childHost->IsReachable() == false); /* 1/2 DNS servers is DOWN. * Expected result: childHost is reachable. */ - dep2->SetRedundancyGroup("DNS"); + childHost->RemoveDependency(dep2); + RegisterDependency(dep2, "DNS"); BOOST_CHECK(childHost->IsReachable() == true); + auto grandParentHost(CreateHost("GrandParentHost")); + grandParentHost->SetLastCheckResult(new CheckResult()); + grandParentHost->SetStateRaw(ServiceCritical); + grandParentHost->SetStateType(StateTypeHard); + + Dependency::Ptr dep3 (CreateDependency(grandParentHost, parentHost1, "dep3")); + dep3->SetStateFilter(StateFilterUp); + RegisterDependency(dep3, ""); + // The grandparent is DOWN but the DNS redundancy group has to be still reachable. + BOOST_CHECK_EQUAL(true, childHost->IsReachable()); + childHost->RemoveDependency(dep3); + /* Both DNS servers are DOWN. * Expected result: childHost is unreachable. */ @@ -98,4 +158,278 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); } +BOOST_AUTO_TEST_CASE(push_dependency_groups_to_registry) +{ + Checkable::Ptr childHostC(CreateHost("C", false)); + Checkable::Ptr childHostD(CreateHost("D", false)); + std::set dependencies; // Keep track of all dependencies to avoid unexpected deletions. + for (auto& parent : {String("A"), String("B"), String("E")}) { + Dependency::Ptr depC(CreateDependency(CreateHost(parent), childHostC, "depC" + parent)); + Dependency::Ptr depD(CreateDependency(depC->GetParent(), childHostD, "depD" + parent)); + if (parent == "A") { + Dependency::Ptr depCA2(CreateDependency(depC->GetParent(), childHostC, "depCA2")); + childHostC->AddDependency(depCA2); + dependencies.emplace(depCA2); + } else { + depC->SetRedundancyGroup("redundant", true); + depD->SetRedundancyGroup("redundant", true); + + if (parent == "B") { // Create an exact duplicate of depC, but with a different name. + Dependency::Ptr depCB2(CreateDependency(depC->GetParent(), childHostC, "depCB2")); + depCB2->SetRedundancyGroup("redundant", true); + childHostC->AddDependency(depCB2); + dependencies.emplace(depCB2); + } + } + childHostC->AddDependency(depC); + childHostD->AddDependency(depD); + dependencies.insert({depC, depD}); + } + + childHostC->PushDependencyGroupsToRegistry(); + childHostD->PushDependencyGroupsToRegistry(); + + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + for (auto& checkable : {childHostC, childHostD}) { + BOOST_CHECK_EQUAL(2, checkable->GetDependencyGroups().size()); + for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + if (dependencyGroup->IsRedundancyGroup()) { + BOOST_CHECK_EQUAL(5, dependencyGroup->GetDependenciesCount()); + BOOST_CHECK_EQUAL(checkable == childHostC ? 5 : 3, checkable->GetDependencies().size()); + } else { + BOOST_CHECK_EQUAL(3, dependencyGroup->GetDependenciesCount()); + BOOST_CHECK_EQUAL(checkable == childHostC ? 5 : 3, checkable->GetDependencies().size()); + } + } + } +} + +BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("C")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, ""); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, ""); + AssertCheckableRedundancyGroup(childHostC, 2, 2, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("D")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, ""); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, ""); + AssertCheckableRedundancyGroup(childHostD, 2, 2, 2); + AssertCheckableRedundancyGroup(childHostC, 2, 2, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + // This is an exact duplicate of depCA, but with a different dependency name. + Dependency::Ptr depCA2(CreateDependency(depCA->GetParent(), childHostC, "depCA2")); + // This is a duplicate of depCA, but with a different state filter. + Dependency::Ptr depCA3(CreateDependency(depCA->GetParent(), childHostC, "depCA3")); + depCA3->SetStateFilter(StateFilterUp, true); + // This is a duplicate of depCA, but with a different ignore_soft_states flag. + Dependency::Ptr depCA4(CreateDependency(depCA->GetParent(), childHostC, "depCA4")); + depCA4->SetIgnoreSoftStates(false, true); + + for (auto& dependency : {depCA2, depCA3, depCA4}) { + bool isAnExactDuplicate = dependency == depCA2; + RegisterDependency(dependency, ""); + + if (isAnExactDuplicate) { + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + } + + for (auto& dependencyGroup : childHostD->GetDependencyGroups()) { + if (dependency->GetParent() == dependencyGroup->GetDependenciesForChild(childHostD.get()).front()->GetParent()) { + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 3 : 1, dependencyGroup->GetDependenciesCount()); + } else { + BOOST_CHECK_EQUAL(2, dependencyGroup->GetDependenciesCount()); + } + BOOST_CHECK_EQUAL(2, childHostD->GetDependencies().size()); + } + + for (auto& dependencyGroup : childHostC->GetDependencyGroups()) { + if (dependency->GetParent() == dependencyGroup->GetDependenciesForChild(childHostC.get()).front()->GetParent()) { + // If depCA2 is currently being processed, then the group should have 3 dependencies, that's because + // depCA2 is an exact duplicate of depCA, and depCA shares the same group with depDA. + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 3 : 2, dependencyGroup->GetDependenciesCount()); + } else { + BOOST_CHECK_EQUAL(2, dependencyGroup->GetDependenciesCount()); + } + // The 3 dependencies are depCA, depCB, and the current one from the loop. + BOOST_CHECK_EQUAL(3, childHostC->GetDependencies().size()); + } + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 2 : 3, DependencyGroup::GetRegistrySize()); + childHostC->RemoveDependency(dependency); + } + + childHostC->RemoveDependency(depCA); + childHostD->RemoveDependency(depDA); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDB); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(simple_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA (CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + // Still 1 redundancy group, but there should be 4 dependencies now, i.e. 2 for each child Checkable. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + childHostC->RemoveDependency(depCA); + // After unregistering depCA, childHostC should have a new redundancy group with only depCB as dependency, and... + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + // ...childHostD should still have the same redundancy group as before but also with only two dependencies. + AssertCheckableRedundancyGroup(childHostD, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + childHostD->RemoveDependency(depDA); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group dependency, i.e. + // both child hosts should have the same redundancy group with only depCB and depDB as dependency. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + RegisterDependency(depDA, depDA->GetRedundancyGroup()); + childHostD->RemoveDependency(depDB); + // Nothing should have changed for childHostC, but both should now have a separate group with only depCB and depDA as dependency. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostE(CreateHost("childE")); + Dependency::Ptr depEA(CreateDependency(depCA->GetParent(), childHostE, "depEA")); + RegisterDependency(depEA, "redundant"); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEB(CreateDependency(depCB->GetParent(), childHostE, "depEB")); + RegisterDependency(depEB, "redundant"); + // All 3 hosts share the same group, and each host has 2 dependencies, thus 6 dependencies in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + auto childHostCGroups(ExtractGroups(childHostC)); + BOOST_TEST((childHostCGroups == ExtractGroups(childHostD) && childHostCGroups == ExtractGroups(childHostE))); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEZ(CreateDependency(CreateHost("Z"), childHostE, "depEZ")); + RegisterDependency(depEZ, "redundant"); + // Child host E should have a new redundancy group with 3 dependencies and the other two should still share the same group. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + AssertCheckableRedundancyGroup(childHostE, 3, 1, 3); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + childHostE->RemoveDependency(depEA); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + RegisterDependency(depEA, depEA->GetRedundancyGroup()); // Re-register depEA and instead... + childHostE->RemoveDependency(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. + // All 3 hosts share the same group again, and each host has 2 dependencies, thus 6 dependencies in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + childHostCGroups = ExtractGroups(childHostC); + BOOST_TEST((childHostCGroups == ExtractGroups(childHostD) && childHostCGroups == ExtractGroups(childHostE))); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + childHostC->RemoveDependency(depCA); + childHostD->RemoveDependency(depDB); + childHostE->RemoveDependency(depEB); + // Child host C has now a separate group with only depCB as dependency, and child hosts D and E share the same group. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostD) == ExtractGroups(childHostE), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDA); + childHostE->RemoveDependency(depEA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostE, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + BOOST_AUTO_TEST_SUITE_END()