Equivalence based queries optimized - vacuity, redundancy and strongEquivalence queries (#536)

* Optimized implementation of EquivalenceQuery.

Signed-off-by: Tanya <[email protected]>

* Added VacuityQuery and RedundancyQuery optimized implementation.
Keeping optimized properties separated per rule (instead of the union of all policy rules)
Fixed handling HostEPs in optimized implementation.

Signed-off-by: Tanya <[email protected]>

* Added VacuityQuery and RedundancyQuery optimized implementation.
Keeping optimized properties separated per rule (instead of the union of all policy rules)
Fixed handling HostEPs in optimized implementation.

Signed-off-by: Tanya <[email protected]>

* Ignoring 'complex function' lint error.
Returning 'passed' code for skipped queries.

Signed-off-by: Tanya <[email protected]>

* Added VacuityQuery and RedundancyQuery optimized implementation.
Keeping optimized properties separated per rule (instead of the union of all policy rules)
Fixed handling HostEPs in optimized implementation.

Signed-off-by: Tanya <[email protected]>

* Removed redundant method.

Signed-off-by: Tanya <[email protected]>

* Added VacuityQuery and RedundancyQuery optimized implementation.
Keeping optimized properties separated per rule (instead of the union of all policy rules)
Fixed handling HostEPs in optimized implementation.

Signed-off-by: Tanya <[email protected]>

* Fixed domain updating mechanism per rule (to avoid activating multiple times for the same rule, for example when a rule appears twice in a config).

Signed-off-by: Tanya <[email protected]>

* Fixed lint errors

Signed-off-by: Tanya <[email protected]>

* Enabled strongEquivalence optimized implementation.

Signed-off-by: Tanya <[email protected]>

* Fixed small inaccuracy in handling host endpoints in optimized solution.
Adding docs

Signed-off-by: Tanya <[email protected]>

---------

Signed-off-by: Tanya <[email protected]>

Author: tanyaveksler

nca/CoreDS/ConnectivityProperties.py

@@ -445,6 +445,17 @@ def get_all_conns_props_per_config_peers(peer_container):
         return ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips_and_dns,
                                                                  "dst_peers": all_peers_and_ips_and_dns})
 
+    @staticmethod
+    def get_all_conns_props_per_domain_peers():
+        """
+        Return all possible between-peers connections.
+        This is a compact way to represent all peers connections, but it is an over-approximation also containing
+        IpBlock->IpBlock connections. Those redundant connections will be eventually filtered out.
+        """
+        src_peers = BasePeerSet().get_peer_set_by_indices(DimensionsManager().get_dimension_domain_by_name("src_peers"))
+        dst_peers = BasePeerSet().get_peer_set_by_indices(DimensionsManager().get_dimension_domain_by_name("dst_peers"))
+        return ConnectivityProperties.make_conn_props_from_dict({"src_peers": src_peers, "dst_peers": dst_peers})
+
     @staticmethod
     def make_empty_props():
         """

nca/NetworkConfig/NetworkConfig.py

@@ -286,8 +286,17 @@ def allowed_connections_optimized(self, layer_name=None):
         # all possible connections involving hostEndpoints
         conn_hep = ConnectivityProperties.make_conn_props_from_dict({"src_peers": host_eps}) | \
             ConnectivityProperties.make_conn_props_from_dict({"dst_peers": host_eps})
-        conns_res = OptimizedPolicyConnections()
-        conns_res.all_allowed_conns = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container)
+        if host_eps and NetworkLayerName.K8s_Calico not in self.policies_container.layers:
+            # maintain K8s_Calico layer as active if peer container has hostEndpoint
+            conns_res = \
+                self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container,
+                                                                                         NetworkLayerName.K8s_Calico)
+            conns_res.allowed_conns &= conn_hep
+            conns_res.denied_conns &= conn_hep
+            conns_res.pass_conns &= conn_hep
+        else:
+            conns_res = OptimizedPolicyConnections()
+            conns_res.all_allowed_conns = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container)
         for layer, layer_obj in self.policies_container.layers.items():
             conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container)
             # only K8s_Calico layer handles host_eps

nca/Parsers/CalicoPolicyYamlParser.py

@@ -558,7 +558,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile):
         if not dst_res_pods and policy_selected_eps and (not is_ingress or not is_profile):
             self.warning('Rule selects no destination endpoints', rule)
 
-        return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action), conn_props
+        return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action, conn_props)
 
     def _verify_named_ports(self, rule, rule_eps, rule_conns):
         """
@@ -698,16 +698,12 @@ def parse_policy(self):
             self.syntax_error('order is not allowed in the spec of a Profile', policy_spec)
 
         for ingress_rule in policy_spec.get('ingress', []):
-            rule, optimized_props = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile)
+            rule = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile)
             res_policy.add_ingress_rule(rule)
-            if self.optimized_run != 'false':
-                res_policy.update_and_add_optimized_props(optimized_props, rule.action, True)
 
         for egress_rule in policy_spec.get('egress', []):
-            rule, optimized_props = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile)
+            rule = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile)
             res_policy.add_egress_rule(rule)
-            if self.optimized_run != 'false':
-                res_policy.update_and_add_optimized_props(optimized_props, rule.action, False)
 
         self._apply_extra_labels(policy_spec, is_profile, res_policy.name)
         res_policy.findings = self.warning_msgs

nca/Parsers/GenericIngressLikeYamlParser.py

@@ -8,6 +8,7 @@
 from nca.CoreDS.DimensionsManager import DimensionsManager
 from nca.CoreDS.Peer import PeerSet
 from nca.CoreDS.PortSet import PortSet
+from nca.CoreDS.ProtocolSet import ProtocolSet
 from nca.CoreDS.ConnectivityProperties import ConnectivityProperties
 from nca.CoreDS.ConnectionSet import ConnectionSet
 from nca.Resources.IngressPolicy import IngressPolicyRule
@@ -56,40 +57,31 @@ def parse_regex_host_value(self, regex_value, rule):
             regex_value = regex_value.replace("*", allowed_chars + '*')
         return MinDFA.dfa_from_regex(regex_value)
 
-    def _make_allow_rules(self, allowed_conns):
-        """
-        Make deny rules from the given connections
-        :param ConnectivityProperties allowed_conns: the given allowed connections
-        :return: the list of deny IngressPolicyRules
-        """
-        return self._make_rules_from_conns(allowed_conns)
-
     @staticmethod
-    def _make_rules_from_conns(conn_props):
+    def _make_allow_rules(conn_props, src_peers):
         """
         Make IngressPolicyRules from the given connections
         :param ConnectivityProperties conn_props: the given connections
+        :param PeerSet src_peers: the source peers to add to optimized props
         :return: the list of IngressPolicyRules
         """
         assert not conn_props.named_ports
         assert not conn_props.excluded_named_ports
-        peers_to_conns = {}
         res = []
-        # extract peers dimension from cubes
         assert not conn_props.is_active_dimension("src_peers")
+        # extract dst_peers dimension from cubes
+        tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP')
         for cube in conn_props:
             conn_cube = conn_props.get_connectivity_cube(cube)
-            dst_peer_set = conn_cube["dst_peers"]
-            conn_cube.unset_dim("dst_peers")
-            new_props = ConnectivityProperties.make_conn_props(conn_cube)
+            new_conn_cube = conn_cube.copy()
+            conn_cube.update({"src_peers": src_peers, "protocols": tcp_protocol})
+            rule_opt_props = ConnectivityProperties.make_conn_props(conn_cube)
+            dst_peer_set = new_conn_cube["dst_peers"]
+            new_conn_cube.unset_dim("dst_peers")
+            new_props = ConnectivityProperties.make_conn_props(new_conn_cube)
             new_conns = ConnectionSet()
             new_conns.add_connections('TCP', new_props)
-            if dst_peer_set in peers_to_conns:
-                peers_to_conns[dst_peer_set] |= new_conns  # optimize conns for the same peers
-            else:
-                peers_to_conns[dst_peer_set] = new_conns
-        for peer_set, conns in peers_to_conns.items():
-            res.append(IngressPolicyRule(peer_set, conns))
+            res.append(IngressPolicyRule(dst_peer_set, new_conns, rule_opt_props))
         return res
 
     @staticmethod

nca/Parsers/IngressPolicyYamlParser.py

@@ -10,7 +10,6 @@
 from nca.CoreDS.PortSet import PortSet
 from nca.CoreDS.ConnectivityCube import ConnectivityCube
 from nca.CoreDS.ConnectivityProperties import ConnectivityProperties
-from nca.CoreDS.ProtocolSet import ProtocolSet
 from nca.Resources.IngressPolicy import IngressPolicy
 from nca.Resources.NetworkPolicy import NetworkPolicy
 from .GenericIngressLikeYamlParser import GenericIngressLikeYamlParser
@@ -287,10 +286,6 @@ def parse_policy(self):
         # allowed_conns = none means that services referenced by this Ingress policy are not found,
         # then no connections rules to add (Ingress policy has no effect)
         if allowed_conns:
-            res_policy.add_rules(self._make_allow_rules(allowed_conns))
-            protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP')
-            allowed_conns &= ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols,
-                                                                               "src_peers": res_policy.selected_peers})
-            res_policy.add_optimized_allow_props(allowed_conns, False)
+            res_policy.add_rules(self._make_allow_rules(allowed_conns, res_policy.selected_peers))
         res_policy.findings = self.warning_msgs
         return res_policy

nca/Parsers/IstioPolicyYamlParser.py

@@ -523,7 +523,7 @@ def parse_ingress_rule(self, rule, selected_peers):
                                                                                  "dst_peers": selected_peers})
         connections &= condition_conns
         conn_props &= condition_props
-        return IstioPolicyRule(res_peers, connections), conn_props
+        return IstioPolicyRule(res_peers, connections, conn_props)
 
     @staticmethod
     def parse_policy_action(action):
@@ -571,14 +571,8 @@ def parse_policy(self):
         pod_selector = policy_spec.get('selector')
         res_policy.selected_peers = self.update_policy_peers(pod_selector, 'matchLabels')
         for ingress_rule in policy_spec.get('rules', []):
-            rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers)
+            rule = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers)
             res_policy.add_ingress_rule(rule)
-            if res_policy.action == IstioNetworkPolicy.ActionType.Allow:
-                res_policy.add_optimized_allow_props(optimized_props, True)
-            else:  # Deny
-                res_policy.add_optimized_deny_props(optimized_props, True)
-        all_props = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container)
-        res_policy.add_optimized_allow_props(all_props, False)
         if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny:
             self.syntax_error("DENY action without rules is meaningless as it will never be triggered")
 

nca/Parsers/IstioSidecarYamlParser.py

@@ -4,7 +4,6 @@
 #
 import re
 from nca.CoreDS.Peer import PeerSet
-from nca.CoreDS.ConnectivityProperties import ConnectivityProperties
 from nca.Resources.NetworkPolicy import NetworkPolicy
 from nca.Resources.IstioSidecar import IstioSidecar, IstioSidecarRule
 from nca.Resources.IstioTrafficResources import istio_root_namespace
@@ -215,8 +214,6 @@ def parse_policy(self):
         self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing)
         res_policy = IstioSidecar(policy_name, self.namespace)
         res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar
-        all_props = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container)
-        res_policy.add_optimized_allow_props(all_props, True)
 
         sidecar_spec = self.policy['spec']
         # currently, supported fields in spec are workloadSelector and egress

nca/Parsers/IstioTrafficResourcesYamlParser.py

@@ -7,7 +7,6 @@
 from nca.CoreDS.MinDFA import MinDFA
 from nca.CoreDS.Peer import PeerSet
 from nca.CoreDS.MethodSet import MethodSet
-from nca.CoreDS.ProtocolSet import ProtocolSet
 from nca.CoreDS.ConnectivityCube import ConnectivityCube
 from nca.CoreDS.ConnectivityProperties import ConnectivityProperties
 from nca.Resources.IstioTrafficResources import Gateway, VirtualService
@@ -396,12 +395,7 @@ def create_istio_traffic_policies(self):
                 res_policy.selected_peers = peer_set
                 allowed_conns = self.make_allowed_connections(vs, host_dfa)
                 if allowed_conns:
-                    res_policy.add_rules(self._make_allow_rules(allowed_conns))
-                    protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP')
-                    allowed_conns &= \
-                        ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols,
-                                                                          "src_peers": res_policy.selected_peers})
-                    res_policy.add_optimized_allow_props(allowed_conns, False)
+                    res_policy.add_rules(self._make_allow_rules(allowed_conns, res_policy.selected_peers))
                     res_policy.findings = self.warning_msgs
                     vs_policies.append(res_policy)
             if not vs_policies:

nca/Parsers/K8sPolicyYamlParser.py

@@ -311,9 +311,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods):
         :param dict rule: The rule to parse
         :param str peer_array_key: The key which defined the peer set ('from' for ingress, 'to' for egress)
         :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to
-        :return: A tuple (K8sPolicyRule, ConnectivityProperties) with the proper PeerSet and attributes, where
-        ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet
-        :rtype: tuple(K8sPolicyRule, ConnectivityProperties)
+        :return: K8sPolicyRule with the proper PeerSet and attributes
+        :rtype: K8sPolicyRule
         """
         self.check_fields_validity(rule, 'ingress/egress rule', {peer_array_key: [0, list], 'ports': [0, list]})
         peer_array = rule.get(peer_array_key, [])
@@ -355,7 +354,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods):
         if not res_pods:
             self.warning('Rule selects no pods', rule)
 
-        return K8sPolicyRule(res_pods, res_conns), res_opt_props
+        return K8sPolicyRule(res_pods, res_conns, res_opt_props)
 
     def verify_named_ports(self, rule, rule_pods, rule_conns):
         """
@@ -393,9 +392,9 @@ def parse_ingress_rule(self, rule, policy_selected_pods):
         ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet
         :rtype: tuple(K8sPolicyRule, ConnectivityProperties)
         """
-        res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'from', policy_selected_pods)
+        res_rule = self.parse_ingress_egress_rule(rule, 'from', policy_selected_pods)
         self.verify_named_ports(rule, policy_selected_pods, res_rule.port_set)
-        return res_rule, res_opt_props
+        return res_rule
 
     def parse_egress_rule(self, rule, policy_selected_pods):
         """
@@ -407,9 +406,9 @@ def parse_egress_rule(self, rule, policy_selected_pods):
         ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet
         :rtype: tuple(K8sPolicyRule, ConnectivityProperties)
         """
-        res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'to', policy_selected_pods)
+        res_rule = self.parse_ingress_egress_rule(rule, 'to', policy_selected_pods)
         self.verify_named_ports(rule, res_rule.peer_set, res_rule.port_set)
-        return res_rule, res_opt_props
+        return res_rule
 
     def parse_policy(self):
         """
@@ -458,16 +457,14 @@ def parse_policy(self):
         ingress_rules = policy_spec.get('ingress', [])
         if ingress_rules:
             for ingress_rule in ingress_rules:
-                rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers)
+                rule = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers)
                 res_policy.add_ingress_rule(rule)
-                res_policy.add_optimized_allow_props(optimized_props, True)
 
         egress_rules = policy_spec.get('egress', [])
         if egress_rules:
             for egress_rule in egress_rules:
-                rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers)
+                rule = self.parse_egress_rule(egress_rule, res_policy.selected_peers)
                 res_policy.add_egress_rule(rule)
-                res_policy.add_optimized_allow_props(optimized_props, False)
 
         res_policy.findings = self.warning_msgs
         res_policy.referenced_labels = self.referenced_labels