Merge with master.

Signed-off-by: Tanya <tatyana@il.ibm.com>

nca/NetworkConfig/NetworkLayer.py

@@ -89,14 +89,14 @@ def does_contain_only_gateway_layers(self):
         Checks if the map contains only gateway layers.
         :return: True if the map contains only gateway layers, False otherwise
         """
-        return set(self.keys()).issubset({NetworkLayerName.K8sGateway, NetworkLayerName.IstioGateway})
+        return bool(self) and set(self.keys()).issubset({NetworkLayerName.K8sGateway, NetworkLayerName.IstioGateway})
 
     def does_contain_istio_layers(self):
         """
         Checks if any of Istio layers is in the map.
         :return: True if any of Istio layers is in the map, False otherwise
         """
-        return {NetworkLayerName.Istio, NetworkLayerName.IstioGateway} & set(self.keys())
+        return bool({NetworkLayerName.Istio, NetworkLayerName.IstioGateway} & set(self.keys()))
 
     @staticmethod
     def empty_layer_allowed_connections(layer_name, from_peer, to_peer):

nca/NetworkConfig/PoliciesFinder.py

@@ -127,8 +127,8 @@ def parse_policies_in_parse_queue(self):  # noqa: C901
                                        )
         if istio_gtw_parser or istio_vs_parser:
             istio_gtw_policy_gen = IstioGatewayPolicyGenerator(istio_gtw_parser, istio_vs_parser)
-            istio_traffic_policies = istio_gtw_policy_gen.create_istio_gateway_policies()
-            for istio_traffic_policy in istio_traffic_policies:
+            istio_gateway_policies = istio_gtw_policy_gen.create_istio_gateway_policies()
+            for istio_traffic_policy in istio_gateway_policies:
                 self._add_policy(istio_traffic_policy)
                 if ExplTracker().is_active():
                     ExplTracker().derive_item(istio_traffic_policy.name)

nca/Parsers/GenericGatewayYamlParser.py

@@ -17,7 +17,8 @@
 
 class GenericGatewayYamlParser(GenericYamlParser):
     """
-    A parser for Ingress like objects (common for k8s ingress and Istio ingress)
+    A general parser, common for K8s Ingress resources, as well as for Istio Gateway/VirtualService resources.
+    Specific K8s Ingress / Istio Gateway / Istio VirtualService parsers are inherited from this class.
     """
 
     def __init__(self, peer_container, ingress_file_name=''):

nca/Parsers/IstioGatewayYamlParser.py

@@ -10,7 +10,7 @@
 
 class IstioGatewayYamlParser(GenericGatewayYamlParser):
     """
-    A parser for Istio gateway resource.
+    A parser for Istio Gateway resource. (see https://istio.io/latest/docs/reference/config/networking/gateway/)
     Currently, we support only standard istio ingress or egress gateways, which are identified by
     'istio: ingressgateway' or 'istio: egressgateway' selectors correspondingly.
     """
@@ -21,9 +21,10 @@ def __init__(self, peer_container):
         against this set of peers
         """
         GenericGatewayYamlParser.__init__(self, peer_container)
-        self.gateways = {}  # a map from a name to a Gateway
+        self.gateways = {}  # a map from a gateway's full name (namespace/name) to a Gateway object
         # missing_istio_gw_pods_with_labels is a set of labels - (key,value) pairs
         # of gateway resource that has no matching pods
+        # this field is used by "livesim" mechanism - to add the underlying pods of the configured gateway
         self.missing_istio_gw_pods_with_labels = set()
 
     def add_gateway(self, gateway):

nca/Parsers/IstioVirtualServiceYamlParser.py

@@ -56,13 +56,15 @@ def parse_virtual_service(self, vs_resource, vs_file_name):
         # (see https://github.com/istio/api/blob/bb3cb9c034df2b5cc1de1d77689d201a0cf961c5/networking/v1alpha3/
         #      virtual_service.proto#L209-L238)
         # Hosts field is used for matching virtual services to gateways (whenever 'gateways' field is specified
-        # in the virtual service)
+        # in the virtual service). Also, the matched hosts appear in the 'hosts' attribute of the connections.
         hosts = vs_spec.get('hosts')
         for host in hosts or []:
             host_dfa = self.parse_host_value(host, vs_resource)
             if host_dfa:
                 vs.add_host_dfa(host_dfa)
 
+        # gateways field: A single VirtualService is used to configure connectivity of sidecars inside the mesh as well
+        # as for one or more gateways (with the matching hosts)
         self.parse_vs_gateways(vs.namespace, vs_spec, vs, True)
         self.parse_vs_http_route(vs, vs_spec)
         self.parse_vs_tls_route(vs, vs_spec)
@@ -76,7 +78,7 @@ def parse_vs_gateways(self, namespace, resource_spec, result, are_global_gtw=Fal
         internal gateway list
         :param K8sNamespace namespace: the virtual service namespace
         :param dict resource_spec: the resource containing gateways to parse
-        :param result: the object to put the resulting gateways to
+        :param Union[VirtualService, VirtualService.Route] result: the object to put the resulting gateways to
         :param bool are_global_gtw: whether the parsed list is a global virtual service gateway list
         """
         gateways = resource_spec.get('gateways')
@@ -210,6 +212,8 @@ def parse_http_match_request(self, route, result_route, vs):
                 result_route.add_methods(methods)
             else:
                 result_route.add_methods(MethodSet(True))
+            # gateways field: Names of gateways where the rule should be applied. Gateway names in the top-level
+            # gateways field of the VirtualService (if any) are overridden.
             self.parse_vs_gateways(vs.namespace, item, result_route)
 
     def parse_tls_match_attributes(self, route, vs):
@@ -241,6 +245,8 @@ def parse_tls_match_attributes(self, route, vs):
                 self.warning('sniHosts mentioned in the tls.match are not a subset of hosts. This match will be ignored',
                              vs)
                 return None
+            # gateways field: Names of gateways where the rule should be applied. Gateway names in the top-level
+            # gateways field of the VirtualService (if any) are overridden.
             self.parse_vs_gateways(vs.namespace, item, tls_route)
         return tls_route
 
@@ -260,6 +266,8 @@ def parse_l4_match_attributes(self, route, result_route, vs):
                                        {'destinationSubnets': [3, list], 'port': [3, int],
                                         'sourceLabels': [3, dict], 'gateways': [0, list], 'sourceNamespace': [3, str]})
             # TODO - understand 'destinationSubnets' usage
+            # gateways field: Names of gateways where the rule should be applied. Gateway names in the top-level
+            # gateways field of the VirtualService (if any) are overridden.
             self.parse_vs_gateways(vs.namespace, item, result_route)
 
     def parse_route_destinations(self, route, result_route, vs, is_http_route):

nca/Resources/OtherResources/Gateway.py

@@ -11,7 +11,7 @@
 
 class Gateway:
     """
-    A class for keeping some elements of parsed Istio Gateway, needed for building GatewayPolicy
+    A class for keeping some elements of parsed Istio Gateway, needed for building GatewayPolicy.
     """
 
     class GatewayType(Enum):
@@ -23,13 +23,17 @@ class GatewayType(Enum):
 
     @dataclass
     class Server:
+        """
+        A class that holds server attributes of a Gateway, as described in
+        https://istio.io/latest/docs/reference/config/networking/gateway/#Server
+        """
         @dataclass
         class GatewayPort:
             number: int
             protocol: str
             name: str
 
-        port: GatewayPort  # the port field is not currently  used.
+        port: GatewayPort  # the port field is not currently used.
         hosts_dfa: MinDFA or None = None
         name: str = ''
 
@@ -50,11 +54,16 @@ def __init__(self, name, namespace):
         :param str name: the gateway name
         :param K8sNamespace namespace: the gateway namespace
         """
-        self.name = name
-        self.namespace = namespace
-        self.type = None
+        self.name = name  # the name of the gateway, as appears in the metadata
+        self.namespace = namespace  # the namespace of the gateway, as appears in the metadata
+        self.type = None  # whether this is an ingress gateway or an egress gateway
+        # the 'peers' field defines the set of pods on which this gateway is applied.
+        # It is calculated from Gateway.selector attribute, as described in
+        # https://istio.io/latest/docs/reference/config/networking/gateway/#Gateway
         self.peers = PeerSet()
-        self.servers = []
+        self.servers = []  # a list of servers, where a server is described in Gateway.Server above.
+        # the 'all_hosts_dfa' field is calculated as the union of 'hosts_dfa' field of all servers of this gateway.
+        # It is used for matching the gateway to virtual services that reference it.
         self.all_hosts_dfa = None
 
     def full_name(self):

nca/Resources/OtherResources/VirtualService.py

@@ -16,7 +16,13 @@ class VirtualService:
     @dataclass
     class Destination:
         """
-        A class for keeping a parsed HTTP/TLS/TCP route destination of a VirtualService
+        A class for keeping a parsed HTTP/TLS/TCP route destination of a VirtualService.
+        It originates from Destination attribute of a virtual service, as described in
+        https://istio.io/latest/docs/reference/config/networking/virtual-service/#Destination;
+        'name' field is taken directly as 'Destination.host' attribute,
+        whereas 'pods' field is calculated from 'Destination.host' attribute, by mapping it to a local service
+        or to a remote dns entry;
+        finally, 'ports' field is taken directly from 'Destination.port' attribute.
         """
         name: str
         pods: PeerSet
@@ -27,16 +33,33 @@ def is_egress_gateway(self):
 
     class Route:
         """
-        A class for various route kinds (HTTPRoute/TLSRoute/TCPRoute) of a VirtualService
+        A class for holding (some of) the attributes of various route kinds (HTTPRoute/TLSRoute/TCPRoute)
+        of a VirtualService. Some fields of this class are unique to a certain types of routes; they have values
+        in relevant types of routes, while being None in other types of routes.
+        During parsing, all Route lists are built for every virtual service, keeping those attributes that are needed
+        for building GatewayPolicies. On the second phase, GatewayPolicies are built from every Route.
         """
 
         def __init__(self):
+            # 'is_internal_dest' field represents whether the destination if to internal (True)
+            # or external (False) service. It is True for Ingress flow routes, as well as for mesh-to-egress-gateway
+            # routes of Egress flow. It is False for egress-gateway-to-dns-service routes of Egress flow.
             self.is_internal_dest = None
+            # 'destinations' field is a list of possible Destinations (as described in VirtualService.Destination above.
             self.destinations = []
-            self.uri_dfa = None  # only in HTTP routes
-            self.methods = None  # only in HTTP routes
-            self.all_sni_hosts_dfa = None  # only in TLS routes
-            self.gateway_names = []
+            # 'uri_dfa' and 'methods' fields originate from 'uri' and 'method' attribute respectively
+            # of HTTPMatchRequest of a virtual service, as described in
+            # https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPMatchRequest
+            # They are rlevant only for HTTP types of routes.
+            self.uri_dfa = None
+            self.methods = None
+            # 'all_sni_hosts_dfa' field is relevant only in TLS routes. It originates from 'sniHosts' attribute of
+            # TLSMatchAttributes, as described in
+            # https://istio.io/latest/docs/reference/config/networking/virtual-service/#TLSMatchAttributes
+            # In case of TLS routes, it refines the 'hosts_dfa' field of a virtual service, and is assigned to
+            # 'hosts' attribute of the resulting gateway policy connections.
+            self.all_sni_hosts_dfa = None
+            self.gateway_names = []  # list of gateways full names in format "namespace/name"
 
         def add_destination(self, name, pods, port, is_internal_dest):
             """
@@ -82,13 +105,17 @@ def __init__(self, name, namespace):
         :param str name: the name of the VirtualService
         :param K8sNamespace namespace: the namespace of the VirtualService
         """
-        self.name = name
-        self.namespace = namespace
+        self.name = name  # the name of the virtual service, as appears in the metadata
+        self.namespace = namespace  # the namespace of the virtual service, as appears in the metadata
+        # the 'hosts_dfa' field originates in VirtualService.hosts attribute, as described in
+        # https://istio.io/latest/docs/reference/config/networking/virtual-service/#VirtualService
+        # It is used for matching gateways to this virtual service, as well as for 'hosts' attribute
+        # of the resulting gateway policy connections.
         self.hosts_dfa = []
-        self.gateway_names = []
-        self.http_routes = []
-        self.tls_routes = []
-        self.tcp_routes = []
+        self.gateway_names = []  # list of gateways full names in format "namespace/name"
+        self.http_routes = []  # a list of HTTP routes of this virtual service. See Route description above.
+        self.tls_routes = []  # a list of TLS routes of this virtual service
+        self.tcp_routes = []  # a list of TCP routes of this virtual service
 
     def full_name(self):
         """
@@ -139,7 +166,7 @@ def add_gateway(gtw_namespace, gtw_name, result):
     def add_mesh(result):
         """
         Add mesh gateway to the list of gateway names of the VirtualService
-        :param result: the object to add the gateway to (assuming it has 'gateway_names' attribute).
+        :param VirtualService.Route result: the object to add the gateway to (assuming it has 'gateway_names' attribute)
         """
         if 'mesh' not in result.gateway_names:
-            result.gateway_names.append("mesh")
+            result.gateway_names.append('mesh')

nca/Resources/PolicyResources/GatewayPolicy.py

@@ -63,14 +63,15 @@ class ActionType(Enum):
         Deny = 0
         Allow = 1
 
-    def __init__(self, name, namespace, action):
+    def __init__(self, name, namespace, action, origin=""):
         """
         :param str name: gateway poilcy name
         :param K8sNamespace namespace: the namespace containing this policy
         :param ActionType action: whether Allow or Deny
         """
         super().__init__(name, namespace)
         self.action = action
+        self.origin = origin or name  # originating virtual service/route/destination/gateway for this policy
 
     def __eq__(self, other):
         return super().__eq__(other) and self.action == other.action
@@ -191,7 +192,7 @@ def clone_without_rule(self, rule_to_exclude, ingress_rule):
         :return: A copy of 'self' without the provided rule
         :rtype: GatewayPolicy
         """
-        res = GatewayPolicy(self.name, self.namespace, self.action)
+        res = GatewayPolicy(self.name, self.namespace, self.action, self.origin)
         res.selected_peers = self.selected_peers
         res.affects_egress = self.affects_egress
         res.affects_ingress = self.affects_ingress

tests/expected_runtime/istio_tests_expected_runtime.csv

@@ -24,6 +24,8 @@ istio_testcases/example_policies/complex-ingress-test/complex-k8s-ingress-test-s
 istio_testcases/example_policies/fly-istio-ingress-test/fly-istio-ingress-test-scheme.yaml,0.48
 istio_testcases/example_policies/istio-ingress-test/istio-ingress-test-scheme.yaml,1.91
 istio_testcases/example_policies/istio-egress-test/istio-egress-test-scheme.yaml,0.63
+istio_testcases/example_policies/istio-egress-test-partial-flow1/istio-egress-test-partial-flow1-scheme.yaml,0.12
+istio_testcases/example_policies/istio-egress-test-partial-flow2/istio-egress-test-partial-flow2-scheme.yaml,0.12
 istio_testcases/example_policies/multi-layer-tests/onlineboutique-multi-layer-tests-scheme.yaml,32.84
 istio_testcases/example_policies/online_boutique/connectivity-scheme.yaml,27.69
 istio_testcases/example_policies/sidecar_examples_w_onlineboutique/frontend_tests/frontend-sidecar-test-scheme.yaml,0.16

tests/istio_testcases/example_policies/istio-egress-test-partial-flow1/istio-egress-test-partial-flow1-scheme.yaml

@@ -0,0 +1,18 @@
+namespaceList: ./
+podList: ./
+
+networkConfigList:
+  - name: istio-egress
+    resourceList:
+      - ./resources
+    expectedWarnings: 2
+
+
+queries:
+  - name: connectivity
+    connectivityMap:
+      - istio-egress
+    outputConfiguration:
+      outputFormat: txt
+      fwRulesRunInTestMode: false
+    expectedOutput: ../../expected_output/istio_egress_test_partial_flow1_connectivity_map.txt

tests/istio_testcases/example_policies/istio-egress-test-partial-flow1/resources/gateway.yaml

@@ -0,0 +1,15 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: my-gateway
+  namespace: istio-system
+spec:
+  selector:
+      istio: egressgateway
+  servers:
+  - port:
+      number: 443
+      name: https
+      protocol: HTTPS
+    hosts:
+    - httpbin.example.com