Merge branch 'develop' into CactuseSecurity#2692--Modelling---short-duplication-of-objects-when-saving

Author: Robin-Smets

roles/database/files/sql/idempotent/fworch-texts.sql

@@ -283,6 +283,8 @@ INSERT INTO txt VALUES ('PagerPagesize',        'German', 	'Seitengrösse');
 INSERT INTO txt VALUES ('PagerPagesize',        'English',  'Page size');
 INSERT INTO txt VALUES ('PagerSubmit',          'German', 	'Speichern');
 INSERT INTO txt VALUES ('PagerSubmit',          'English',  'Save');
+INSERT INTO txt VALUES ('order_by',          'German',  'Sortieren nach');
+INSERT INTO txt VALUES ('order_by',          'English',  'Order by');
 
 -- (re)login
 INSERT INTO txt VALUES ('login', 				'German',	'Anmelden');

roles/importer/files/importer/fortiadom5ff/fmgr_gw_networking.py

@@ -62,12 +62,12 @@ def normalize_network_data(native_config, normalized_config, mgm_details):
         normalized_config['routing'].sort(key=getRouteDestination,reverse=True)
 
         for interface in native_config['interfaces_per_device/' + full_vdom_name]:
-            if interface['ipv6']['ip6-address']!='::/0':
+            if 'ipv6' in interface and 'ip6-address' in interface['ipv6'] and interface['ipv6']['ip6-address']!='::/0':
                 ipv6, netmask_bits = interface['ipv6']['ip6-address'].split('/')
                 normIfV6 = Interface(dev_id, interface['name'], IPAddress(ipv6), netmask_bits, ip_version=6)
                 normalized_config['interfaces'].append(normIfV6)
 
-            if interface['ip']!=['0.0.0.0','0.0.0.0']:
+            if 'ip' in interface and interface['ip']!=['0.0.0.0','0.0.0.0']:
                 ipv4 = IPAddress(interface['ip'][0])
                 netmask_bits = IPAddress(interface['ip'][1]).netmask_bits()
                 normIfV4 = Interface(dev_id, interface['name'], ipv4, netmask_bits, ip_version=4)

roles/importer/files/importer/fortiadom5ff/fmgr_network.py

@@ -103,6 +103,22 @@ def normalize_nwobjects(full_config, config2import, import_id, nw_obj_types, jwt
             obj.update({'control_id': import_id})
             nw_objects.append(obj)
 
+    # dynamic objects have different return structure
+    if 'response' in full_config['nw_obj_global_firewall/internet-service-basic'][0] and 'results' in full_config['nw_obj_global_firewall/internet-service-basic'][0]['response']:
+        for obj_orig in full_config['nw_obj_global_firewall/internet-service-basic'][0]['response']['results']:
+            if 'name' in obj_orig and 'q_origin_key' in obj_orig:
+                obj = {
+                    'obj_name': obj_orig['name'],
+                    'obj_typ': 'network',
+                    'obj_ip': '0.0.0.0/0',
+                    'obj_uid': 'q_origin_key_' + str(obj_orig['q_origin_key']),
+                    'control_id': import_id,
+                    'obj_zone': 'global'
+                    }
+                nw_objects.append(obj)
+    else:
+        logger.warning("internet service objects return format broken")
+
     # finally add "Original" network object for natting
     original_obj_name = 'Original'
     original_obj_uid = 'Original'
@@ -189,7 +205,11 @@ def get_first_ip_of_destination(obj_ref, config2import):
 
     for obj in config2import['network_objects']:
         if 'obj_uid' in obj and obj['obj_uid']==obj_ref:
-            return obj['obj_ip']
+            if 'obj_type' in obj and obj['obj_type']=='group':
+                if 'obj_member_refs' in obj and list_delimiter in obj['obj_member_refs']:
+                    return get_first_ip_of_destination(obj['obj_member_refs'].split(list_delimiter)[0], config2import)
+            elif 'obj_ip' in obj:
+                return obj['obj_ip']
     logger.warning('src nat behind interface: found no IP info for destination object ' + obj_ref)
     return None
 
@@ -210,13 +230,14 @@ def resolve_raw_objects (obj_name_string_list, delimiter, obj_dict, name_key, ui
         if rule_type is not None:
             if obj_type == 'network':
                 if 'v4' in rule_type and 'global' in rule_type:
-                    object_tables = [obj_dict['nw_obj_global_firewall/address'], obj_dict['nw_obj_global_firewall/addrgrp']]
+                    object_tables = [obj_dict['nw_obj_global_firewall/address'], obj_dict['nw_obj_global_firewall/addrgrp'], obj_dict['nw_obj_global_firewall/internet-service-basic'][0]['response']['results']]
                 elif 'v6' in rule_type and 'global' in rule_type:
                     object_tables = [obj_dict['nw_obj_global_firewall/address6'], obj_dict['nw_obj_global_firewall/addrgrp6']]
                 elif 'v4' in rule_type and 'adom' in rule_type:
                     object_tables = [obj_dict['nw_obj_adom_firewall/address'], obj_dict['nw_obj_adom_firewall/addrgrp'], \
                         obj_dict['nw_obj_global_firewall/address'], obj_dict['nw_obj_global_firewall/addrgrp'], \
-                        obj_dict['nw_obj_adom_firewall/vip'] ]
+                        obj_dict['nw_obj_adom_firewall/vip'], obj_dict['nw_obj_adom_system/external-resource'], \
+                        obj_dict['nw_obj_global_firewall/internet-service-basic'][0]['response']['results'] ]
                 elif 'v6' in rule_type and 'adom' in rule_type:
                     object_tables = [obj_dict['nw_obj_adom_firewall/address6'], obj_dict['nw_obj_adom_firewall/addrgrp6'], \
                         obj_dict['nw_obj_global_firewall/address6'], obj_dict['nw_obj_global_firewall/addrgrp6']]
@@ -235,7 +256,13 @@ def resolve_raw_objects (obj_name_string_list, delimiter, obj_dict, name_key, ui
                     else:
                         for obj in tab:
                             if obj[name_key] == el:
-                                ref_list.append(obj[uid_key])
+                                if uid_key in obj:
+                                    ref_list.append(obj[uid_key])
+                                # in case of internet-service-object we find no uid field, but custom q_origin_key_
+                                elif 'q_origin_key' in obj:
+                                    ref_list.append('q_origin_key_' + str(obj['q_origin_key']))
+                                else:
+                                    logger.error('found object without expected uid')
                                 break_flag = True
                                 found = True
                                 break

roles/importer/files/importer/fortiadom5ff/fmgr_rule.py

@@ -184,6 +184,7 @@ def normalize_access_rules(full_config, config2import, import_id, mgm_details={}
                     rule['rule_svc'] = extend_string_list(rule['rule_svc'], rule_orig, 'service', list_delimiter, jwt=jwt, import_id=import_id)
                     rule['rule_src'] = extend_string_list(rule['rule_src'], rule_orig, 'srcaddr6', list_delimiter, jwt=jwt, import_id=import_id)
                     rule['rule_dst'] = extend_string_list(rule['rule_dst'], rule_orig, 'dstaddr6', list_delimiter, jwt=jwt, import_id=import_id)
+                    rule['rule_src'] = extend_string_list(rule['rule_src'], rule_orig, 'internet-service-src-name', list_delimiter, jwt=jwt, import_id=import_id)
 
                     if len(rule_orig['srcintf'])>0:
                         src_obj_zone = fmgr_zone.add_zone_if_missing (config2import, rule_orig['srcintf'][0], import_id)
@@ -194,6 +195,8 @@ def normalize_access_rules(full_config, config2import, import_id, mgm_details={}
 
                     if 'srcaddr-negate' in rule_orig:
                         rule.update({ 'rule_src_neg': rule_orig['srcaddr-negate']=='disable'})
+                    elif 'internet-service-src-negate' in rule_orig:
+                        rule.update({ 'rule_src_neg': rule_orig['internet-service-src-negate']=='disable'})
                     if 'dstaddr-negate' in rule_orig:
                         rule.update({ 'rule_dst_neg': rule_orig['dstaddr-negate']=='disable'})
                     if 'service-negate' in rule_orig:
@@ -411,9 +414,9 @@ def handle_combined_nat_rule(rule, rule_orig, config2import, nat_rule_number, im
                 if hideInterface is not None:
                     obj_name = 'hide_IF_ip_' + str(hideInterface) + '_' + str(destination_interface_ip)
                     obj_comment = 'FWO auto-generated dummy object for source nat'
-                    if type(ipaddress.ip_address(str(destination_interface_ip))) is ipaddress.IPv6Address:
+                    if destination_interface_ip is not None and type(ipaddress.ip_address(str(destination_interface_ip))) is ipaddress.IPv6Address:
                         HideNatIp = str(destination_interface_ip) + '/128'
-                    elif type(ipaddress.ip_address(str(destination_interface_ip))) is ipaddress.IPv4Address:
+                    elif destination_interface_ip is not None and type(ipaddress.ip_address(str(destination_interface_ip))) is ipaddress.IPv4Address:
                         HideNatIp = str(destination_interface_ip) + '/32'
                     else:
                         HideNatIp = '0.0.0.0/32'

roles/importer/files/importer/fortiadom5ff/fwcommon.py

@@ -14,7 +14,7 @@
 
 scope = ['global', 'adom']
 nw_obj_types = ['firewall/address', 'firewall/address6', 'firewall/addrgrp',
-                'firewall/addrgrp6', 'firewall/ippool', 'firewall/vip']
+                'firewall/addrgrp6', 'firewall/ippool', 'firewall/vip', 'system/external-resource']
 svc_obj_types = ['application/list', 'application/group', 'application/categories',
                  'application/custom', 'firewall/service/custom', 'firewall/service/group']
 
@@ -107,6 +107,7 @@ def get_config(config2import, full_config, current_import_id, mgm_details, limit
 
 
 def getObjects(sid, fm_api_url, raw_config, adom_name, limit, scope, nw_obj_types, svc_obj_types):
+    logger = getFwoLogger()
     # get those objects that exist globally and on adom level
     for s in scope:
         # get network objects:
@@ -137,6 +138,35 @@ def getObjects(sid, fm_api_url, raw_config, adom_name, limit, scope, nw_obj_type
                 adom_scope = s
             fmgr_getter.update_config_with_fortinet_api_call(
                 raw_config, sid, fm_api_url, "/pm/config/"+adom_scope+"/obj/" + object_type, "user_obj_" + s + "_" + object_type, limit=limit)
+            
+    # get one arbitrary device and vdom to get dynamic objects
+    # they are equal across all adoms, vdoms, devices
+    devices = fmgr_getter.fortinet_api_call(sid, fm_api_url, '/dvmdb/adom/' + adom_name + '/device')
+    if len(devices)>0 and 'name' in devices[0] and 'vdom' in devices[0] and 'name' in devices[0]['vdom'][0]:
+        arbitraryDevice = devices[0]['name']
+        arbitraryVdom = devices[0]['vdom'][0]['name']
+    else:
+        logger.error('no device or vdom info for adom: ' + adom_name)
+
+    # get dynamic objects
+    payload = {
+        'params': [
+            {
+                'data': {
+                    'action': 'get',
+                    'resource': '/api/v2/monitor/firewall/internet-service-basic?vdom=' + arbitraryVdom,
+                    'target': [
+                        'adom/' + adom_name + '/device/' + arbitraryDevice
+                    ]
+                }
+            }
+        ]
+    }
+    fmgr_getter.update_config_with_fortinet_api_call(
+        raw_config, sid, fm_api_url, "sys/proxy/json", "nw_obj_global_firewall/internet-service-basic", limit=limit, payload=payload, method='exec')
+
+
+
 
 
 # def getZones(sid, fm_api_url, raw_config, adom_name, limit, debug_level):

roles/lib/files/FWO.Basics/Comparer/IPAddressComparer.cs

@@ -0,0 +1,26 @@
+using System.Net;
+using NetTools;
+using FWO.Basics;
+
+namespace FWO.Basics.Comparer
+{
+    public class IPAdressComparer : IComparer<IPAddress>
+    {
+        public int Compare(IPAddress? x, IPAddress? y)
+        {
+            if(x is null || y is null)
+            {
+                return 0;
+            } 
+
+            int compareIPFamiliesResult = IpOperations.CompareIpFamilies(x, y);
+
+            if (compareIPFamiliesResult != 0)
+            {
+                return compareIPFamiliesResult;
+            }
+
+            return IpOperations.CompareIpValues(x, y);
+        }
+    }
+}

roles/lib/files/FWO.Basics/Comparer/IPAddressRangeComparer.cs

@@ -0,0 +1,54 @@
+using System.Numerics;
+using NetTools;
+
+namespace FWO.Basics.Comparer
+{
+    public class IPAddressRangeComparer : IComparer<IPAddressRange>
+    {
+        public int Compare(IPAddressRange? x, IPAddressRange? y)
+        {
+            if(x is null || y is null)
+            {
+                return 0;
+            }
+
+            IPAdressComparer iPAdressComparer = new ();
+            int compareIPAddressResult = iPAdressComparer.Compare(x.Begin, y.Begin);
+
+            if (compareIPAddressResult != 0)
+            {
+                return compareIPAddressResult;
+            }
+
+            BigInteger xRangeSize = GetIPRangeSize(x);
+            BigInteger yRangeSize = GetIPRangeSize(y);
+
+            if(xRangeSize < yRangeSize)
+            {
+                return -1;
+            }
+
+            if(xRangeSize > yRangeSize)
+            {
+                return 1;
+            }
+
+            return 0;
+        }
+
+        public BigInteger GetIPRangeSize(IPAddressRange range)
+        {
+            byte[] startBytes = range.Begin.GetAddressBytes();
+            byte[] endBytes = range.End.GetAddressBytes();
+
+            // prevents overflow problem
+            byte[] startBytesPadded = startBytes.Reverse().Concat(new byte[] { 0 }).ToArray();
+            byte[] endBytesPadded = endBytes.Reverse().Concat(new byte[] { 0 }).ToArray();
+
+            BigInteger startValue = new BigInteger(startBytesPadded);
+            BigInteger endValue = new BigInteger(endBytesPadded);
+
+            return endValue - startValue;
+        }
+    }
+}

roles/lib/files/FWO.Basics/IpOperations.cs

@@ -193,5 +193,93 @@ private static string GetIPv4SubnetMask(int prefixLength)
             uint[] bytes = [(mask >> 24) & 0xff, (mask >> 16) & 0xff, (mask >> 8) & 0xff, mask & 0xff];
             return string.Join(".", bytes);
         }
+
+        /// <summary>
+        /// Compares the values of two IP adresses byte by byte.
+        /// </summary>
+        public static int CompareIpValues(IPAddress ip1, IPAddress ip2)
+        {
+            var ip1Bytes = ip1.GetAddressBytes();
+            var ip2Bytes = ip2.GetAddressBytes();
+
+            for (int i = 0; i < ip1Bytes.Length; i++)
+            {
+                if (ip1Bytes[i] < ip2Bytes[i])
+                {
+                    return -1;
+                }
+                if (ip1Bytes[i] > ip2Bytes[i])
+                {
+                    return 1;
+                }
+            }
+
+            return 0;
+        }
+
+        /// <summary>
+        /// Compares to strings that may contain subnet masks by their format and value.
+        /// </summary>
+        public static int CompareSubnetMasks(string subnetMask1, string subnetMask2)
+        {
+            if (subnetMask1.StartsWith(@"\")) subnetMask1 = subnetMask1.Substring(1);
+            if (subnetMask2.StartsWith(@"\")) subnetMask2 = subnetMask2.Substring(1);
+
+            if(subnetMask1 != subnetMask2)
+            {
+                // first without subnet masks
+                if (subnetMask1 == "") return -1;
+                if (subnetMask2 == "") return 1;
+
+                // then cidr
+                int subnet1CIDR;
+                int subnet2CIDR;
+                bool subnet1IsInt = int.TryParse(subnetMask1, out subnet1CIDR);
+                bool subnet2IsInt = int.TryParse(subnetMask2, out subnet2CIDR);
+                if (subnet1IsInt && subnet2IsInt)
+                {
+                    if (subnet1CIDR < subnet2CIDR) return -1;
+                    if (subnet1CIDR > subnet2CIDR) return 1;
+                }
+                if (subnet1IsInt) return -1;
+                if (subnet2IsInt) return 1;
+
+                
+                IPAddress subnet1IP;
+                IPAddress subnet2IP;
+                bool subnet1IsIp = IPAddress.TryParse(subnetMask1, out subnet1IP);
+                bool subnet2IsIp = IPAddress.TryParse(subnetMask2, out subnet2IP);
+                if (subnet1IsIp && subnet2IsIp)
+                {
+                    // if both in ip format order by value
+                    int compareIpValuesResult = CompareIpValues(subnet1IP, subnet2IP);
+                    if (compareIpValuesResult != 0) return compareIpValuesResult;                  
+                }
+
+                // if one is ip format it should come before the unhandled case
+                if (subnet1IsIp) return -1;
+                if (subnet2IsIp) return 1;
+            }
+
+            // if nothing fits just treat as they were the same
+            return 0;
+        }
+
+        /// <summary>
+        /// Compares two IPAdress objects by their family (IPv4 or IPv6).
+        /// </summary>
+        public static int CompareIpFamilies(IPAddress ip1, IPAddress ip2)
+        {
+            if (ip1.AddressFamily == AddressFamily.InterNetwork && ip2.AddressFamily == AddressFamily.InterNetworkV6 )
+            {
+                return -1;
+            }
+            if (ip1.AddressFamily == AddressFamily.InterNetworkV6 && ip2.AddressFamily == AddressFamily.InterNetwork )
+            {
+                return 1;
+            }
+
+            return 0;
+        }
     }
 }

roles/lib/files/FWO.Basics/StringExtensionsIp.cs

@@ -196,6 +196,17 @@ private static byte[] NormalizeBytes(byte[] bytes, int targetLength)
             return bytes.Take(targetLength).ToArray();  // Ensure it's exactly targetLength bytes
         }
 
+        /// <summary>
+        /// Parses a string to an IPAdress object and a string that represents the subnet mask (empty if there is no subnet mask).
+        /// </summary>
+        public static (IPAddress, string) ToIPAdressAndSubnetMask(this string str)
+        {
+            IPAddress ipAdress = IPAddress.Parse(str.StripOffNetmask());
+            string subnetMask = "";
+            var hasSubnetMask = str.TryGetNetmask(out subnetMask);
+
+            return (ipAdress, hasSubnetMask ? subnetMask.Substring(1) : "");
+        }
+
     }
 }
-