feat(importer-rework): enhance table rule_metadata (#3863)

roles/api/files/replace_metadata.json

@@ -8837,6 +8837,18 @@
                     }
                   }
                 },
+                {
+                  "name": "rule_metadata",
+                  "using": {
+                    "foreign_key_constraint_on": {
+                      "column": "mgm_id",
+                      "table": {
+                        "name": "rule_metadata",
+                        "schema": "public"
+                      }
+                    }
+                  }
+                },
                 {
                   "name": "rule_nwobj_resolveds",
                   "using": {
@@ -15850,6 +15862,12 @@
                 "schema": "public"
               },
               "object_relationships": [
+                {
+                  "name": "management",
+                  "using": {
+                    "foreign_key_constraint_on": "mgm_id"
+                  }
+                },
                 {
                   "name": "uiuser",
                   "using": {
@@ -15908,6 +15926,7 @@
                     "check": {},
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15933,6 +15952,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15958,6 +15978,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15983,6 +16004,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16008,6 +16030,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16033,6 +16056,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16058,6 +16082,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16083,6 +16108,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16108,6 +16134,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16135,6 +16162,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",

roles/database/files/sql/creation/fworch-create-constraints.sql

@@ -32,6 +32,7 @@ Alter Table "rule" ADD Constraint "rule_unique_mgm_id_rule_uid_rule_create_xlate
 -- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id");
 -- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id","rulebase_id");
 ALTER TABLE rule_metadata ADD Constraint "rule_metadata_rule_uid_unique" unique ("rule_uid");
+ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
 Alter table "rulebase" add CONSTRAINT unique_rulebase_mgm_id_uid UNIQUE ("mgm_id", "uid");
 Alter table "rulebase_link" add CONSTRAINT unique_rulebase_link
 	UNIQUE (

roles/database/files/sql/creation/fworch-create-foreign-keys.sql

@@ -126,6 +126,8 @@ Alter table "rule_metadata" add constraint "rule_metadata_rule_last_certifier_ui
   foreign key ("rule_last_certifier") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
 Alter table "rule_metadata" add constraint "rule_metadata_rule_owner_uiuser_uiuser_id_f_key"
   foreign key ("rule_owner") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
+ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
+ON update restrict on delete cascade;
 
 Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_rule_rule_id foreign key ("rule_id") references "rule" ("rule_id") on update restrict on delete cascade;
 Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_device_dev_id foreign key ("dev_id") references "device" ("dev_id") on update restrict on delete cascade;

roles/database/files/sql/creation/fworch-create-tables.sql

@@ -198,6 +198,7 @@ Create table "rule_metadata"
 (
 	"rule_metadata_id" BIGSERIAL,
 	"rule_uid" Text NOT NULL,
+	"mgm_id" Integer NOT NULL,
 	"rule_created" Timestamp NOT NULL Default now(),
 	"rule_last_modified" Timestamp NOT NULL Default now(),
 	"rule_first_hit" Timestamp,

roles/database/files/upgrade/9.0.sql

@@ -1683,4 +1683,148 @@ insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufac
 
 insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
     VALUES (29,'Cisco Asa on FirePower','9','Cisco','',false,true,false)
-    ON CONFLICT (dev_typ_id) DO NOTHING;
+    ON CONFLICT (dev_typ_id) DO NOTHING;
+
+
+
+-- rule_metadata add mgm_id + fk, drop constraint
+ALTER TABLE rule_metadata ADD COLUMN IF NOT EXISTS mgm_id Integer;
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 
+        FROM pg_constraint
+        WHERE conname = 'rule_metadata_mgm_id_management_id_fk'
+    ) THEN
+        ALTER TABLE rule_metadata
+        ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk
+        FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
+        ON UPDATE RESTRICT ON DELETE CASCADE;
+    END IF;
+END$$;
+
+
+
+DO $$
+DECLARE
+    rec RECORD;
+    v_do_not_import_true_count INT;
+    v_do_not_import_false_count INT;
+	missing_uids TEXT;
+	too_many_mgm_ids_on_uid_and_no_resolve TEXT;
+	all_errors_with_no_resolve TEXT := '';
+
+BEGIN
+--Check rule_metadata has entries in rule
+    SELECT string_agg(rm.rule_uid::text, ', ')
+    INTO missing_uids
+    FROM rule_metadata rm
+    LEFT JOIN rule r ON rm.rule_uid = r.rule_uid
+    WHERE r.rule_uid IS NULL;
+
+    IF missing_uids IS NOT NULL THEN
+        RAISE EXCEPTION 'Missing rule(s): %', missing_uids;
+    END IF;
+
+
+    -- Constraints droppen
+    ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_metadatum;
+    ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_rule_metadata_rule_uid_f_key;
+    ALTER TABLE rule_metadata DROP CONSTRAINT IF EXISTS rule_metadata_rule_uid_unique;
+
+-- Start loop for rule_uid und mgm_id import/transfer
+    FOR rec IN
+        SELECT 
+            rm.rule_uid,
+            COUNT(DISTINCT r.mgm_id) AS mgm_count
+        FROM rule_metadata rm
+        JOIN rule r ON rm.rule_uid = r.rule_uid
+        GROUP BY rm.rule_uid
+        HAVING COUNT(DISTINCT r.mgm_id) >= 1
+    LOOP
+        -- Case 1: exactly one mgm_id gefunden
+        IF rec.mgm_count = 1 THEN
+            --
+            UPDATE rule_metadata rm
+            SET mgm_id = r.mgm_id
+            FROM rule r
+            WHERE rm.rule_uid = r.rule_uid
+              AND rm.mgm_id IS NULL
+              AND rm.rule_uid = rec.rule_uid;
+
+        -- Case 2: found more then two mgm_id found
+        ELSIF rec.mgm_count >= 2 THEN
+            -- Count flag "do_not_import" for rule_uid 
+            SELECT 
+			COUNT(*) FILTER (WHERE m.do_not_import IS TRUE),
+			COUNT(*) FILTER (WHERE m.do_not_import IS FALSE)
+			INTO v_do_not_import_true_count, v_do_not_import_false_count
+			FROM rule r
+			JOIN management m ON r.mgm_id = m.mgm_id
+			WHERE r.rule_uid = rec.rule_uid;
+
+            -- check if there is just 1 "do_not_import" = false
+			IF v_do_not_import_false_count = 1 THEN
+				UPDATE rule_metadata rm
+					SET mgm_id = r.mgm_id
+					FROM rule r
+					JOIN management m ON r.mgm_id = m.mgm_id
+					WHERE rm.rule_uid = r.rule_uid
+					AND m.do_not_import IS FALSE
+					AND rm.rule_uid = rec.rule_uid
+					AND rm.mgm_id IS NULL;
+
+			-- Warning: Not used mgm_ids where do_not_import=true
+			RAISE NOTICE 'rule_uid % has % additional mgm_id(s) marked do_not_import=true: %', 
+			rec.rule_uid, v_do_not_import_true_count,
+				(SELECT string_agg(format('mgm_id=%s', r.mgm_id), ', ')
+					FROM rule r
+					JOIN management m ON r.mgm_id = m.mgm_id
+					WHERE r.rule_uid = rec.rule_uid
+					AND m.do_not_import IS TRUE);
+
+			ELSE
+				-- No resolve
+				SELECT string_agg(
+                       format('rule_uid=%s → mgm_id=%s (do_not_import=%s)', 
+                              r.rule_uid, r.mgm_id, m.do_not_import),
+                       E'\n'
+					)
+				INTO too_many_mgm_ids_on_uid_and_no_resolve
+				FROM rule r
+				JOIN management m ON r.mgm_id = m.mgm_id
+				WHERE r.rule_uid = rec.rule_uid;	
+
+				all_errors_with_no_resolve := all_errors_with_no_resolve || format(
+                    E'\n\nrule_uid %s has ambiguous mgm_id assignments:\n%s',
+                    rec.rule_uid,
+                    too_many_mgm_ids_on_uid_and_no_resolve
+                );
+
+            END IF;                   
+        END IF;
+    END LOOP;
+
+	    IF all_errors_with_no_resolve <> '' THEN
+			RAISE EXCEPTION 'Ambiguous mgm_id assignments detected:%s', all_errors_with_no_resolve;
+		END IF;
+
+	-- redo constraints
+	    ALTER TABLE rule_metadata ALTER COLUMN mgm_id SET NOT NULL;
+        ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_rule_uid_unique UNIQUE(rule_uid);
+        ALTER TABLE rule ADD CONSTRAINT rule_rule_metadata_rule_uid_f_key 
+            FOREIGN KEY (rule_uid) REFERENCES rule_metadata (rule_uid);
+
+			-- set Unique constraint to (mgm_id + rule_uid)
+        IF NOT EXISTS (
+            SELECT 1
+            FROM pg_constraint
+            WHERE conname = 'rule_metadata_mgm_id_rule_uid_unique'
+        ) THEN
+            ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);			
+        END IF;
+END$$;
+
+
+
+