feat(importer-rework): enhance table rule_metadata (#3863)

roles/api/files/replace_metadata.json

@@ -8836,6 +8836,18 @@
                     }
                   }
                 },
+                {
+                  "name": "rule_metadata",
+                  "using": {
+                    "foreign_key_constraint_on": {
+                      "column": "mgm_id",
+                      "table": {
+                        "name": "rule_metadata",
+                        "schema": "public"
+                      }
+                    }
+                  }
+                },
                 {
                   "name": "rule_nwobj_resolveds",
                   "using": {
@@ -15849,6 +15861,12 @@
                 "schema": "public"
               },
               "object_relationships": [
+                {
+                  "name": "management",
+                  "using": {
+                    "foreign_key_constraint_on": "mgm_id"
+                  }
+                },
                 {
                   "name": "uiuser",
                   "using": {
@@ -15907,6 +15925,7 @@
                     "check": {},
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15932,6 +15951,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15957,6 +15977,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -15982,6 +16003,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16007,6 +16029,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16032,6 +16055,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16057,6 +16081,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16082,6 +16107,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16107,6 +16133,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",
@@ -16134,6 +16161,7 @@
                   "permission": {
                     "columns": [
                       "last_change_admin",
+                      "mgm_id",
                       "rule_created",
                       "rule_decert_date",
                       "rule_first_hit",

roles/database/files/sql/creation/fworch-create-constraints.sql

@@ -32,6 +32,7 @@ Alter Table "rule" ADD Constraint "rule_unique_mgm_id_rule_uid_rule_create_xlate
 -- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id");
 -- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id","rulebase_id");
 ALTER TABLE rule_metadata ADD Constraint "rule_metadata_rule_uid_unique" unique ("rule_uid");
+ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
 Alter table "rulebase" add CONSTRAINT unique_rulebase_mgm_id_uid UNIQUE ("mgm_id", "uid");
 Alter table "rulebase_link" add CONSTRAINT unique_rulebase_link
 	UNIQUE (

roles/database/files/sql/creation/fworch-create-foreign-keys.sql

@@ -126,6 +126,8 @@ Alter table "rule_metadata" add constraint "rule_metadata_rule_last_certifier_ui
   foreign key ("rule_last_certifier") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
 Alter table "rule_metadata" add constraint "rule_metadata_rule_owner_uiuser_uiuser_id_f_key"
   foreign key ("rule_owner") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
+ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
+ON update restrict on delete cascade;
 
 Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_rule_rule_id foreign key ("rule_id") references "rule" ("rule_id") on update restrict on delete cascade;
 Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_device_dev_id foreign key ("dev_id") references "device" ("dev_id") on update restrict on delete cascade;

roles/database/files/sql/creation/fworch-create-tables.sql

@@ -198,6 +198,7 @@ Create table "rule_metadata"
 (
 	"rule_metadata_id" BIGSERIAL,
 	"rule_uid" Text NOT NULL,
+	"mgm_id" Integer NOT NULL,
 	"rule_created" Timestamp NOT NULL Default now(),
 	"rule_last_modified" Timestamp NOT NULL Default now(),
 	"rule_first_hit" Timestamp,

roles/database/files/upgrade/9.0.sql

@@ -1677,4 +1677,110 @@ insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufac
 
 insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
     VALUES (29,'Cisco Asa on FirePower','9','Cisco','',false,true,false)
-    ON CONFLICT (dev_typ_id) DO NOTHING;
+    ON CONFLICT (dev_typ_id) DO NOTHING;
+
+
+
+-- rule_metadata add mgm_id + fk, drop constraint
+ALTER TABLE rule_metadata ADD COLUMN IF NOT EXISTS mgm_id Integer;
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 
+        FROM pg_constraint
+        WHERE conname = 'rule_metadata_mgm_id_management_id_fk'
+    ) THEN
+        ALTER TABLE rule_metadata
+        ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk
+        FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
+        ON UPDATE RESTRICT ON DELETE CASCADE;
+    END IF;
+END$$;
+
+
+
+DO $$
+DECLARE
+    rec RECORD;
+    v_do_not_import_true_count INT;
+    v_do_not_import_false_count INT;
+
+BEGIN
+--Check rule_metadata has entries in rule
+    IF EXISTS (
+        SELECT 1
+        FROM rule_metadata rm
+        LEFT JOIN rule r ON rm.rule_uid = r.rule_uid
+        WHERE r.rule_uid IS NULL
+    ) THEN
+        RAISE EXCEPTION 'Some rule_metadata.rule_uid have no matching rule!';
+    ELSE
+        -- Constraints droppen
+        ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_metadatum;
+        ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_rule_metadata_rule_uid_f_key;
+        ALTER TABLE rule_metadata DROP CONSTRAINT IF EXISTS rule_metadata_rule_uid_unique;
+
+-- Start loop for rule_uid und mgm_id import/transfer
+    FOR rec IN
+        SELECT 
+            rm.rule_uid,
+            COUNT(DISTINCT r.mgm_id) AS mgm_count
+        FROM rule_metadata rm
+        JOIN rule r ON rm.rule_uid = r.rule_uid
+        GROUP BY rm.rule_uid
+        HAVING COUNT(DISTINCT r.mgm_id) >= 1
+    LOOP
+        -- Case 1: exactly one mgm_id gefunden
+        IF rec.mgm_count = 1 THEN
+            --
+            UPDATE rule_metadata rm
+            SET mgm_id = r.mgm_id
+            FROM rule r
+            WHERE rm.rule_uid = r.rule_uid
+              AND rm.mgm_id IS NULL
+              AND rm.rule_uid = rec.rule_uid;
+
+        -- Fall 2: genau zwei mgm_id gefunden
+        ELSIF rec.mgm_count >= 2 THEN
+            -- Count flag "do_not_import" for rule_uid 
+            SELECT 
+			COUNT(*) FILTER (WHERE m.do_not_import IS TRUE),
+			COUNT(*) FILTER (WHERE m.do_not_import IS FALSE)
+			INTO v_do_not_import_true_count, v_do_not_import_false_count
+			FROM rule r
+			JOIN management m ON r.mgm_id = m.mgm_id
+			WHERE r.rule_uid = rec.rule_uid;
+
+
+            -- check if there is just 1 "do_not_import" = false
+				IF v_do_not_import_false_count = 1 THEN
+				UPDATE rule_metadata rm
+					SET mgm_id = r.mgm_id
+					FROM rule r
+					JOIN management m ON r.mgm_id = m.mgm_id
+					WHERE rm.rule_uid = r.rule_uid
+					AND m.do_not_import IS FALSE
+					AND rm.rule_uid = rec.rule_uid
+					AND rm.mgm_id IS NULL;
+            END IF;                   
+        END IF;
+    END LOOP;
+	-- redo constraints
+	        ALTER TABLE rule_metadata ALTER COLUMN mgm_id SET NOT NULL;
+        ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_rule_uid_unique UNIQUE(rule_uid);
+        ALTER TABLE rule ADD CONSTRAINT rule_rule_metadata_rule_uid_f_key 
+            FOREIGN KEY (rule_uid) REFERENCES rule_metadata (rule_uid);
+
+			-- set Unique constraint to (mgm_id + rule_uid)
+        IF NOT EXISTS (
+            SELECT 1
+            FROM pg_constraint
+            WHERE conname = 'rule_metadata_mgm_id_rule_uid_unique'
+        ) THEN
+            ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);			
+        END IF;
+    END IF;
+END$$;
+
+
+