Skip to content

Commit ba22881

Browse files
tpurschketpurschke
authored
Merge pull request #3885 from Imat00/feature/importer-rework-rule_metadataEnhance
feat(importer-rework): enhance table rule_metadata (#3863)
2 parents f99bed8 + d20ac9e commit ba22881

File tree

7 files changed

+180
-2
lines changed

7 files changed

+180
-2
lines changed

‎roles/api/files/replace_metadata.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8837,6 +8837,18 @@
88378837
}
88388838
}
88398839
},
8840+
{
8841+
"name": "rule_metadata",
8842+
"using": {
8843+
"foreign_key_constraint_on": {
8844+
"column": "mgm_id",
8845+
"table": {
8846+
"name": "rule_metadata",
8847+
"schema": "public"
8848+
}
8849+
}
8850+
}
8851+
},
88408852
{
88418853
"name": "rule_nwobj_resolveds",
88428854
"using": {
@@ -15850,6 +15862,12 @@
1585015862
"schema": "public"
1585115863
},
1585215864
"object_relationships": [
15865+
{
15866+
"name": "management",
15867+
"using": {
15868+
"foreign_key_constraint_on": "mgm_id"
15869+
}
15870+
},
1585315871
{
1585415872
"name": "uiuser",
1585515873
"using": {
@@ -15908,6 +15926,7 @@
1590815926
"check": {},
1590915927
"columns": [
1591015928
"last_change_admin",
15929+
"mgm_id",
1591115930
"rule_created",
1591215931
"rule_decert_date",
1591315932
"rule_first_hit",
@@ -15933,6 +15952,7 @@
1593315952
"permission": {
1593415953
"columns": [
1593515954
"last_change_admin",
15955+
"mgm_id",
1593615956
"rule_created",
1593715957
"rule_decert_date",
1593815958
"rule_first_hit",
@@ -15958,6 +15978,7 @@
1595815978
"permission": {
1595915979
"columns": [
1596015980
"last_change_admin",
15981+
"mgm_id",
1596115982
"rule_created",
1596215983
"rule_decert_date",
1596315984
"rule_first_hit",
@@ -15983,6 +16004,7 @@
1598316004
"permission": {
1598416005
"columns": [
1598516006
"last_change_admin",
16007+
"mgm_id",
1598616008
"rule_created",
1598716009
"rule_decert_date",
1598816010
"rule_first_hit",
@@ -16008,6 +16030,7 @@
1600816030
"permission": {
1600916031
"columns": [
1601016032
"last_change_admin",
16033+
"mgm_id",
1601116034
"rule_created",
1601216035
"rule_decert_date",
1601316036
"rule_first_hit",
@@ -16033,6 +16056,7 @@
1603316056
"permission": {
1603416057
"columns": [
1603516058
"last_change_admin",
16059+
"mgm_id",
1603616060
"rule_created",
1603716061
"rule_decert_date",
1603816062
"rule_first_hit",
@@ -16058,6 +16082,7 @@
1605816082
"permission": {
1605916083
"columns": [
1606016084
"last_change_admin",
16085+
"mgm_id",
1606116086
"rule_created",
1606216087
"rule_decert_date",
1606316088
"rule_first_hit",
@@ -16083,6 +16108,7 @@
1608316108
"permission": {
1608416109
"columns": [
1608516110
"last_change_admin",
16111+
"mgm_id",
1608616112
"rule_created",
1608716113
"rule_decert_date",
1608816114
"rule_first_hit",
@@ -16108,6 +16134,7 @@
1610816134
"permission": {
1610916135
"columns": [
1611016136
"last_change_admin",
16137+
"mgm_id",
1611116138
"rule_created",
1611216139
"rule_decert_date",
1611316140
"rule_first_hit",
@@ -16135,6 +16162,7 @@
1613516162
"permission": {
1613616163
"columns": [
1613716164
"last_change_admin",
16165+
"mgm_id",
1613816166
"rule_created",
1613916167
"rule_decert_date",
1614016168
"rule_first_hit",

‎roles/database/files/sql/creation/fworch-create-constraints.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Alter Table "rule" ADD Constraint "rule_unique_mgm_id_rule_uid_rule_create_xlate
3232
-- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id");
3333
-- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id","rulebase_id");
3434
ALTER TABLE rule_metadata ADD Constraint "rule_metadata_rule_uid_unique" unique ("rule_uid");
35+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
3536
Alter table "rulebase" add CONSTRAINT unique_rulebase_mgm_id_uid UNIQUE ("mgm_id", "uid");
3637
Alter table "rulebase_link" add CONSTRAINT unique_rulebase_link
3738
UNIQUE (

‎roles/database/files/sql/creation/fworch-create-foreign-keys.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ Alter table "rule_metadata" add constraint "rule_metadata_rule_last_certifier_ui
126126
foreign key ("rule_last_certifier") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
127127
Alter table "rule_metadata" add constraint "rule_metadata_rule_owner_uiuser_uiuser_id_f_key"
128128
foreign key ("rule_owner") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
129+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
130+
ON update restrict on delete cascade;
129131

130132
Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_rule_rule_id foreign key ("rule_id") references "rule" ("rule_id") on update restrict on delete cascade;
131133
Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_device_dev_id foreign key ("dev_id") references "device" ("dev_id") on update restrict on delete cascade;

‎roles/database/files/sql/creation/fworch-create-tables.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ Create table "rule_metadata"
198198
(
199199
"rule_metadata_id" BIGSERIAL,
200200
"rule_uid" Text NOT NULL,
201+
"mgm_id" Integer NOT NULL,
201202
"rule_created" Timestamp NOT NULL Default now(),
202203
"rule_last_modified" Timestamp NOT NULL Default now(),
203204
"rule_first_hit" Timestamp,

‎roles/database/files/upgrade/9.0.sql‎

Lines changed: 145 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1683,4 +1683,148 @@ insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufac
16831683

16841684
insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
16851685
VALUES (29,'Cisco Asa on FirePower','9','Cisco','',false,true,false)
1686-
ON CONFLICT (dev_typ_id) DO NOTHING;
1686+
ON CONFLICT (dev_typ_id) DO NOTHING;
1687+
1688+
1689+
1690+
-- rule_metadata add mgm_id + fk, drop constraint
1691+
ALTER TABLE rule_metadata ADD COLUMN IF NOT EXISTS mgm_id Integer;
1692+
DO $$
1693+
BEGIN
1694+
IF NOT EXISTS (
1695+
SELECT 1
1696+
FROM pg_constraint
1697+
WHERE conname = 'rule_metadata_mgm_id_management_id_fk'
1698+
) THEN
1699+
ALTER TABLE rule_metadata
1700+
ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk
1701+
FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
1702+
ON UPDATE RESTRICT ON DELETE CASCADE;
1703+
END IF;
1704+
END$$;
1705+
1706+
1707+
1708+
DO $$
1709+
DECLARE
1710+
rec RECORD;
1711+
v_do_not_import_true_count INT;
1712+
v_do_not_import_false_count INT;
1713+
missing_uids TEXT;
1714+
too_many_mgm_ids_on_uid_and_no_resolve TEXT;
1715+
all_errors_with_no_resolve TEXT := '';
1716+
1717+
BEGIN
1718+
--Check rule_metadata has entries in rule
1719+
SELECT string_agg(rm.rule_uid::text, ', ')
1720+
INTO missing_uids
1721+
FROM rule_metadata rm
1722+
LEFT JOIN rule r ON rm.rule_uid = r.rule_uid
1723+
WHERE r.rule_uid IS NULL;
1724+
1725+
IF missing_uids IS NOT NULL THEN
1726+
RAISE EXCEPTION 'Missing rule(s): %', missing_uids;
1727+
END IF;
1728+
1729+
1730+
-- Constraints droppen
1731+
ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_metadatum;
1732+
ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_rule_metadata_rule_uid_f_key;
1733+
ALTER TABLE rule_metadata DROP CONSTRAINT IF EXISTS rule_metadata_rule_uid_unique;
1734+
1735+
-- Start loop for rule_uid und mgm_id import/transfer
1736+
FOR rec IN
1737+
SELECT
1738+
rm.rule_uid,
1739+
COUNT(DISTINCT r.mgm_id) AS mgm_count
1740+
FROM rule_metadata rm
1741+
JOIN rule r ON rm.rule_uid = r.rule_uid
1742+
GROUP BY rm.rule_uid
1743+
HAVING COUNT(DISTINCT r.mgm_id) >= 1
1744+
LOOP
1745+
-- Case 1: exactly one mgm_id gefunden
1746+
IF rec.mgm_count = 1 THEN
1747+
--
1748+
UPDATE rule_metadata rm
1749+
SET mgm_id = r.mgm_id
1750+
FROM rule r
1751+
WHERE rm.rule_uid = r.rule_uid
1752+
AND rm.mgm_id IS NULL
1753+
AND rm.rule_uid = rec.rule_uid;
1754+
1755+
-- Case 2: found more then two mgm_id found
1756+
ELSIF rec.mgm_count >= 2 THEN
1757+
-- Count flag "do_not_import" for rule_uid
1758+
SELECT
1759+
COUNT(*) FILTER (WHERE m.do_not_import IS TRUE),
1760+
COUNT(*) FILTER (WHERE m.do_not_import IS FALSE)
1761+
INTO v_do_not_import_true_count, v_do_not_import_false_count
1762+
FROM rule r
1763+
JOIN management m ON r.mgm_id = m.mgm_id
1764+
WHERE r.rule_uid = rec.rule_uid;
1765+
1766+
-- check if there is just 1 "do_not_import" = false
1767+
IF v_do_not_import_false_count = 1 THEN
1768+
UPDATE rule_metadata rm
1769+
SET mgm_id = r.mgm_id
1770+
FROM rule r
1771+
JOIN management m ON r.mgm_id = m.mgm_id
1772+
WHERE rm.rule_uid = r.rule_uid
1773+
AND m.do_not_import IS FALSE
1774+
AND rm.rule_uid = rec.rule_uid
1775+
AND rm.mgm_id IS NULL;
1776+
1777+
-- Warning: Not used mgm_ids where do_not_import=true
1778+
RAISE NOTICE 'rule_uid % has % additional mgm_id(s) marked do_not_import=true: %',
1779+
rec.rule_uid, v_do_not_import_true_count,
1780+
(SELECT string_agg(format('mgm_id=%s', r.mgm_id), ', ')
1781+
FROM rule r
1782+
JOIN management m ON r.mgm_id = m.mgm_id
1783+
WHERE r.rule_uid = rec.rule_uid
1784+
AND m.do_not_import IS TRUE);
1785+
1786+
ELSE
1787+
-- No resolve
1788+
SELECT string_agg(
1789+
format('rule_uid=%s → mgm_id=%s (do_not_import=%s)',
1790+
r.rule_uid, r.mgm_id, m.do_not_import),
1791+
E'\n'
1792+
)
1793+
INTO too_many_mgm_ids_on_uid_and_no_resolve
1794+
FROM rule r
1795+
JOIN management m ON r.mgm_id = m.mgm_id
1796+
WHERE r.rule_uid = rec.rule_uid;
1797+
1798+
all_errors_with_no_resolve := all_errors_with_no_resolve || format(
1799+
E'\n\nrule_uid %s has ambiguous mgm_id assignments:\n%s',
1800+
rec.rule_uid,
1801+
too_many_mgm_ids_on_uid_and_no_resolve
1802+
);
1803+
1804+
END IF;
1805+
END IF;
1806+
END LOOP;
1807+
1808+
IF all_errors_with_no_resolve <> '' THEN
1809+
RAISE EXCEPTION 'Ambiguous mgm_id assignments detected:%s', all_errors_with_no_resolve;
1810+
END IF;
1811+
1812+
-- redo constraints
1813+
ALTER TABLE rule_metadata ALTER COLUMN mgm_id SET NOT NULL;
1814+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_rule_uid_unique UNIQUE(rule_uid);
1815+
ALTER TABLE rule ADD CONSTRAINT rule_rule_metadata_rule_uid_f_key
1816+
FOREIGN KEY (rule_uid) REFERENCES rule_metadata (rule_uid);
1817+
1818+
-- set Unique constraint to (mgm_id + rule_uid)
1819+
IF NOT EXISTS (
1820+
SELECT 1
1821+
FROM pg_constraint
1822+
WHERE conname = 'rule_metadata_mgm_id_rule_uid_unique'
1823+
) THEN
1824+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
1825+
END IF;
1826+
END$$;
1827+
1828+
1829+
1830+

‎roles/importer/files/importer/model_controllers/fwconfig_import_rule.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ def collect_all_hit_information(self, prev_config: FwConfigNormalized, new_hit_i
180180
def add_hit_update(new_hit_information: list[dict], rule: RuleNormalized):
181181
"""Add a hit information update entry for a rule."""
182182
new_hit_information.append({
183-
"where": { "rule_uid": { "_eq": rule.rule_uid } },
183+
"where": { "rule_uid": { "_eq": rule.rule_uid }, "mgm_id": { "_eq": self.import_details.MgmDetails.CurrentMgmId } },
184184
"_set": { "rule_last_hit": rule.last_hit }
185185
})
186186

@@ -728,6 +728,7 @@ def PrepareNewRuleMetadata(self, newRules: list[Rulebase]) -> list[dict]:
728728
for rule_uid, rule in rulebase.rules.items():
729729
rm4import = RuleMetadatum(
730730
rule_uid=rule_uid,
731+
mgm_id=self.import_details.MgmDetails.CurrentMgmId,
731732
rule_last_modified=now,
732733
rule_created=now,
733734
rule_last_hit=rule.last_hit,

‎roles/importer/files/importer/models/rule_metadatum.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
# Rule is the model for a normalized rule_metadata
2828
class RuleMetadatum(BaseModel):
2929
rule_uid: str
30+
mgm_id: int
3031
rule_created: str|None = None
3132
rule_last_modified: str|None = None
3233
rule_first_hit: str|None = None

0 commit comments

Comments
 (0)