Skip to content

Commit 40b4ccd

Browse files
Imat00Imat00
authored
Merge branch 'importer-rework' into feature/3782-crosstablesRulesZone
2 parents eb7bc34 + ba22881 commit 40b4ccd

File tree

7 files changed

+179
-1
lines changed

7 files changed

+179
-1
lines changed

‎roles/api/files/replace_metadata.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8849,6 +8849,18 @@
88498849
}
88508850
}
88518851
},
8852+
{
8853+
"name": "rule_metadata",
8854+
"using": {
8855+
"foreign_key_constraint_on": {
8856+
"column": "mgm_id",
8857+
"table": {
8858+
"name": "rule_metadata",
8859+
"schema": "public"
8860+
}
8861+
}
8862+
}
8863+
},
88528864
{
88538865
"name": "rule_nwobj_resolveds",
88548866
"using": {
@@ -16113,6 +16125,12 @@
1611316125
"schema": "public"
1611416126
},
1611516127
"object_relationships": [
16128+
{
16129+
"name": "management",
16130+
"using": {
16131+
"foreign_key_constraint_on": "mgm_id"
16132+
}
16133+
},
1611616134
{
1611716135
"name": "uiuser",
1611816136
"using": {
@@ -16171,6 +16189,7 @@
1617116189
"check": {},
1617216190
"columns": [
1617316191
"last_change_admin",
16192+
"mgm_id",
1617416193
"rule_created",
1617516194
"rule_decert_date",
1617616195
"rule_first_hit",
@@ -16196,6 +16215,7 @@
1619616215
"permission": {
1619716216
"columns": [
1619816217
"last_change_admin",
16218+
"mgm_id",
1619916219
"rule_created",
1620016220
"rule_decert_date",
1620116221
"rule_first_hit",
@@ -16221,6 +16241,7 @@
1622116241
"permission": {
1622216242
"columns": [
1622316243
"last_change_admin",
16244+
"mgm_id",
1622416245
"rule_created",
1622516246
"rule_decert_date",
1622616247
"rule_first_hit",
@@ -16246,6 +16267,7 @@
1624616267
"permission": {
1624716268
"columns": [
1624816269
"last_change_admin",
16270+
"mgm_id",
1624916271
"rule_created",
1625016272
"rule_decert_date",
1625116273
"rule_first_hit",
@@ -16271,6 +16293,7 @@
1627116293
"permission": {
1627216294
"columns": [
1627316295
"last_change_admin",
16296+
"mgm_id",
1627416297
"rule_created",
1627516298
"rule_decert_date",
1627616299
"rule_first_hit",
@@ -16296,6 +16319,7 @@
1629616319
"permission": {
1629716320
"columns": [
1629816321
"last_change_admin",
16322+
"mgm_id",
1629916323
"rule_created",
1630016324
"rule_decert_date",
1630116325
"rule_first_hit",
@@ -16321,6 +16345,7 @@
1632116345
"permission": {
1632216346
"columns": [
1632316347
"last_change_admin",
16348+
"mgm_id",
1632416349
"rule_created",
1632516350
"rule_decert_date",
1632616351
"rule_first_hit",
@@ -16346,6 +16371,7 @@
1634616371
"permission": {
1634716372
"columns": [
1634816373
"last_change_admin",
16374+
"mgm_id",
1634916375
"rule_created",
1635016376
"rule_decert_date",
1635116377
"rule_first_hit",
@@ -16371,6 +16397,7 @@
1637116397
"permission": {
1637216398
"columns": [
1637316399
"last_change_admin",
16400+
"mgm_id",
1637416401
"rule_created",
1637516402
"rule_decert_date",
1637616403
"rule_first_hit",
@@ -16398,6 +16425,7 @@
1639816425
"permission": {
1639916426
"columns": [
1640016427
"last_change_admin",
16428+
"mgm_id",
1640116429
"rule_created",
1640216430
"rule_decert_date",
1640316431
"rule_first_hit",

‎roles/database/files/sql/creation/fworch-create-constraints.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Alter Table "rule" ADD Constraint "rule_unique_mgm_id_rule_uid_rule_create_xlate
3232
-- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id");
3333
-- Alter Table "rule_metadata" add Constraint "rule_metadata_alt_key" UNIQUE ("rule_uid","dev_id","rulebase_id");
3434
ALTER TABLE rule_metadata ADD Constraint "rule_metadata_rule_uid_unique" unique ("rule_uid");
35+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
3536
Alter table "rulebase" add CONSTRAINT unique_rulebase_mgm_id_uid UNIQUE ("mgm_id", "uid");
3637
Alter table "rulebase_link" add CONSTRAINT unique_rulebase_link
3738
UNIQUE (

‎roles/database/files/sql/creation/fworch-create-foreign-keys.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ Alter table "rule_metadata" add constraint "rule_metadata_rule_last_certifier_ui
131131
foreign key ("rule_last_certifier") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
132132
Alter table "rule_metadata" add constraint "rule_metadata_rule_owner_uiuser_uiuser_id_f_key"
133133
foreign key ("rule_owner") references "uiuser" ("uiuser_id") on update restrict on delete cascade;
134+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
135+
ON update restrict on delete cascade;
134136

135137
Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_rule_rule_id foreign key ("rule_id") references "rule" ("rule_id") on update restrict on delete cascade;
136138
Alter table "rule_enforced_on_gateway" add CONSTRAINT fk_rule_enforced_on_gateway_device_dev_id foreign key ("dev_id") references "device" ("dev_id") on update restrict on delete cascade;

‎roles/database/files/sql/creation/fworch-create-tables.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ Create table "rule_metadata"
198198
(
199199
"rule_metadata_id" BIGSERIAL,
200200
"rule_uid" Text NOT NULL,
201+
"mgm_id" Integer NOT NULL,
201202
"rule_created" Timestamp NOT NULL Default now(),
202203
"rule_last_modified" Timestamp NOT NULL Default now(),
203204
"rule_first_hit" Timestamp,

‎roles/database/files/upgrade/9.0.sql‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1823,3 +1823,147 @@ insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufac
18231823
insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
18241824
VALUES (29,'Cisco Asa on FirePower','9','Cisco','',false,true,false)
18251825
ON CONFLICT (dev_typ_id) DO NOTHING;
1826+
1827+
1828+
1829+
-- rule_metadata add mgm_id + fk, drop constraint
1830+
ALTER TABLE rule_metadata ADD COLUMN IF NOT EXISTS mgm_id Integer;
1831+
DO $$
1832+
BEGIN
1833+
IF NOT EXISTS (
1834+
SELECT 1
1835+
FROM pg_constraint
1836+
WHERE conname = 'rule_metadata_mgm_id_management_id_fk'
1837+
) THEN
1838+
ALTER TABLE rule_metadata
1839+
ADD CONSTRAINT rule_metadata_mgm_id_management_id_fk
1840+
FOREIGN KEY (mgm_id) REFERENCES management(mgm_id)
1841+
ON UPDATE RESTRICT ON DELETE CASCADE;
1842+
END IF;
1843+
END$$;
1844+
1845+
1846+
1847+
DO $$
1848+
DECLARE
1849+
rec RECORD;
1850+
v_do_not_import_true_count INT;
1851+
v_do_not_import_false_count INT;
1852+
missing_uids TEXT;
1853+
too_many_mgm_ids_on_uid_and_no_resolve TEXT;
1854+
all_errors_with_no_resolve TEXT := '';
1855+
1856+
BEGIN
1857+
--Check rule_metadata has entries in rule
1858+
SELECT string_agg(rm.rule_uid::text, ', ')
1859+
INTO missing_uids
1860+
FROM rule_metadata rm
1861+
LEFT JOIN rule r ON rm.rule_uid = r.rule_uid
1862+
WHERE r.rule_uid IS NULL;
1863+
1864+
IF missing_uids IS NOT NULL THEN
1865+
RAISE EXCEPTION 'Missing rule(s): %', missing_uids;
1866+
END IF;
1867+
1868+
1869+
-- Constraints droppen
1870+
ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_metadatum;
1871+
ALTER TABLE rule DROP CONSTRAINT IF EXISTS rule_rule_metadata_rule_uid_f_key;
1872+
ALTER TABLE rule_metadata DROP CONSTRAINT IF EXISTS rule_metadata_rule_uid_unique;
1873+
1874+
-- Start loop for rule_uid und mgm_id import/transfer
1875+
FOR rec IN
1876+
SELECT
1877+
rm.rule_uid,
1878+
COUNT(DISTINCT r.mgm_id) AS mgm_count
1879+
FROM rule_metadata rm
1880+
JOIN rule r ON rm.rule_uid = r.rule_uid
1881+
GROUP BY rm.rule_uid
1882+
HAVING COUNT(DISTINCT r.mgm_id) >= 1
1883+
LOOP
1884+
-- Case 1: exactly one mgm_id gefunden
1885+
IF rec.mgm_count = 1 THEN
1886+
--
1887+
UPDATE rule_metadata rm
1888+
SET mgm_id = r.mgm_id
1889+
FROM rule r
1890+
WHERE rm.rule_uid = r.rule_uid
1891+
AND rm.mgm_id IS NULL
1892+
AND rm.rule_uid = rec.rule_uid;
1893+
1894+
-- Case 2: found more then two mgm_id found
1895+
ELSIF rec.mgm_count >= 2 THEN
1896+
-- Count flag "do_not_import" for rule_uid
1897+
SELECT
1898+
COUNT(*) FILTER (WHERE m.do_not_import IS TRUE),
1899+
COUNT(*) FILTER (WHERE m.do_not_import IS FALSE)
1900+
INTO v_do_not_import_true_count, v_do_not_import_false_count
1901+
FROM rule r
1902+
JOIN management m ON r.mgm_id = m.mgm_id
1903+
WHERE r.rule_uid = rec.rule_uid;
1904+
1905+
-- check if there is just 1 "do_not_import" = false
1906+
IF v_do_not_import_false_count = 1 THEN
1907+
UPDATE rule_metadata rm
1908+
SET mgm_id = r.mgm_id
1909+
FROM rule r
1910+
JOIN management m ON r.mgm_id = m.mgm_id
1911+
WHERE rm.rule_uid = r.rule_uid
1912+
AND m.do_not_import IS FALSE
1913+
AND rm.rule_uid = rec.rule_uid
1914+
AND rm.mgm_id IS NULL;
1915+
1916+
-- Warning: Not used mgm_ids where do_not_import=true
1917+
RAISE NOTICE 'rule_uid % has % additional mgm_id(s) marked do_not_import=true: %',
1918+
rec.rule_uid, v_do_not_import_true_count,
1919+
(SELECT string_agg(format('mgm_id=%s', r.mgm_id), ', ')
1920+
FROM rule r
1921+
JOIN management m ON r.mgm_id = m.mgm_id
1922+
WHERE r.rule_uid = rec.rule_uid
1923+
AND m.do_not_import IS TRUE);
1924+
1925+
ELSE
1926+
-- No resolve
1927+
SELECT string_agg(
1928+
format('rule_uid=%s → mgm_id=%s (do_not_import=%s)',
1929+
r.rule_uid, r.mgm_id, m.do_not_import),
1930+
E'\n'
1931+
)
1932+
INTO too_many_mgm_ids_on_uid_and_no_resolve
1933+
FROM rule r
1934+
JOIN management m ON r.mgm_id = m.mgm_id
1935+
WHERE r.rule_uid = rec.rule_uid;
1936+
1937+
all_errors_with_no_resolve := all_errors_with_no_resolve || format(
1938+
E'\n\nrule_uid %s has ambiguous mgm_id assignments:\n%s',
1939+
rec.rule_uid,
1940+
too_many_mgm_ids_on_uid_and_no_resolve
1941+
);
1942+
1943+
END IF;
1944+
END IF;
1945+
END LOOP;
1946+
1947+
IF all_errors_with_no_resolve <> '' THEN
1948+
RAISE EXCEPTION 'Ambiguous mgm_id assignments detected:%s', all_errors_with_no_resolve;
1949+
END IF;
1950+
1951+
-- redo constraints
1952+
ALTER TABLE rule_metadata ALTER COLUMN mgm_id SET NOT NULL;
1953+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_rule_uid_unique UNIQUE(rule_uid);
1954+
ALTER TABLE rule ADD CONSTRAINT rule_rule_metadata_rule_uid_f_key
1955+
FOREIGN KEY (rule_uid) REFERENCES rule_metadata (rule_uid);
1956+
1957+
-- set Unique constraint to (mgm_id + rule_uid)
1958+
IF NOT EXISTS (
1959+
SELECT 1
1960+
FROM pg_constraint
1961+
WHERE conname = 'rule_metadata_mgm_id_rule_uid_unique'
1962+
) THEN
1963+
ALTER TABLE rule_metadata ADD CONSTRAINT rule_metadata_mgm_id_rule_uid_unique UNIQUE (mgm_id, rule_uid);
1964+
END IF;
1965+
END$$;
1966+
1967+
1968+
1969+

‎roles/importer/files/importer/model_controllers/fwconfig_import_rule.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ def collect_all_hit_information(self, prev_config: FwConfigNormalized, new_hit_i
182182
def add_hit_update(new_hit_information: list[dict], rule: RuleNormalized):
183183
"""Add a hit information update entry for a rule."""
184184
new_hit_information.append({
185-
"where": { "rule_uid": { "_eq": rule.rule_uid } },
185+
"where": { "rule_uid": { "_eq": rule.rule_uid }, "mgm_id": { "_eq": self.import_details.MgmDetails.CurrentMgmId } },
186186
"_set": { "rule_last_hit": rule.last_hit }
187187
})
188188

@@ -763,6 +763,7 @@ def PrepareNewRuleMetadata(self, newRules: list[Rulebase]) -> list[dict]:
763763
for rule_uid, rule in rulebase.rules.items():
764764
rm4import = RuleMetadatum(
765765
rule_uid=rule_uid,
766+
mgm_id=self.import_details.MgmDetails.CurrentMgmId,
766767
rule_last_modified=now,
767768
rule_created=now,
768769
rule_last_hit=rule.last_hit,

‎roles/importer/files/importer/models/rule_metadatum.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
# Rule is the model for a normalized rule_metadata
2828
class RuleMetadatum(BaseModel):
2929
rule_uid: str
30+
mgm_id: int
3031
rule_created: str|None = None
3132
rule_last_modified: str|None = None
3233
rule_first_hit: str|None = None

0 commit comments

Comments
 (0)