Backport 6.2: Fix rule fragment loading in rule builder (#22645) (#22674)

* Fix rule fragment loading in rule builder (#22645)

* update rule fragments in RuleBuilderRegistry on cluster event when they are changed via the service, cache these and reload only on change

* cl

* rename local freemarker configuration variable

* move event bus post after db action

(cherry picked from commit defe540)

* add backport pr to changelog

Author: moesterheld

changelog/unreleased/pr-22645.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Fixed using certain conditions and actions in the rule builder after inital setup of Graylog without restart."
+
+issues = ["20793"]
+pulls = ["22645", "22674"]

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/RuleBuilderRegistry.java

@@ -17,11 +17,16 @@
 package org.graylog.plugins.pipelineprocessor.rulebuilder;
 
 import com.google.common.collect.Streams;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import freemarker.cache.StringTemplateLoader;
+import freemarker.template.Configuration;
 import jakarta.inject.Inject;
 import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
 import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.db.RuleFragment;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.db.RuleFragmentService;
+import org.graylog2.bindings.providers.SecureFreemarkerConfigProvider;
 
 import java.util.Collection;
 import java.util.Map;
@@ -32,12 +37,21 @@ public class RuleBuilderRegistry {
 
     private final FunctionRegistry functionRegistry;
     private final RuleFragmentService ruleFragmentService;
+    private final SecureFreemarkerConfigProvider secureFreemarkerConfigProvider;
+    private Collection<RuleFragment> ruleFragments;
+    private Configuration freemarkerConfiguration;
 
     @Inject
     public RuleBuilderRegistry(FunctionRegistry functionRegistry,
-                               RuleFragmentService ruleFragmentService) {
+                               RuleFragmentService ruleFragmentService,
+                               SecureFreemarkerConfigProvider secureFreemarkerConfigProvider,
+                               EventBus eventBus) {
         this.functionRegistry = functionRegistry;
         this.ruleFragmentService = ruleFragmentService;
+        this.ruleFragments = ruleFragmentService.all();
+        this.secureFreemarkerConfigProvider = secureFreemarkerConfigProvider;
+        initializeFragmentTemplates();
+        eventBus.register(this);
     }
 
     /**
@@ -68,7 +82,7 @@ private Map<String, RuleFragment> collectConditions(Collection<Function<?>> func
                         .descriptor(f.descriptor())
                         .build()
                 );
-        final Stream<RuleFragment> fragmentConditions = ruleFragmentService.all().stream()
+        final Stream<RuleFragment> fragmentConditions = ruleFragments.stream()
                 .filter(f -> f.descriptor().ruleBuilderEnabled() && f.isCondition());
         return Streams.concat(functionConditions, fragmentConditions)
                 .collect(Collectors.toMap(f -> f.descriptor().name(), function -> function));
@@ -103,10 +117,30 @@ public Map<String, RuleFragment> collectActions(Collection<Function<?>> function
                         .build()
                 );
         final Stream<RuleFragment> fragmentActions =
-                ruleFragmentService.all().stream()
+                ruleFragments.stream()
                         .filter(f -> f.descriptor().ruleBuilderEnabled() && !f.isCondition());
         return Streams.concat(functionActions, fragmentActions)
                 .collect(Collectors.toMap(f -> f.descriptor().name(), function -> function));
     }
 
+    public Configuration getFreemarkerConfiguration() {
+        return freemarkerConfiguration;
+    }
+
+    private void initializeFragmentTemplates() {
+        Map<String, RuleFragment> fragments = conditionsWithInternal();
+        fragments.putAll(actionsWithInternal());
+        final Configuration config = secureFreemarkerConfigProvider.get();
+        StringTemplateLoader stringTemplateLoader = new StringTemplateLoader();
+        fragments.entrySet().stream().filter(c -> c.getValue().isFragment()).forEach(c -> stringTemplateLoader.putTemplate(c.getKey(), c.getValue().fragment()));
+        config.setTemplateLoader(stringTemplateLoader);
+        this.freemarkerConfiguration = config;
+    }
+
+    @Subscribe
+    public void updateRuleFragments(final RuleFragmentUpdateEvent event) {
+        this.ruleFragments = ruleFragmentService.all();
+        initializeFragmentTemplates();
+    }
+
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/RuleFragmentUpdateEvent.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+
+package org.graylog.plugins.pipelineprocessor.rulebuilder;
+
+public record RuleFragmentUpdateEvent() {
+}

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/db/MongoDBRuleFragmentService.java

@@ -24,8 +24,10 @@
 import com.mongodb.client.model.Sorts;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleFragmentUpdateEvent;
 import org.graylog2.database.MongoCollections;
 import org.graylog2.database.utils.MongoUtils;
+import org.graylog2.events.ClusterEventBus;
 
 import java.util.Collection;
 import java.util.Optional;
@@ -38,27 +40,33 @@ public class MongoDBRuleFragmentService implements RuleFragmentService {
 
     private final MongoCollection<RuleFragment> collection;
     private final MongoUtils<RuleFragment> mongoUtils;
+    private final ClusterEventBus clusterEventBus;
 
     @Inject
-    public MongoDBRuleFragmentService(MongoCollections mongoCollections) {
+    public MongoDBRuleFragmentService(MongoCollections mongoCollections, ClusterEventBus clusterEventBus) {
         collection = mongoCollections.collection(COLLECTION_NAME, RuleFragment.class);
         mongoUtils = mongoCollections.utils(collection);
         collection.createIndex(Indexes.ascending("name"), new IndexOptions().unique(true));
+        this.clusterEventBus = clusterEventBus;
     }
 
     @Override
     public RuleFragment save(RuleFragment ruleFragment) {
-        return mongoUtils.save(ruleFragment);
+        RuleFragment fragment = mongoUtils.save(ruleFragment);
+        clusterEventBus.post(new RuleFragmentUpdateEvent());
+        return fragment;
     }
 
     @Override
     public void delete(String name) {
         collection.deleteOne(eq("name", name));
+        clusterEventBus.post(new RuleFragmentUpdateEvent());
     }
 
     @Override
     public void deleteAll() {
         collection.deleteMany(Filters.empty());
+        clusterEventBus.post(new RuleFragmentUpdateEvent());
     }
 
     @Override

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/parser/ActionParser.java

@@ -16,14 +16,12 @@
  */
 package org.graylog.plugins.pipelineprocessor.rulebuilder.parser;
 
-import freemarker.template.Configuration;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderRegistry;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderStep;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.db.RuleFragment;
-import org.graylog2.bindings.providers.SecureFreemarkerConfigProvider;
 
 import java.util.List;
 import java.util.Map;
@@ -35,18 +33,16 @@
 public class ActionParser {
 
     public static final String NL = System.lineSeparator();
-    protected final Map<String, RuleFragment> actions;
 
-    private final Configuration freemarkerConfiguration;
+    private final RuleBuilderRegistry ruleBuilderRegistry;
 
     @Inject
-    public ActionParser(RuleBuilderRegistry ruleBuilderRegistry, SecureFreemarkerConfigProvider secureFreemarkerConfigProvider) {
-        this.actions = ruleBuilderRegistry.actionsWithInternal();
-        this.freemarkerConfiguration = ParserUtil.initializeFragmentTemplates(secureFreemarkerConfigProvider, actions);
+    public ActionParser(RuleBuilderRegistry ruleBuilderRegistry) {
+        this.ruleBuilderRegistry = ruleBuilderRegistry;
     }
 
     public Map<String, RuleFragment> getActions() {
-        return actions;
+        return ruleBuilderRegistry.actionsWithInternal();
     }
 
     public String generate(List<RuleBuilderStep> actions, boolean generateSimulatorFields) {
@@ -55,7 +51,7 @@ public String generate(List<RuleBuilderStep> actions, boolean generateSimulatorF
     }
 
     String generateAction(RuleBuilderStep step, boolean generateSimulatorFields, int index) {
-        final RuleFragment ruleFragment = actions.get(step.function());
+        final RuleFragment ruleFragment = getActions().get(step.function());
         if (Objects.isNull(ruleFragment)) {
             return "";
         }
@@ -72,7 +68,7 @@ String generateAction(RuleBuilderStep step, boolean generateSimulatorFields, int
         }
 
         if (ruleFragment.isFragment()) {
-            syntax += ParserUtil.generateForFragment(step, freemarkerConfiguration);
+            syntax += ParserUtil.generateForFragment(step, ruleBuilderRegistry.getFreemarkerConfiguration());
         } else {
             syntax += ParserUtil.generateForFunction(step, function) + ";";
         }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/parser/ConditionParser.java

@@ -16,15 +16,13 @@
  */
 package org.graylog.plugins.pipelineprocessor.rulebuilder.parser;
 
-import freemarker.template.Configuration;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 import org.apache.commons.lang3.StringUtils;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderRegistry;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderStep;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.db.RuleFragment;
-import org.graylog2.bindings.providers.SecureFreemarkerConfigProvider;
 
 import java.util.List;
 import java.util.Map;
@@ -37,18 +35,16 @@ public class ConditionParser {
 
     public static final String NL = System.lineSeparator();
     private static final String INDENT = "  ";
-    protected final Map<String, RuleFragment> conditions;
 
-    private final Configuration freemarkerConfiguration;
+    private final RuleBuilderRegistry ruleBuilderRegistry;
 
     @Inject
-    public ConditionParser(RuleBuilderRegistry ruleBuilderRegistry, SecureFreemarkerConfigProvider secureFreemarkerConfigProvider) {
-        this.conditions = ruleBuilderRegistry.conditionsWithInternal();
-        freemarkerConfiguration = ParserUtil.initializeFragmentTemplates(secureFreemarkerConfigProvider, conditions);
+    public ConditionParser(RuleBuilderRegistry ruleBuilderRegistry) {
+        this.ruleBuilderRegistry = ruleBuilderRegistry;
     }
 
     public Map<String, RuleFragment> getConditions() {
-        return conditions;
+        return ruleBuilderRegistry.conditions();
     }
 
     public String generate(List<RuleBuilderStep> ruleConditions, RuleBuilderStep.Operator operator, int level) {
@@ -90,14 +86,14 @@ String generateCondition(RuleBuilderStep step, int level) {
             syntax += "! ";
         }
 
-        final RuleFragment ruleFragment = conditions.get(step.function());
+        final RuleFragment ruleFragment = getConditions().get(step.function());
         if (Objects.isNull(ruleFragment)) {
             return "";
         }
         FunctionDescriptor<?> function = ruleFragment.descriptor();
 
         if (ruleFragment.isFragment()) {
-            syntax += ParserUtil.generateForFragment(step, freemarkerConfiguration);
+            syntax += ParserUtil.generateForFragment(step, ruleBuilderRegistry.getFreemarkerConfiguration());
         } else {
             syntax += ParserUtil.generateForFunction(step, function, level);
         }
@@ -116,4 +112,5 @@ private String generateConditionVariable(int index, RuleBuilderStep step) {
         String fieldname = (step.outputvariable() == null) ? Integer.toString(index) : step.outputvariable();
         return "set_field(\"gl2_simulator_condition_" + fieldname + "\", " + condition + ");";
     }
+
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rulebuilder/parser/ParserUtil.java

@@ -16,7 +16,6 @@
  */
 package org.graylog.plugins.pipelineprocessor.rulebuilder.parser;
 
-import freemarker.cache.StringTemplateLoader;
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template.TemplateNotFoundException;
@@ -25,8 +24,6 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderStep;
-import org.graylog.plugins.pipelineprocessor.rulebuilder.db.RuleFragment;
-import org.graylog2.bindings.providers.SecureFreemarkerConfigProvider;
 
 import java.io.StringWriter;
 import java.util.HashMap;
@@ -80,14 +77,6 @@ static final String addFunctionParameter(ParameterDescriptor descriptor, RuleBui
         return syntax;
     }
 
-    static final Configuration initializeFragmentTemplates(SecureFreemarkerConfigProvider secureFreemarkerConfigProvider, Map<String, RuleFragment> fragments) {
-        final Configuration freemarkerConfiguration = secureFreemarkerConfigProvider.get();
-        StringTemplateLoader stringTemplateLoader = new StringTemplateLoader();
-        fragments.entrySet().stream().filter(c -> c.getValue().isFragment()).forEach(c -> stringTemplateLoader.putTemplate(c.getKey(), c.getValue().fragment()));
-        freemarkerConfiguration.setTemplateLoader(stringTemplateLoader);
-        return freemarkerConfiguration;
-    }
-
     static final String generateForFragment(RuleBuilderStep step, Configuration configuration) {
         final String fragmentName = step.function();
         try {

graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rulebuilder/parser/ActionParserTest.java

@@ -18,6 +18,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
+import com.google.common.eventbus.EventBus;
 import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
 import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderRegistry;
@@ -73,16 +74,18 @@ public static void registerFunctions() {
         RuleFragmentService ruleFragmentService = mock(RuleFragmentService.class);
         when(ruleFragmentService.all()).thenReturn(new ArrayList<>());
 
+        final SecureFreemarkerConfigProvider secureFreemarkerConfigProvider = new SecureFreemarkerConfigProvider();
+        secureFreemarkerConfigProvider.get().setLogTemplateExceptions(false);
         ruleBuilderRegistry = new RuleBuilderRegistry(
                 new FunctionRegistry(functions),
-                ruleFragmentService);
+                ruleFragmentService, secureFreemarkerConfigProvider, new EventBus());
     }
 
     @Before
     public void initialize() {
         final SecureFreemarkerConfigProvider secureFreemarkerConfigProvider = new SecureFreemarkerConfigProvider();
         secureFreemarkerConfigProvider.get().setLogTemplateExceptions(false);
-        actionParser = new ActionParser(ruleBuilderRegistry, secureFreemarkerConfigProvider);
+        actionParser = new ActionParser(ruleBuilderRegistry);
     }
 
 

graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rulebuilder/parser/ConditionParserTest.java

@@ -18,6 +18,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
+import com.google.common.eventbus.EventBus;
 import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
 import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry;
 import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderRegistry;
@@ -68,15 +69,17 @@ public static void registerFunctions() {
         RuleFragmentService ruleFragmentService = mock(RuleFragmentService.class);
         when(ruleFragmentService.all()).thenReturn(new ArrayList<>());
 
+        final SecureFreemarkerConfigProvider secureFreemarkerConfigProvider = new SecureFreemarkerConfigProvider();
+        secureFreemarkerConfigProvider.get().setLogTemplateExceptions(false);
         ruleBuilderRegistry = new RuleBuilderRegistry(new FunctionRegistry(functions),
-                ruleFragmentService);
+                ruleFragmentService, secureFreemarkerConfigProvider, mock(EventBus.class));
     }
 
     @Before
     public void initialize() {
         final SecureFreemarkerConfigProvider secureFreemarkerConfigProvider = new SecureFreemarkerConfigProvider();
         secureFreemarkerConfigProvider.get().setLogTemplateExceptions(false);
-        conditionParser = new ConditionParser(ruleBuilderRegistry, secureFreemarkerConfigProvider);
+        conditionParser = new ConditionParser(ruleBuilderRegistry);
     }