add experimental support for ignored allowed groups

Author: nvdk

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## Unreleased
+**Features**
+- added ignored groups: groups that should not be taken into account when searching
+
 ## v0.10.0
 **Fixes**
 - [#92](https://github.com/mu-semtech/mu-search/issues/92) fix restoring update-handler if file exceeds 3MB

README.md

@@ -787,6 +787,10 @@ Assume your application contains a company-specific user group in the authorizat
 
 A typical group to be specified as a single `eager_indexing_group` is `{ "variables": [], "name": "clean" }`. The index will not contain any data, but will be used in the combination to fully match the user's allowed groups.
 
+#### [EXPERIMENTAL] Ignoring allowed groups
+In some cases you may search to ignore certain allowed groups when looking for matching indexes. Typically because they will not relate to data that has to be indexed and you want to avoid having many empty indexes. In this case you will have to provide an entry in the `ignored_allowed_groups` list for each group, currently this means including each possible variable value. 
+For example the clean group can be added to `ignored_allowed_groups` by adding `{ "variables": [], "name": "clean" }` to the list. 
+
 ### Delta integration
 Mu-search integrates with the delta's generated by [mu-authorization](https://github.com/mu-semtech/mu-authorization) and dispatched by the [delta-notifier](https://github.com/mu-semtech/delta-notifier).
 

config/config.json

@@ -4,6 +4,9 @@
     "automatic_index_updates" : true,
     "eager_indexing_groups" : [[{"name" : "documents", "variables" : ["human"]}],
                                [{"name" : "documents", "variables" : ["chicken"]}]],
+    "ignored_allowed_groups": [
+      { "variables": [], "name": "clean" }
+    ],
     "attachments_path_base" : "/local/files/directory",
     "persist_indexes" : false,
     "default_settings" : {

lib/mu_search/config_parser.rb

@@ -47,7 +47,7 @@ def self.parse(path)
       if json_config["eager_indexing_groups"]
         config[:eager_indexing_groups] = json_config["eager_indexing_groups"]
       end
-
+      config[:ignored_allowed_groups] = json_config["ignored_allowed_groups"] || []
       config[:type_definitions] = Hash[MuSearch::IndexDefinition.from_json_config(json_config["types"])]
       config
     end
@@ -117,12 +117,33 @@ def self.validate_config(json_config)
       else
         errors << "no type definitions specified, expected field 'types' not found"
       end
+      if json_config.has_key?("ignored_allowed_groups")
+        errors = errors.concat(self.validate_ignored_allowed_groups(json_config["ignored_allowed_groups"]))
+      end
       if errors.length > 0
         Mu::log.error("CONFIG_PARSER") { errors.join("\n") }
         raise "invalid config"
       end
     end
 
+    def self.validate_ignored_allowed_groups(groups)
+      errors = []
+      if ! groups.kind_of?(Array)
+        errors << "ignored_allowed_groups should be an array"
+      else
+        groups.each do |group|
+          if ! group.kind_of?(Hash)
+            errors << "ignored_allowed_group is not an object: #{group.inspect}"
+          else
+            if ! group.has_key?("name") && group.has_key?("variables")
+              errors << "ignored_allowed_group should have both name and variables set: #{group.inspect}"
+            end
+          end
+        end
+      end
+      errors
+    end
+
     ##
     # basic validations of typedefinitions
     #

lib/mu_search/index_manager.rb

@@ -8,7 +8,7 @@ module MuSearch
   # - triplestore
   ###
   class IndexManager
-    attr_reader :indexes
+    attr_reader :indexes, :ignored_allowed_groups
 
     def initialize(logger:, elasticsearch:, tika:, sparql_connection_pool:, search_configuration:)
       @logger = logger
@@ -18,7 +18,7 @@ def initialize(logger:, elasticsearch:, tika:, sparql_connection_pool:, search_c
       @master_mutex = Mutex.new
       @configuration = search_configuration
       @indexes = {} # indexes per type
-
+      @ignored_allowed_groups = search_configuration[:ignored_allowed_groups]
       initialize_indexes
     end
 
@@ -200,7 +200,10 @@ def find_single_index_for_groups(type_name, allowed_groups, used_groups = [])
     # TODO take used_groups into account when they are supported by mu-authorization
     def ensure_index_combination_for_groups(type_name, allowed_groups, used_groups = [])
       @logger.debug("INDEX MGMT") { "Trying to combine indexes in cache for type '#{type_name}' to match allowed_groups #{allowed_groups} and used_groups #{used_groups}" }
-
+      if !ignored_allowed_groups.empty?
+        @logger.debug("INDEX MGMT") { "Ignoring the following allowed groups: #{ignored_allowed_groups.inspect}" }
+        allowed_groups.reject!{ |ag| ignored_allowed_groups.include? ag }
+      end
       indexes = @indexes[type_name].values.find_all(&:eager_index?)
       @logger.debug("INDEX MGMT") { "Currently known indexes for type '#{type_name}': #{indexes.map(&:allowed_groups).to_json}" }
       # Find all indexes with allowed_groups that are a subset of the given allowed_groups

web.rb

@@ -52,7 +52,8 @@ def setup_index_manager(elasticsearch, tika, sparql_connection_pool, config)
   search_configuration = config.select do |key|
     [:type_definitions, :default_index_settings,
      :persist_indexes, :eager_indexing_groups, :number_of_threads,
-     :batch_size, :max_batches, :attachment_path_base].include? key
+     :batch_size, :max_batches, :attachment_path_base,
+     :ignored_allowed_groups].include? key
   end
 
   MuSearch::IndexManager.new(