Improve substitution package handling and filter uses locally

Currently substitution packages handling can lead to a situation where
the number of performed permutation increase considerably and the
finding of a solution takes very long. In some cases it even times out
and results in a failed resolve state.

This now is improved by the following changes:

When a permutation is removed from the stack, every requirement is
checked if it is a substitution package and then resolve it to either
external or internal case first:
- if more candidates exits for this requirement a permutation is added
to account for this alternative solution
- now all other providers except the current one are dropped to make
this a single provider as the choice can not be reverted and therefore
other permutations in this path will be invalid
- then it must be checked if the package resolves to an external
provider in which case we need to drop the exported package capabilities
of this bundle for the given package
- after that all requirements are checked if they have only a single
provider (what can happen independently or as part of the resolving to
either external or internal) the use constraints are checked if they are
in conflict with a currently selected provider and dropping such invalid
choices
- finally after that we check the package space consistency (that is a
far more expensive check) what probably will result in more permutations
created and checked until we found the best result to use

Author: laeubi

bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/container/TestModuleContainer.java

@@ -26,7 +26,10 @@
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -1094,7 +1097,7 @@ public void testSubstitutableExports03() throws BundleException, IOException {
 
 		List<ModuleWire> providedWiresF = wiringF.getProvidedModuleWires(PackageNamespace.PACKAGE_NAMESPACE);
 		assertEquals("Wrong number of provided wires: " + providedWiresF, 0, providedWiresF.size());
-		assertSucessfulWith(report, 2, 1, 1, 1);
+		assertSucessfulWith(report, 2, 1, 1, 1); // TODO check why we now need MORE
 	}
 
 	@Test
@@ -3941,7 +3944,7 @@ public void testSubstitutionWithMoreThan2Providers() throws BundleException, IOE
 			modules.add(installDummyModule(manifest, manifest, container));
 		}
 		report = container.resolve(modules, true);
-		assertSucessfulWith(report, 15, 62, 47, 5);
+		assertSucessfulWith(report, 4, 75, 11, 1);
 	}
 
 	protected void assertSucessfulWith(ResolutionReport report, int maxProcessed, int maxSubstitution,
@@ -4370,14 +4373,14 @@ private static void assertWires(List<ModuleWire> required, List<ModuleWire>... p
 	@Test
 	public void testLocalUseConstraintViolations() throws Exception {
 		ResolutionReport result = resolveTestSet("set1");
-		assertSucessfulWith(result, 6, 20, 23, 6);
+		assertSucessfulWith(result, 3, 245, 7, 0);
 	}
 
 	@Test
 
 	public void testLocalUseConstraintViolations2() throws Exception {
 		ResolutionReport result = resolveTestSet("set2");
-		assertSucessfulWith(result, 3, 3, 1, 3);
+		assertSucessfulWith(result, 1, 2, 1, 0);
 	}
 
 	@Test
@@ -4394,17 +4397,24 @@ public void testSubstitutionPackageResolution() throws Exception {
 
 	@Test
 	public void testLargeSet() throws Exception {
-		ResolutionReport result = resolveModuleDatabaseDump("big", TimeUnit.MINUTES.toSeconds(5));
-		assertSucessfulWith(result, 1821, 29, 26359, 6736);
+		ResolutionReport result = resolveModuleDatabaseDump("big", 30, TimeUnit.SECONDS);
+		assertSucessfulWith(result, 1, 28, 1, 0);
 	}
 
 	@Test
 	public void testSdkSet() throws Exception {
-		ResolutionReport result = resolveModuleDatabaseDump("sdk202509", TimeUnit.MINUTES.toSeconds(1));
-		assertSucessfulWith(result, 9, 18, 1, 42);
+		ResolutionReport result = resolveModuleDatabaseDump("sdk202509", 10, TimeUnit.SECONDS);
+		assertSucessfulWith(result, 1, 17, 1, 0);
+	}
+
+	@Test
+	public void testBcSet() throws Exception {
+		ResolutionReport result = resolveModuleDatabaseDump("bc", 10, TimeUnit.MINUTES);
+		assertSucessfulWith(result, 100000, 100000, 100000, 100000);
 	}
 
-	private ResolutionReport resolveModuleDatabaseDump(String testSetName, long batchTimeoutSeconds) throws Exception {
+	private ResolutionReport resolveModuleDatabaseDump(String testSetName, long batchTimeout, TimeUnit timeoutUnit)
+			throws Exception {
 		URL entry = getBundle().getEntry("/test_files/containerTests/" + testSetName + ".state");
 		assertNotNull("can't find test set: " + testSetName, entry);
 		int maxThreads = Math.max(Runtime.getRuntime().availableProcessors() - 1, 1);
@@ -4421,7 +4431,7 @@ private ResolutionReport resolveModuleDatabaseDump(String testSetName, long batc
 		ScheduledExecutorService timeoutExecutor = new ScheduledThreadPoolExecutor(1);
 		Map<String, String> configuration = new HashMap<>();
 		configuration.put(EquinoxConfiguration.PROP_RESOLVER_BATCH_TIMEOUT,
-				Long.toString(TimeUnit.SECONDS.toMillis(batchTimeoutSeconds)));
+				Long.toString(timeoutUnit.toMillis(batchTimeout)));
 		DummyContainerAdaptor adaptor = new DummyContainerAdaptor(new DummyCollisionHook(false), configuration,
 				new DummyResolverHookFactory(), new DummyDebugOptions(Collections.emptyMap()));
 		adaptor.setResolverExecutor(executor);
@@ -4439,7 +4449,7 @@ private ResolutionReport resolveModuleDatabaseDump(String testSetName, long batc
 			}
 		}
 		AtomicBoolean timeout = new AtomicBoolean();
-		ScheduledFuture<?> watch = watchDog.schedule(() -> timeout.set(true), batchTimeoutSeconds, TimeUnit.SECONDS);
+		ScheduledFuture<?> watch = watchDog.schedule(() -> timeout.set(true), batchTimeout, timeoutUnit);
 		ResolutionReport report = container.resolve(container.getModules(), true);
 		watch.cancel(true);
 		assertFalse("Resolve operation timed out!", timeout.get());
@@ -4554,4 +4564,29 @@ private List<DummyModuleEvent> removeFirstListOfCommonEvents(List<DummyModuleEve
 		}
 		return result;
 	}
+
+	/**
+	 * The main class takes a module database and compress/anonymous it by replace
+	 * all locations with a running number
+	 * 
+	 * @param args
+	 * @throws Exception
+	 */
+	public static void main(String[] args) throws Exception {
+		DummyContainerAdaptor adaptor = new DummyContainerAdaptor(new DummyCollisionHook(false), null);
+		DummyModuleDatabase moduleDatabase = adaptor.getDatabase();
+		try (DataInputStream stream = new DataInputStream(new FileInputStream(args[0]))) {
+			moduleDatabase.load(stream);
+		}
+		List<Module> modules = adaptor.getContainer().getModules();
+		Field field = Module.class.getDeclaredField("location");
+		field.setAccessible(true);
+		int cnt = 0;
+		for (Module module : modules) {
+			field.set(module, Integer.toString(cnt++));
+		}
+		try (DataOutputStream out = new DataOutputStream(new FileOutputStream(args[0] + ".compressed"))) {
+			moduleDatabase.store(out, false);
+		}
+	}
 }

bundles/org.eclipse.osgi.tests/test_files/containerTests/bc.state

@@ -20,6 +20,7 @@
 
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import org.apache.felix.resolver.Candidates.FaultyResourcesReport;
@@ -57,7 +58,8 @@ public Backlog(ResolveSession session) {
     public Candidates getNext() {
         Candidates candidates;
         while ((candidates = session.getNextPermutation()) != null) {
-            ResolutionError substituteError = candidates.checkSubstitutes();
+            List<Candidates> alternatives = candidates.process(session.getLogger());
+            alternatives.forEach(alt -> session.addPermutation(PermutationType.SUBSTITUTE, alt));
             FaultyResourcesReport report = candidates.getFaultyResources(Collections.emptyMap());
             if (!report.isMissing() || session.isCancelled()) {
                 return candidates;