Skip to content

Lazy loading model for module imports #4481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: develop
Choose a base branch
from
Open

Lazy loading model for module imports #4481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4c0119b
[feature] Allow lazy loading of buid-in modules.
rhubner Jun 30, 2022
4766e97
[feature] Enable lazy loading for unit tests, fix bugs.
rhubner Jul 6, 2022
c9fcba2
[feature] Lazy loading modules - fix tests.
rhubner Jul 12, 2022
99824e4
[bugfix] - Load module only to appropriate context.
rhubner Jul 12, 2022
a4fc7ac
[ignore] Fix formatting
adamretter Aug 3, 2022
d434d6c
[refactor] Switch terms from 'lazily'/'always' to 'lazy'/'eager'
adamretter Oct 5, 2022
99fb337
[refactor] Set the default to eagerly load modules to preserve backwa…
adamretter Oct 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 27 additions & 14 deletions exist-core/src/main/java/org/exist/util/Configuration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,11 +393,13 @@ private void configureXQuery( Element xquery ) throws DatabaseConfigurationExcep
config.put( PerformanceStats.CONFIG_PROPERTY_TRACE, trace );

// built-in-modules
final Map<String, Class<?>> classMap = new HashMap<>();
final Map<String, String> knownMappings = new HashMap<>();
final Map<String, Class<?>> eagerModuleClassMap = new HashMap<>();
final Map<String, Class<?>> lazyModuleClassMap = new HashMap<>();
final Map<String, String> knownMappings = new HashMap<>();
final Map<String, Map<String, List<? extends Object>>> moduleParameters = new HashMap<>();
loadModuleClasses(xquery, classMap, knownMappings, moduleParameters);
config.put( XQueryContext.PROPERTY_BUILT_IN_MODULES, classMap);
loadModuleClasses(xquery, eagerModuleClassMap, lazyModuleClassMap, knownMappings, moduleParameters);
config.put( XQueryContext.PROPERTY_BUILT_IN_MODULES, eagerModuleClassMap);
config.put( XQueryContext.PROPERTY_BUILT_IN_LAZY_MODULES, lazyModuleClassMap);
config.put( XQueryContext.PROPERTY_STATIC_MODULE_MAP, knownMappings);
config.put( XQueryContext.PROPERTY_MODULE_PARAMETERS, moduleParameters);
}
Expand All @@ -407,14 +409,15 @@ private void configureXQuery( Element xquery ) throws DatabaseConfigurationExcep
* that the specified module class exists and is a subclass of {@link org.exist.xquery.Module}.
*
* @param xquery configuration root
* @param modulesClassMap map containing all classes of modules
* @param modulesSourceMap map containing all source uris to external resources
* @param eagerModuleClassMap map containing all classes of modules which are always loaded to XQueryContext
* @param lazyModuleClassMap map containing all classes of modules which are lazy loaded to XQueryContext only if they are explicitly imported
* @param modulesSourceMap map containing all source uris to external resources
*
* @throws DatabaseConfigurationException
*/
private void loadModuleClasses( Element xquery, Map<String, Class<?>> modulesClassMap, Map<String, String> modulesSourceMap, Map<String, Map<String, List<? extends Object>>> moduleParameters) throws DatabaseConfigurationException {
private void loadModuleClasses( Element xquery, Map<String, Class<?>> eagerModuleClassMap, Map<String, Class<?>> lazyModuleClassMap, Map<String, String> modulesSourceMap, Map<String, Map<String, List<? extends Object>>> moduleParameters) throws DatabaseConfigurationException {
// add the standard function module
modulesClassMap.put(Namespaces.XPATH_FUNCTIONS_NS, org.exist.xquery.functions.fn.FnModule.class);
eagerModuleClassMap.put(Namespaces.XPATH_FUNCTIONS_NS, org.exist.xquery.functions.fn.FnModule.class);

// add other modules specified in configuration
final NodeList builtins = xquery.getElementsByTagName(XQUERY_BUILTIN_MODULES_CONFIGURATION_MODULES_ELEMENT_NAME);
Expand All @@ -436,6 +439,8 @@ private void loadModuleClasses( Element xquery, Map<String, Class<?>> modulesCla
final String uri = elem.getAttribute(XQueryContext.BUILT_IN_MODULE_URI_ATTRIBUTE);
final String clazz = elem.getAttribute(XQueryContext.BUILT_IN_MODULE_CLASS_ATTRIBUTE);
final String source = elem.getAttribute(XQueryContext.BUILT_IN_MODULE_SOURCE_ATTRIBUTE);
final String load = elem.getAttribute(XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE);


// uri attribute is the identifier and is always required
if(uri == null) {
Expand All @@ -447,6 +452,10 @@ private void loadModuleClasses( Element xquery, Map<String, Class<?>> modulesCla
throw(new DatabaseConfigurationException("element 'module' requires either an attribute " + "'class' or 'src'" ));
}

if (load != null && !(XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_EAGER.equals(load) || XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_LAZY.equals(load))) {
throw new DatabaseConfigurationException(String.format("parameter '{}' on module can only contain the value: '{}' or '{}'", XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE, XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_EAGER, XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_LAZY));
}

if(source != null) {
// Store src attribute info

Expand All @@ -462,13 +471,17 @@ private void loadModuleClasses( Element xquery, Map<String, Class<?>> modulesCla
// Get class of module
final Class<?> moduleClass = lookupModuleClass(uri, clazz);

// Store class if thw module class actually exists
if( moduleClass != null) {
modulesClassMap.put(uri, moduleClass);
}
// Store class if the module class actually exists
if (moduleClass != null) {
if (XQueryContext.BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_LAZY.equals(load)) {
lazyModuleClassMap.put(uri, moduleClass);
} else {
eagerModuleClassMap.put(uri, moduleClass);
}

if(LOG.isDebugEnabled()) {
LOG.debug("Configured module '{}' implemented in '{}'", uri, clazz);
if (LOG.isDebugEnabled()) {
LOG.debug("Configured module '{}' implemented in '{}'", uri, clazz);
}
}
}

Expand Down
24 changes: 21 additions & 3 deletions exist-core/src/main/java/org/exist/xquery/XQueryContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,10 @@ public class XQueryContext implements BinaryValueManager, Context {
public static final String BUILT_IN_MODULE_URI_ATTRIBUTE = "uri";
public static final String BUILT_IN_MODULE_CLASS_ATTRIBUTE = "class";
public static final String BUILT_IN_MODULE_SOURCE_ATTRIBUTE = "src";
public static final String BUILT_IN_MODULE_LOAD_ATTRIBUTE = "load";
public static final String BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_EAGER = "eager";
public static final String BUILT_IN_MODULE_LOAD_ATTRIBUTE_VALUE_LAZY = "lazy";


public static final String PROPERTY_XQUERY_BACKWARD_COMPATIBLE = "xquery.backwardCompatible";
public static final String PROPERTY_ENABLE_QUERY_REWRITING = "xquery.enable-query-rewriting";
Expand All @@ -134,7 +138,8 @@ public class XQueryContext implements BinaryValueManager, Context {
public static final String PROPERTY_ENFORCE_INDEX_USE = "xquery.enforce-index-use";

//TODO : move elsewhere ?
public static final String PROPERTY_BUILT_IN_MODULES = "xquery.modules";
public static final String PROPERTY_BUILT_IN_MODULES = "xquery.modules";
public static final String PROPERTY_BUILT_IN_LAZY_MODULES = "xquery.modules.lazy";
public static final String PROPERTY_STATIC_MODULE_MAP = "xquery.modules.static";
public static final String PROPERTY_MODULE_PARAMETERS = "xquery.modules.parameters";

Expand Down Expand Up @@ -1760,8 +1765,8 @@ Module initBuiltInModule(final String namespaceURI, final String moduleClass) {
declareNamespace(module.getDefaultPrefix(), module.getNamespaceURI());
}

modules.compute(module.getNamespaceURI(), addToMapValueArray(module));
allModules.compute(module.getNamespaceURI(), addToMapValueArray(module));
this.addModule(module.getNamespaceURI(), module);
this.addRootModule(module.getNamespaceURI(), module);

if (module instanceof InternalModule) {
((InternalModule) module).prepare(this);
Expand Down Expand Up @@ -2512,6 +2517,19 @@ public Module[] importModule(@Nullable String namespaceURI, @Nullable String pre

if (namespaceURI != null) {
modules = getRootModules(namespaceURI);

//Lazy load modules
if (modules == null) {
final Class<Module> moduleClass = (Class<Module>) ((Map) getConfiguration()
.getProperty(PROPERTY_BUILT_IN_LAZY_MODULES))
.get(namespaceURI);
if(moduleClass != null) {
instantiateModule(namespaceURI, moduleClass ,
(Map<String, Map<String, List<? extends Object>>>) getConfiguration().getProperty(PROPERTY_MODULE_PARAMETERS));

modules = getRootModules(namespaceURI);
}
}
}

if (isNotEmpty(modules)) {
Expand Down
2 changes: 2 additions & 0 deletions exist-core/src/main/resources/org/exist/test/runner/xquery-test-runner.xq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ xquery version "3.1";
import module namespace test = "http://exist-db.org/xquery/xqsuite"
at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";

import module namespace inspect = "http://exist-db.org/xquery/inspection";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change is a clear hint that this constitutes a breaking change.
Library and main modules that use one of the functions in the inspect namespace will no longer function until they are imported.


declare variable $test-module-uri as xs:anyURI external;

(: hooks for sending external notifications about test events :)
Expand Down
10 changes: 8 additions & 2 deletions exist-core/src/test/java/org/exist/validation/CollectionConfigurationValidationModeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,18 @@ public class CollectionConfigurationValidationModeTest {

@AfterClass
public static void tearDownClass() throws Exception {
existEmbeddedServer.executeQuery("validation:clear-grammar-cache()");
existEmbeddedServer.executeQuery(
"import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:clear-grammar-cache()"
);
}

@Before
public void setUp() throws Exception {
existEmbeddedServer.executeQuery("validation:clear-grammar-cache()");
existEmbeddedServer.executeQuery(
"import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:clear-grammar-cache()"
);
}

private void createCollection(final String collection) throws XMLDBException {
Expand Down
1 change: 1 addition & 0 deletions exist-core/src/test/java/org/exist/xquery/ForwardReferenceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ public void test1() throws EXistException, PermissionDeniedException, IOExceptio
"\n" +
"import module namespace xqsuite = \"http://exist-db.org/xquery/xqsuite\"\n" +
" at \"resource:org/exist/xquery/lib/xqsuite/xqsuite.xql\";\n" +
"import module namespace inspect = \"http://exist-db.org/xquery/inspection\";" +
"\n" +
"xqsuite:suite((\n" +
" inspect:module-functions(xs:anyURI(\"xmldb:exist://" + TEST_PAGES_MODULE_URI + "\"))\n" +
Expand Down
2 changes: 1 addition & 1 deletion exist-core/src/test/java/org/exist/xquery/XQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -922,7 +922,7 @@ public void namespaceWithTransform() throws XMLDBException {

String query =
"xquery version \"1.0\";\n" +
"declare namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
"import module namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interesting change.....

"declare variable $xml {\n" +
" <node>text</node>\n" +
"};\n" +
Expand Down
1 change: 1 addition & 0 deletions exist-core/src/test/java/org/exist/xquery/functions/transform/TransformTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ public class TransformTest {
"xquery version \"3.0\";\n" +
"\n" +
"(:Read document with xsl:for-each and look for key in the dictionary document :)\n" +
"import module namespace transform = \"http://exist-db.org/xquery/transform\";\n" +
"declare variable $xsltPath as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "';\n" +
"declare variable $listOpsFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath()+ "/listOpsErr.xml';\n" +
"declare variable $inputFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "/inputListOps.xml';\n" +
Expand Down
12 changes: 8 additions & 4 deletions exist-core/src/test/java/org/exist/xquery/functions/validate/AdditionalJingXsdRngTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public class AdditionalJingXsdRngTest {

@Test
public void testValidateXSDwithJing() throws XMLDBException {
final String query = "let $v := <doc>\n" +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"let $v := <doc>\n" +
"\t<title>Title</title>\n" +
"\t<p>Some paragraph.</p>\n" +
" </doc>\n" +
Expand Down Expand Up @@ -71,7 +72,8 @@ public void testValidateXSDwithJing() throws XMLDBException {

@Test
public void testValidateXSDwithJing_invalid() throws XMLDBException {
final String query = "let $v := <doc>\n" +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"let $v := <doc>\n" +
"\t<title1>Title</title1>\n" +
"\t<p>Some paragraph.</p>\n" +
" </doc>\n" +
Expand Down Expand Up @@ -99,7 +101,8 @@ public void testValidateXSDwithJing_invalid() throws XMLDBException {

@Test
public void testValidateRNGwithJing() throws XPathException, XMLDBException {
final String query = "let $v := <doc>\n" +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"let $v := <doc>\n" +
"\t<title>Title</title>\n" +
"\t<p>Some paragraph.</p>\n" +
" </doc>\n" +
Expand Down Expand Up @@ -139,7 +142,8 @@ public void testValidateRNGwithJing() throws XPathException, XMLDBException {

@Test
public void testValidateRNGwithJing_invalid() throws XMLDBException {
final String query = "let $v := <doc>\n" +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"let $v := <doc>\n" +
"\t<title1>Title</title1>\n" +
"\t<p>Some paragraph.</p>\n" +
" </doc>\n" +
Expand Down
22 changes: 15 additions & 7 deletions exist-core/src/test/java/org/exist/xquery/functions/validate/JaxpDtdCatalogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ public static void prepareResources() throws Exception {

@Before
public void clearGrammarCache() throws XMLDBException {
final ResourceSet results = existEmbeddedServer.executeQuery("validation:clear-grammar-cache()");
final ResourceSet results = existEmbeddedServer.executeQuery(
"import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:clear-grammar-cache()");
results.getResource(0).getContent();
}

Expand All @@ -127,31 +129,35 @@ public void clearGrammarCache() throws XMLDBException {
*/
@Test
public void dtd_stored_catalog_valid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/valid-dtd.xml'), false()," +
"doc('/db/parse/catalog.xml') )";
executeAndEvaluate(query,"valid");
}

@Test
public void dtd_stored_catalog_invalid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/invalid-dtd.xml'), false()," +
"doc('/db/parse/catalog.xml') )";
executeAndEvaluate(query,"invalid");
}

@Test
public void dtd_anyURI_catalog_valid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/valid-dtd.xml'), false()," +
"xs:anyURI('/db/parse/catalog.xml') )";
executeAndEvaluate(query,"valid");
}

@Test
public void dtd_anyURI_catalog_invalid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/invalid-dtd.xml'), false()," +
"xs:anyURI('/db/parse/catalog.xml') )";
executeAndEvaluate(query,"invalid");
Expand All @@ -165,15 +171,17 @@ public void dtd_anyURI_catalog_invalid() throws XMLDBException, SAXException, Xp
*/
@Test
public void dtd_searched_valid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/valid-dtd.xml'), false()," +
"xs:anyURI('/db/parse/') )";
executeAndEvaluate(query,"valid");
}

@Test
public void dtd_searched_invalid() throws XMLDBException, SAXException, XpathException, IOException {
final String query = "validation:jaxp-report( " +
final String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:jaxp-report( " +
"xs:anyURI('/db/parse/instance/invalid-dtd.xml'), false()," +
"xs:anyURI('/db/parse/') )";
executeAndEvaluate(query,"invalid");
Expand Down
10 changes: 7 additions & 3 deletions exist-core/src/test/java/org/exist/xquery/functions/validate/JaxpParseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,17 +90,21 @@ public static void prepareResources() throws Exception {

@Before
public void clearGrammarCache() throws XMLDBException {
final ResourceSet results = existEmbeddedServer.executeQuery("validation:clear-grammar-cache()");
final ResourceSet results = existEmbeddedServer.executeQuery(
"import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:clear-grammar-cache()");
results.getResource(0).getContent();
}

@Test
public void parse_and_fill_defaults() throws XMLDBException, IOException, SAXException {
String query = "validation:pre-parse-grammar(xs:anyURI('/db/parse_validate/defaultValue.xsd'))";
String query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"validation:pre-parse-grammar(xs:anyURI('/db/parse_validate/defaultValue.xsd'))";
String result = execute(query);
assertEquals(result, "defaultTest");

query = "declare option exist:serialize 'indent=no'; " +
query = "import module namespace validation = \"http://exist-db.org/xquery/validation\";\n" +
"declare option exist:serialize 'indent=no'; " +
"validation:jaxp-parse(xs:anyURI('/db/parse_validate/defaultValue.xml'), true(), ())";
result = execute(query);

Expand Down
Loading