Add support for importing modules without extension, replace the module index name by file.

viferga · viferga · commit c09d4bddff5e · 2021-01-14T02:36:51.000-09:00
diff --git a/source/loader/include/loader/loader_naming.h b/source/loader/include/loader/loader_naming.h
@@ -29,7 +29,7 @@ extern "C" {
 
 #define LOADER_NAMING_PATH_SIZE			0x01FF
 #define LOADER_NAMING_NAME_SIZE			0xFF
-#define LOADER_NAMING_TAG_SIZE			0x12
+#define LOADER_NAMING_TAG_SIZE			0x40
 
 typedef char loader_naming_path[LOADER_NAMING_PATH_SIZE];
 typedef char loader_naming_name[LOADER_NAMING_NAME_SIZE];
diff --git a/source/loader/include/loader/loader_path.h b/source/loader/include/loader/loader_path.h
@@ -37,6 +37,8 @@ LOADER_API size_t loader_path_get_fullname(const loader_naming_path path, loader
 
 LOADER_API size_t loader_path_get_extension(const loader_naming_path path, loader_naming_tag extension);
 
+LOADER_API size_t loader_path_get_module_name(const loader_naming_path path, loader_naming_name name, const loader_naming_tag extension);
+
 LOADER_API size_t loader_path_get_path(const loader_naming_path path, size_t size, loader_naming_path absolute);
 
 LOADER_API size_t loader_path_get_relative(const loader_naming_path base, const loader_naming_path path, loader_naming_path relative);
diff --git a/source/loader/source/loader_impl.c b/source/loader/source/loader_impl.c
@@ -608,24 +608,8 @@ int loader_impl_load_from_file(loader_impl impl, const loader_naming_path paths[
 				return 1;
 			}
 
-			/* TODO: Refactor loader_path_get_name from path 0 (for avoiding collisions of scripts): */
-			/*
-			int loader_impl_load_from_file_ex(loader_impl impl, const loader_naming_path paths[], size_t size, const char * name, size_t length, void ** handle_ptr)
-			{
-				...
-				if (name == NULL)
-				{
-					loader_path_get_name(paths[0], module_name)
-				}
-				else
-				{
-					// TODO: Name must be a generated UUID to avoid collisions
-					strncpy(module_name, name, length);
-				}
-			}
-			*/
-
-			if (loader_path_get_name(paths[0], module_name) > 1 && loader_impl_get_handle(impl, module_name) != NULL)
+			/* TODO: Refactor loader_path_get_fullname from path 0 (for avoiding collisions of scripts): */
+			if (loader_path_get_fullname(paths[0], module_name) > 1 && loader_impl_get_handle(impl, module_name) != NULL)
 			{
 				log_write("metacall", LOG_LEVEL_ERROR, "Load from file handle failed, handle with name %s already loaded", module_name);
 
diff --git a/source/loader/source/loader_path.c b/source/loader/source/loader_path.c
@@ -34,7 +34,7 @@ size_t loader_path_get_name(const loader_naming_path path, loader_naming_name na
 	size_t i, count, last;
 
 	for (i = 0, count = 0, last = 0; path[i] != '\0' &&
-		i < LOADER_NAMING_PATH_SIZE /*&& count < LOADER_NAMING_NAME_SIZE*/; ++i)
+		i < LOADER_NAMING_PATH_SIZE && count < LOADER_NAMING_NAME_SIZE; ++i)
 	{
 		name[count++] = path[i];
 
@@ -44,17 +44,30 @@ size_t loader_path_get_name(const loader_naming_path path, loader_naming_name na
 		}
 		else if (path[i] == '.')
 		{
-			if (count > 0)
+			if (i > 0 && path[i - 1] == '.')
 			{
-				last = count - 1;
+				last = 0;
+				count = 0;
 			}
 			else
 			{
-				last = 0;
+				if (count > 0)
+				{
+					last = count - 1;
+				}
+				else
+				{
+					last = 0;
+				}
 			}
 		}
 	}
 
+	if (last == 0 && count > 1)
+	{
+		last = count;
+	}
+
 	name[last] = '\0';
 
 	return last + 1;
@@ -65,7 +78,7 @@ size_t loader_path_get_fullname(const loader_naming_path path, loader_naming_nam
 	size_t i, count;
 
 	for (i = 0, count = 0; path[i] != '\0' &&
-		i < LOADER_NAMING_PATH_SIZE /*&& count < LOADER_NAMING_NAME_SIZE*/; ++i)
+		i < LOADER_NAMING_PATH_SIZE && count < LOADER_NAMING_NAME_SIZE; ++i)
 	{
 		name[count++] = path[i];
 
@@ -85,11 +98,11 @@ size_t loader_path_get_extension(const loader_naming_path path, loader_naming_ta
 	size_t i, count;
 
 	for (i = 0, count = 0; path[i] != '\0' &&
-		i < LOADER_NAMING_PATH_SIZE /*&& count < LOADER_NAMING_TAG_SIZE*/; ++i)
+		i < LOADER_NAMING_PATH_SIZE; ++i)
 	{
 		extension[count++] = path[i];
 
-		if (path[i] == '.')
+		if (LOADER_PATH_SEPARATOR(path[i]) || path[i] == '.' || count == LOADER_NAMING_TAG_SIZE)
 		{
 			count = 0;
 		}
@@ -100,6 +113,23 @@ size_t loader_path_get_extension(const loader_naming_path path, loader_naming_ta
 	return count + 1;
 }
 
+size_t loader_path_get_module_name(const loader_naming_path path, loader_naming_name name, const loader_naming_tag extension)
+{
+	loader_naming_tag name_extension;
+
+	size_t i, size = loader_path_get_extension(path, name_extension);
+
+	for (i = 0; i < size && extension[i] != '\0'; ++i)
+	{
+		if (name_extension[i] != extension[i])
+		{
+			return loader_path_get_fullname(path, name);
+		}
+	}
+
+	return loader_path_get_name(path, name);
+}
+
 size_t loader_path_get_path(const loader_naming_path path, size_t size, loader_naming_path absolute)
 {
 	size_t i, last, path_size = size > LOADER_NAMING_PATH_SIZE ? LOADER_NAMING_PATH_SIZE : size;
diff --git a/source/loaders/py_loader/source/py_loader_impl.c b/source/loaders/py_loader/source/py_loader_impl.c
@@ -1821,7 +1821,7 @@ loader_handle py_loader_impl_load_from_file(loader_impl impl, const loader_namin
 	{
 		loader_naming_name module_name;
 
-		loader_path_get_name(paths[iterator], module_name);
+		loader_path_get_module_name(paths[iterator], module_name, "py");
 
 		py_handle->modules[iterator].name = PyUnicode_DecodeFSDefault(module_name);
 
diff --git a/source/loaders/rb_loader/source/rb_loader_impl.c b/source/loaders/rb_loader/source/rb_loader_impl.c
@@ -1083,7 +1083,7 @@ loader_handle rb_loader_impl_load_from_file(loader_impl impl, const loader_namin
 
 		loader_naming_name module_name;
 
-		loader_path_get_name(paths[iterator], module_name);
+		loader_path_get_module_name(paths[iterator], module_name, "rb");
 
 		rb_module = rb_loader_impl_load_from_file_module(impl, paths[iterator], module_name);
 
diff --git a/source/ports/node_port/index.js b/source/ports/node_port/index.js
@@ -97,12 +97,13 @@ const metacall_handle = (tag, name) => {
 	return ctx.find(script => script.name === name);
 };
 
-const metacall_require = (tag, name, id) => {
+const metacall_require = (tag, name) => {
 	// TODO: Inspect only the handle instead of the whole metacall namespace
 	/* return */ addon.metacall_load_from_file(tag, [ name ]);
 
+
 	const inspect = metacall_inspect();
-	const script = inspect[tag].find(script => script.name === id);
+	const script = inspect[tag].find(s => s.name === name);
 	const obj = {};
 
 	for (const func of script.scope.funcs) {
@@ -164,16 +165,15 @@ mod.prototype.require = function (name) {
 	if (index !== -1) {
 		/* If there is extension, load the module depending on the tag */
 		const extension = name.substr(index + 1);
-		const id = path.basename(name.substr(0, index));
 		const tag = tags[extension];
 
 		if (tag && tag !== 'node') {
 			/* Load with MetaCall if we found a tag and it is not NodeJS */
-			return metacall_require(tag, name, id);
+			return metacall_require(tag, name);
 		}
 	}
 
-	/* If there is no extension or the extension is not supported, load it with NodeJS require */
+	/* If there is no extension or the extension is not supported or it is 'node', load it with NodeJS require */
 	try {
 		/* Cache the port */
 		if (require.resolve(name) === path.resolve(__filename)) {
@@ -182,7 +182,22 @@ mod.prototype.require = function (name) {
 		/* Call to real NodeJS require */
 		return node_require.apply(this, [ name ]);
 	} catch (e) {
-		/* Print the exception and rethrow it */
+		/* If it is not a NodeJS module, try to guess the runtime */
+		const loaders = new Set(Object.values(tags));
+
+		/* Mock and node loaders are not included */
+		loaders.delete('mock');
+		loaders.delete('node');
+
+		for (let it = loaders.values(), tag = null; tag = it.next().value; ) {
+			try {
+				return metacall_require(tag, name);
+			} catch (_) {
+				/* Keep trying with the next loader */
+			}
+		}
+
+		/* It could not be loaded */
 		console.log(e);
 		throw e;
 	}
diff --git a/source/ports/node_port/package-lock.json b/source/ports/node_port/package-lock.json
diff --git a/source/ports/node_port/test/index.js b/source/ports/node_port/test/index.js
@@ -46,16 +46,16 @@ describe('metacall', () => {
 		it('metacall_load_from_file (py)', () => {
 			assert.strictEqual(metacall_load_from_file('py', [ 'helloworld.py' ] ), undefined);
 
-			const script = metacall_handle('py', 'helloworld');
+			const script = metacall_handle('py', 'helloworld.py');
 			assert.notStrictEqual(script, undefined);
-			assert.strictEqual(script.name, 'helloworld');
+			assert.strictEqual(script.name, 'helloworld.py');
 		});
 		it('metacall_load_from_file (rb)', () => {
 			assert.strictEqual(metacall_load_from_file('rb', [ 'ducktype.rb' ]), undefined);
 
-			const script = metacall_handle('rb', 'ducktype');
+			const script = metacall_handle('rb', 'ducktype.rb');
 			assert.notStrictEqual(script, undefined);
-			assert.strictEqual(script.name, 'ducktype');
+			assert.strictEqual(script.name, 'ducktype.rb');
 		});
 		it('metacall_load_from_memory (py)', () => {
 			assert.strictEqual(metacall_load_from_memory('py', 'def py_memory():\n\treturn 4;\n'), undefined);
@@ -95,6 +95,18 @@ describe('metacall', () => {
 			assert.deepStrictEqual(example.return_array(), [1, 2, 3]);
 			assert.deepStrictEqual(example.return_same_array([1, 2, 3]), [1, 2, 3]);
 		});
+		it('require (py module)', () => {
+			// This code loads directly a module without extension from Python
+			const { loads } = require('json');
+			assert.notStrictEqual(loads, undefined);
+			assert.deepStrictEqual(loads('["foo", "bar"]'), ['foo', 'bar']);
+		});
+		it('require (py submodule)', () => {
+			// This code loads directly a module without extension from Python
+			const { py_encode_basestring_ascii } = require('json.encoder');
+			assert.notStrictEqual(py_encode_basestring_ascii, undefined);
+			assert.strictEqual(py_encode_basestring_ascii('asd'), '"asd"');
+		});
 		it('require (rb)', () => {
 			const cache = require('cache.rb');
 			assert.notStrictEqual(cache, undefined);
diff --git a/source/tests/loader_path_test/source/loader_path_test.cpp b/source/tests/loader_path_test/source/loader_path_test.cpp
diff --git a/source/tests/metacall_node_port_test/CMakeLists.txt b/source/tests/metacall_node_port_test/CMakeLists.txt
diff --git a/source/tests/metacall_python_port_https_test/CMakeLists.txt b/source/tests/metacall_python_port_https_test/CMakeLists.txt

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ size_t loader_path_get_name(const loader_naming_path path, loader_naming_name na`
`34`	`34`	`size_t i, count, last;`
`35`	`35`
`36`	`36`	`for (i = 0, count = 0, last = 0; path[i] != '\0' &&`
`37`		`- i < LOADER_NAMING_PATH_SIZE /&& count < LOADER_NAMING_NAME_SIZE/; ++i)`
	`37`	`+ i < LOADER_NAMING_PATH_SIZE && count < LOADER_NAMING_NAME_SIZE; ++i)`
`38`	`38`	`{`
`39`	`39`	`name[count++] = path[i];`
`40`	`40`
`@@ -44,17 +44,30 @@ size_t loader_path_get_name(const loader_naming_path path, loader_naming_name na`
`44`	`44`	`}`
`45`	`45`	`else if (path[i] == '.')`
`46`	`46`	`{`
`47`		`- if (count > 0)`
	`47`	`+ if (i > 0 && path[i - 1] == '.')`
`48`	`48`	`{`
`49`		`- last = count - 1;`
	`49`	`+ last = 0;`
	`50`	`+ count = 0;`
`50`	`51`	`}`
`51`	`52`	`else`
`52`	`53`	`{`
`53`		`- last = 0;`
	`54`	`+ if (count > 0)`
	`55`	`+ {`
	`56`	`+ last = count - 1;`
	`57`	`+ }`
	`58`	`+ else`
	`59`	`+ {`
	`60`	`+ last = 0;`
	`61`	`+ }`
`54`	`62`	`}`
`55`	`63`	`}`
`56`	`64`	`}`
`57`	`65`
	`66`	`+ if (last == 0 && count > 1)`
	`67`	`+ {`
	`68`	`+ last = count;`
	`69`	`+ }`
	`70`	`+`
`58`	`71`	`name[last] = '\0';`
`59`	`72`
`60`	`73`	`return last + 1;`
`@@ -65,7 +78,7 @@ size_t loader_path_get_fullname(const loader_naming_path path, loader_naming_nam`
`65`	`78`	`size_t i, count;`
`66`	`79`
`67`	`80`	`for (i = 0, count = 0; path[i] != '\0' &&`
`68`		`- i < LOADER_NAMING_PATH_SIZE /&& count < LOADER_NAMING_NAME_SIZE/; ++i)`
	`81`	`+ i < LOADER_NAMING_PATH_SIZE && count < LOADER_NAMING_NAME_SIZE; ++i)`
`69`	`82`	`{`
`70`	`83`	`name[count++] = path[i];`
`71`	`84`
`@@ -85,11 +98,11 @@ size_t loader_path_get_extension(const loader_naming_path path, loader_naming_ta`
`85`	`98`	`size_t i, count;`
`86`	`99`
`87`	`100`	`for (i = 0, count = 0; path[i] != '\0' &&`
`88`		`- i < LOADER_NAMING_PATH_SIZE /&& count < LOADER_NAMING_TAG_SIZE/; ++i)`
	`101`	`+ i < LOADER_NAMING_PATH_SIZE; ++i)`
`89`	`102`	`{`
`90`	`103`	`extension[count++] = path[i];`
`91`	`104`
`92`		`- if (path[i] == '.')`
	`105`	`+ if (LOADER_PATH_SEPARATOR(path[i]) \|\| path[i] == '.' \|\| count == LOADER_NAMING_TAG_SIZE)`
`93`	`106`	`{`
`94`	`107`	`count = 0;`
`95`	`108`	`}`
`@@ -100,6 +113,23 @@ size_t loader_path_get_extension(const loader_naming_path path, loader_naming_ta`
`100`	`113`	`return count + 1;`
`101`	`114`	`}`
`102`	`115`
	`116`	`+size_t loader_path_get_module_name(const loader_naming_path path, loader_naming_name name, const loader_naming_tag extension)`
	`117`	`+{`
	`118`	`+ loader_naming_tag name_extension;`
	`119`	`+`
	`120`	`+ size_t i, size = loader_path_get_extension(path, name_extension);`
	`121`	`+`
	`122`	`+ for (i = 0; i < size && extension[i] != '\0'; ++i)`
	`123`	`+ {`
	`124`	`+ if (name_extension[i] != extension[i])`
	`125`	`+ {`
	`126`	`+ return loader_path_get_fullname(path, name);`
	`127`	`+ }`
	`128`	`+ }`
	`129`	`+`
	`130`	`+ return loader_path_get_name(path, name);`
	`131`	`+}`
	`132`	`+`
`103`	`133`	`size_t loader_path_get_path(const loader_naming_path path, size_t size, loader_naming_path absolute)`
`104`	`134`	`{`
`105`	`135`	`size_t i, last, path_size = size > LOADER_NAMING_PATH_SIZE ? LOADER_NAMING_PATH_SIZE : size;`
Original file line number	Diff line number	Diff line change
`@@ -1821,7 +1821,7 @@ loader_handle py_loader_impl_load_from_file(loader_impl impl, const loader_namin`
`1821`	`1821`	`{`
`1822`	`1822`	`loader_naming_name module_name;`
`1823`	`1823`
`1824`		`- loader_path_get_name(paths[iterator], module_name);`
	`1824`	`+ loader_path_get_module_name(paths[iterator], module_name, "py");`
`1825`	`1825`
`1826`	`1826`	`py_handle->modules[iterator].name = PyUnicode_DecodeFSDefault(module_name);`
`1827`	`1827`