From c1a665b4ef19885d02c55ff0d13486ada1ee5411 Mon Sep 17 00:00:00 2001 From: Frank Weigel Date: Fri, 25 Sep 2020 15:43:28 +0200 Subject: [PATCH 1/6] [WIP] ZIP Archive Adapter (not complete) --- lib/adapters/ZipArchive.js | 195 +++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 lib/adapters/ZipArchive.js diff --git a/lib/adapters/ZipArchive.js b/lib/adapters/ZipArchive.js new file mode 100644 index 00000000..8795dd71 --- /dev/null +++ b/lib/adapters/ZipArchive.js @@ -0,0 +1,195 @@ +const log = require("@ui5/logger").getLogger("resources:adapters:ZIPArchive"); +const micromatch = require("micromatch"); +const Resource = require("../Resource"); +const AbstractAdapter = require("./AbstractAdapter"); +const StreamZip = require('node-stream-zip'); + +/** + * Virtual resource Adapter + * + * @public + * @memberof module:@ui5/fs.adapters + * @augments module:@ui5/fs.adapters.AbstractAdapter + */ +class ZIPArchive extends AbstractAdapter { + /** + * The constructor. + * + * @public + * @param {object} parameters Parameters + * @param {string} parameters.virBasePath Virtual base path + * @param {string[]} [parameters.excludes] List of glob patterns to exclude + */ + constructor({virBasePath, project, fsArchive, excludes}) { + super({virBasePath, project, excludes}); + this._zipPath = fsArchive; + this._zipLoaded = null; + this._virFiles = {}; // map full of files + this._virDirs = {}; // map full of directories + } + + async _prepare() { + + if ( this._zipLoaded == null ) { + + this._zipLoaded = new Promise((resolve, reject) => { + + const zip = this._zip = new StreamZip({ + file: this._zipPath, + storeEntries: true + }); + + // Handle errors + zip.on('error', err => { + console.error(err); + reject(err); + }); + zip.on('ready', () => { + console.log('Entries read: ' + zip.entriesCount); + for (const entry of Object.values(zip.entries())) { + const desc = entry.isDirectory ? 'directory' : `${entry.size} bytes`; + if ( entry.name.startsWith("META-INF/resources/") + || entry.name.startsWith("META-INF/test-resources/") ) { + const virPath = entry.name.slice("META-INF".length); + if ( entry.isDirectory ) { + this._virDirs[virPath] = new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return true; + } + }, + path: virPath + }); + } else { + this._virFiles[virPath] = new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return false; + } + }, + path: virPath, + createStream: async () => { + return new Promise((resolve, reject) => { + zip.stream("META-INF" + virPath, (err, stm) => { + if ( err ) { + reject(err); + } else { + resolve(stm); + } + }); + }); + } + }); + } + // console.log(`Entry ${virPath}: ${desc}`); + } else { + // console.log(`Entry ignored: ${entry.name}`); + } + } + resolve(); + }); + }); + + } + + return this._zipLoaded; + } + + /** + * Locate resources by glob. + * + * @private + * @param {Array} patterns array of glob patterns + * @param {object} [options={}] glob options + * @param {boolean} [options.nodir=true] Do not match directories + * @param {module:@ui5/fs.tracing.Trace} trace Trace instance + * @returns {Promise} Promise resolving to list of resources + */ + async _runGlob(patterns, options = {nodir: true}, trace) { + await this._prepare(); + if (patterns[0] === "" && !options.nodir) { // Match virtual root directory + return [ + new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return true; + } + }, + path: this._virBasePath.slice(0, -1) + }) + ]; + } + + const filePaths = Object.keys(this._virFiles); + const matchedFilePaths = micromatch(filePaths, patterns, { + dot: true + }); + let matchedResources = matchedFilePaths.map((virPath) => { + return this._virFiles[virPath]; + }); + + if (!options.nodir) { + const dirPaths = Object.keys(this._virDirs); + const matchedDirs = micromatch(dirPaths, patterns, { + dot: true + }); + matchedResources = matchedResources.concat(matchedDirs.map((virPath) => { + return this._virDirs[virPath]; + })); + } + + return matchedResources; + } + + /** + * Locates resources by path. + * + * @private + * @param {string} virPath Virtual path + * @param {object} options Options + * @param {module:@ui5/fs.tracing.Trace} trace Trace instance + * @returns {Promise} Promise resolving to a single resource + */ + async _byPath(virPath, options, trace) { + await this._prepare(); + + if (this.isPathExcluded(virPath)) { + return Promise.resolve(null); + } + return new Promise((resolve, reject) => { + if (!virPath.startsWith(this._virBasePath) && virPath !== this._virBaseDir) { + // Neither starts with basePath, nor equals baseDirectory + resolve(null); + return; + } + + const relPath = virPath.substr(this._virBasePath.length); + trace.pathCall(); + + const resource = this._virFiles[relPath]; + + if (!resource || (options.nodir && resource.getStatInfo().isDirectory())) { + resolve(null); + } else { + resolve(resource); + } + }); + } + +} + +module.exports = ZIPArchive; + +const adapter = new ZIPArchive({ + virBasePath: "", + project: {}, + fsArchive: process.argv[2], + excludes: [] +}); + +adapter._runGlob("/resources/sap/base/**/*").then((result) => { + console.log(result); +}); From 9fa5028bf98ecf1b0755663c273f26497442a55a Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Mon, 28 Sep 2020 19:19:57 +0200 Subject: [PATCH 2/6] [INTERNAL] package.json: Add missing dependency 'node-stream-zip' --- package-lock.json | 5 +++++ package.json | 1 + 2 files changed, 6 insertions(+) diff --git a/package-lock.json b/package-lock.json index a31e341e..0579a674 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3462,6 +3462,11 @@ "process-on-spawn": "^1.0.0" } }, + "node-stream-zip": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.11.3.tgz", + "integrity": "sha512-GY+9LxkQuIT3O7K8BTdHVGKFcBYBy2vAVcTBtkKpu+OlBef/NSb6VuIWSyLiVDfmLMkggHeRJZN0F3W0GWU/uw==" + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", diff --git a/package.json b/package.json index 22d156b9..3aa75c33 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "micromatch": "^4.0.2", "minimatch": "^3.0.3", "mock-require": "^3.0.3", + "node-stream-zip": "^1.11.3", "pretty-hrtime": "^1.0.3", "random-int": "^2.0.1", "slash": "^3.0.0" From f6f939539ee5cd76d8df8680d615c954fe3545a5 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Mon, 28 Sep 2020 19:20:20 +0200 Subject: [PATCH 3/6] [INTERNAL] Fix ESLint issues --- lib/adapters/ZipArchive.js | 89 ++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/lib/adapters/ZipArchive.js b/lib/adapters/ZipArchive.js index 8795dd71..9375c531 100644 --- a/lib/adapters/ZipArchive.js +++ b/lib/adapters/ZipArchive.js @@ -2,7 +2,7 @@ const log = require("@ui5/logger").getLogger("resources:adapters:ZIPArchive"); const micromatch = require("micromatch"); const Resource = require("../Resource"); const AbstractAdapter = require("./AbstractAdapter"); -const StreamZip = require('node-stream-zip'); +const StreamZip = require("node-stream-zip"); /** * Virtual resource Adapter @@ -27,62 +27,59 @@ class ZIPArchive extends AbstractAdapter { this._virFiles = {}; // map full of files this._virDirs = {}; // map full of directories } - - async _prepare() { - - if ( this._zipLoaded == null ) { + async _prepare() { + if ( this._zipLoaded == null ) { this._zipLoaded = new Promise((resolve, reject) => { - const zip = this._zip = new StreamZip({ file: this._zipPath, storeEntries: true }); // Handle errors - zip.on('error', err => { + zip.on("error", err => { console.error(err); reject(err); }); - zip.on('ready', () => { - console.log('Entries read: ' + zip.entriesCount); + zip.on("ready", () => { + console.log("Entries read: " + zip.entriesCount); for (const entry of Object.values(zip.entries())) { - const desc = entry.isDirectory ? 'directory' : `${entry.size} bytes`; - if ( entry.name.startsWith("META-INF/resources/") - || entry.name.startsWith("META-INF/test-resources/") ) { - const virPath = entry.name.slice("META-INF".length); - if ( entry.isDirectory ) { - this._virDirs[virPath] = new Resource({ - project: this.project, - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return true; - } - }, - path: virPath - }); - } else { - this._virFiles[virPath] = new Resource({ - project: this.project, - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return false; - } - }, - path: virPath, - createStream: async () => { - return new Promise((resolve, reject) => { - zip.stream("META-INF" + virPath, (err, stm) => { - if ( err ) { - reject(err); - } else { - resolve(stm); - } - }); - }); + const desc = entry.isDirectory ? "directory" : `${entry.size} bytes`; + if ( entry.name.startsWith("META-INF/resources/") || + entry.name.startsWith("META-INF/test-resources/") ) { + const virPath = entry.name.slice("META-INF".length); + if ( entry.isDirectory ) { + this._virDirs[virPath] = new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return true; } - }); - } + }, + path: virPath + }); + } else { + this._virFiles[virPath] = new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return false; + } + }, + path: virPath, + createStream: async () => { + return new Promise((resolve, reject) => { + zip.stream("META-INF" + virPath, (err, stm) => { + if ( err ) { + reject(err); + } else { + resolve(stm); + } + }); + }); + } + }); + } // console.log(`Entry ${virPath}: ${desc}`); } else { // console.log(`Entry ignored: ${entry.name}`); @@ -91,9 +88,8 @@ class ZIPArchive extends AbstractAdapter { resolve(); }); }); - } - + return this._zipLoaded; } @@ -178,7 +174,6 @@ class ZIPArchive extends AbstractAdapter { } }); } - } module.exports = ZIPArchive; From 6a8594be6a34d79ba48bfff0d6f37ae0689ea9c2 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Mon, 28 Sep 2020 20:02:27 +0200 Subject: [PATCH 4/6] [INTERNAL] index.js: Expose ZipArchive module --- index.js | 6 +++++- lib/adapters/ZipArchive.js | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index b1b1d920..b4dade5b 100644 --- a/index.js +++ b/index.js @@ -20,7 +20,11 @@ module.exports = { /** * @type {typeof import('./lib/adapters/Memory')} */ - Memory: "./lib/adapters/Memory" + Memory: "./lib/adapters/Memory", + /** + * @type {typeof import('./lib/adapters/ZipArchive')} + */ + ZipArchive: "./lib/adapters/ZipArchive" }, /** * @type {typeof import('./lib/AbstractReader')} diff --git a/lib/adapters/ZipArchive.js b/lib/adapters/ZipArchive.js index 9375c531..139f4d5e 100644 --- a/lib/adapters/ZipArchive.js +++ b/lib/adapters/ZipArchive.js @@ -177,14 +177,3 @@ class ZIPArchive extends AbstractAdapter { } module.exports = ZIPArchive; - -const adapter = new ZIPArchive({ - virBasePath: "", - project: {}, - fsArchive: process.argv[2], - excludes: [] -}); - -adapter._runGlob("/resources/sap/base/**/*").then((result) => { - console.log(result); -}); From a2c78f1bc48ebb12c09e8fff44dac4246ffa0e61 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Wed, 30 Sep 2020 22:56:19 +0200 Subject: [PATCH 5/6] [INTERNAL] ZipArchive: Fix path mappings (hacky) --- lib/adapters/ZipArchive.js | 99 +++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/lib/adapters/ZipArchive.js b/lib/adapters/ZipArchive.js index 139f4d5e..6f2466f8 100644 --- a/lib/adapters/ZipArchive.js +++ b/lib/adapters/ZipArchive.js @@ -20,9 +20,10 @@ class ZIPArchive extends AbstractAdapter { * @param {string} parameters.virBasePath Virtual base path * @param {string[]} [parameters.excludes] List of glob patterns to exclude */ - constructor({virBasePath, project, fsArchive, excludes}) { + constructor({virBasePath, project, fsArchive, archivePath = "/", excludes}) { super({virBasePath, project, excludes}); this._zipPath = fsArchive; + this._archivePath = archivePath; this._zipLoaded = null; this._virFiles = {}; // map full of files this._virDirs = {}; // map full of directories @@ -37,7 +38,7 @@ class ZIPArchive extends AbstractAdapter { }); // Handle errors - zip.on("error", err => { + zip.on("error", (err) => { console.error(err); reject(err); }); @@ -83,6 +84,18 @@ class ZIPArchive extends AbstractAdapter { // console.log(`Entry ${virPath}: ${desc}`); } else { // console.log(`Entry ignored: ${entry.name}`); + let virPath = "/" + entry.name; + if (virPath.startsWith(this._archivePath)) { + // console.log("orig path: " + virPath); + // console.log("archive path: " + this._archivePath); + virPath = virPath.replace(this._archivePath, ""); + // console.log("new path: " + virPath); + if ( entry.isDirectory ) { + this._virDirs[virPath] = true; + } else { + this._virFiles[virPath] = true; + } + } } } resolve(); @@ -123,9 +136,28 @@ class ZIPArchive extends AbstractAdapter { const matchedFilePaths = micromatch(filePaths, patterns, { dot: true }); - let matchedResources = matchedFilePaths.map((virPath) => { - return this._virFiles[virPath]; - }); + // console.log(matchedFilePaths); + let matchedResources = await Promise.all(matchedFilePaths.map(async (virPath) => { + const stream = await new Promise((resolve, reject) => { + this._zip.stream(this._archivePath.substring(1) + virPath, (err, stm) => { + if ( err ) { + reject(err); + } else { + resolve(stm); + } + }); + }); + return new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return false; + } + }, + path: this._virBasePath + virPath, + stream: stream + }); + })); if (!options.nodir) { const dirPaths = Object.keys(this._virDirs); @@ -133,10 +165,20 @@ class ZIPArchive extends AbstractAdapter { dot: true }); matchedResources = matchedResources.concat(matchedDirs.map((virPath) => { - return this._virDirs[virPath]; + return new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return true; + } + }, + path: this._virBasePath + virPath + }); })); } + // console.log(matchedResources); + return matchedResources; } @@ -153,25 +195,36 @@ class ZIPArchive extends AbstractAdapter { await this._prepare(); if (this.isPathExcluded(virPath)) { - return Promise.resolve(null); + return null; + } + if (!virPath.startsWith(this._virBasePath) && virPath !== this._virBaseDir) { + // Neither starts with basePath, nor equals baseDirectory + return null; } - return new Promise((resolve, reject) => { - if (!virPath.startsWith(this._virBasePath) && virPath !== this._virBaseDir) { - // Neither starts with basePath, nor equals baseDirectory - resolve(null); - return; - } - - const relPath = virPath.substr(this._virBasePath.length); - trace.pathCall(); - - const resource = this._virFiles[relPath]; - if (!resource || (options.nodir && resource.getStatInfo().isDirectory())) { - resolve(null); - } else { - resolve(resource); - } + const relPath = virPath.substr(this._virBasePath.length); + trace.pathCall(); + if (!this._virFiles[relPath]) { + return null; + } + const stream = await new Promise((resolve, reject) => { + this._zip.stream(this._archivePath.substring(1) + relPath, (err, stm) => { + if ( err ) { + reject(err); + } else { + resolve(stm); + } + }); + }); + return new Resource({ + project: this.project, + statInfo: { // TODO: make closer to fs stat info + isDirectory: function() { + return false; + } + }, + path: virPath, + stream: stream }); } } From 9cd63688d7d76a6b101d677385432560113159af Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Wed, 30 Sep 2020 22:57:50 +0200 Subject: [PATCH 6/6] [INTERNAL] resourceFactory: Use ZipArchive adapter if ZIP file is detected (also hacky) --- lib/resourceFactory.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/resourceFactory.js b/lib/resourceFactory.js index 66767add..535524fb 100644 --- a/lib/resourceFactory.js +++ b/lib/resourceFactory.js @@ -2,6 +2,7 @@ const log = require("@ui5/logger").getLogger("resources:resourceFactory"); const path = require("path"); const FsAdapter = require("./adapters/FileSystem"); const MemAdapter = require("./adapters/Memory"); +const ZipAdapter = require("./adapters/ZipArchive"); const ReaderCollection = require("./ReaderCollection"); const ReaderCollectionPrioritized = require("./ReaderCollectionPrioritized"); const DuplexCollection = require("./DuplexCollection"); @@ -232,6 +233,19 @@ const resourceFactory = { * @returns {module:@ui5/fs.adapters.FileSystem|module:@ui5/fs.adapters.Memory} File System- or Virtual Adapter */ createAdapter({fsBasePath, virBasePath, project, excludes}) { + const dotZipIdx = fsBasePath.indexOf(".zip/"); + if (dotZipIdx !== -1) { + // console.log(virBasePath); + // console.log(fsBasePath); + // console.log(fsBasePath.substring(0, dotZipIdx + 4)); + // console.log(fsBasePath.substring(dotZipIdx + 4)); + // console.log(""); + return new ZipAdapter({ + virBasePath, + fsArchive: fsBasePath.substring(0, dotZipIdx + 4), + archivePath: fsBasePath.substring(dotZipIdx + 4) + "/" + }); + } if (fsBasePath) { return new FsAdapter({fsBasePath, virBasePath, project, excludes}); } else {