Merge pull request #324 from line-o/better-get

Better get

Author: duncdrum

commands/get.js

@@ -3,6 +3,7 @@ import { statSync, existsSync, mkdirSync } from 'node:fs'
 import { writeFile } from 'node:fs/promises'
 import { connect } from '@existdb/node-exist'
 import Bottleneck from 'bottleneck'
+import { getGlobMatcher } from '../utility/glob.js'
 
 /**
  * @typedef { import("@existdb/node-exist").NodeExist } NodeExist
@@ -30,15 +31,6 @@ import Bottleneck from 'bottleneck'
  * @prop {String[]} exclude filter items
  */
 
-const stringList = {
-  type: 'string',
-  array: true,
-  coerce: (values) =>
-    values.length === 1 && values[0].trim() === 'false'
-      ? ['**']
-      : values.reduce((values, value) => values.concat(value.split(',').map((value) => value.trim())), [])
-}
-
 const xmlBooleanOptionValue = new Map([
   ['true', 'yes'],
   ['yes', 'yes'],
@@ -60,7 +52,11 @@ const xmlBooleanSetting = {
     return xmlBooleanOptionValue.get(value)
   }
 }
-const serializationOptionNames = ['insert-final-newline', 'omit-xml-declaration', 'expand-xincludes']
+const serializationOptionNames = ['insert-final-newline', 'omit-xml-declaration', 'expand-xincludes', 'method']
+
+const htmlSerializationMethod = {
+  method: 'html'
+}
 
 const serializationDefaults = {
   'expand-xincludes': 'yes'
@@ -69,15 +65,26 @@ const serializationDefaults = {
   // "output.indent": "no",
   // "compression": "yes"
 }
+function getHtmlSerializationOptions (options) {
+  const serializationOptions = { ...serializationDefaults }
+  serializationOptionNames.forEach((o) => {
+    if (o in options) {
+      serializationOptions[o] = options[o]
+    }
+  })
+  Object.assign(serializationOptions, htmlSerializationMethod)
+  // console.log('Serialization options:', serializationOptions)
+  return serializationOptions
+}
 
 function getSerializationOptions (options) {
-  const serializationOptions = serializationDefaults
+  const serializationOptions = { ...serializationDefaults }
   serializationOptionNames.forEach((o) => {
     if (o in options) {
       serializationOptions[o] = options[o]
     }
   })
-  // console.log(serializationOptions)
+  // console.log('Serialization options:', serializationOptions)
   return serializationOptions
 }
 
@@ -91,12 +98,20 @@ function getSerializationOptions (options) {
  */
 async function downloadResource (db, options, resource, directory, collection, rename) {
   try {
-    const { verbose } = options
+    const { verbose, matchesExcludeGlob, matchesIncludeGlob, matchesHtmlGlob } = options
     let fileContents
     const path = collection ? posix.join(collection, resource.name) : resource.name
+    if (matchesExcludeGlob(resource) || !matchesIncludeGlob(resource)) {
+      if (verbose) {
+        console.log(`- skipping resource ${path}`)
+      }
+      return true
+    }
 
     if (resource.type === 'BinaryResource') {
       fileContents = await db.documents.readBinary(path)
+    } else if (matchesHtmlGlob(resource)) {
+      fileContents = await db.documents.read(path, getHtmlSerializationOptions(options))
     } else {
       fileContents = await db.documents.read(path, getSerializationOptions(options))
     }
@@ -202,16 +217,16 @@ async function getPathInfo (db, path) {
  */
 async function downloadCollectionOrResource (db, source, target, options) {
   // read parameters
-  //  const start = Date.now()
+  // const start = Date.now()
   const root = resolve(target)
 
   if (options.verbose) {
-    console.error('Downloading:', source, 'to', root)
-    if (options.include.length > 1 || options.include[0] !== '**') {
-      console.error('Include:\n', ...options.include, '\n')
+    console.error('Downloading', source, 'to', root)
+    if (options.include !== '**') {
+      console.error('Include', options.include)
     }
-    if (options.exclude.length) {
-      console.error('Exclude:\n', ...options.exclude, '\n')
+    if (options.exclude && options.exclude.length) {
+      console.error('Exclude', options.exclude)
     }
     console.error(`Downloading up to ${options.threads} resources at a time`)
     if (options['expand-xincludes'] === 'false') {
@@ -288,15 +303,15 @@ export function builder (yargs) {
   yargs
     .option('i', {
       alias: 'include',
-      describe: 'Include only files matching one or more of include patterns (comma separated)',
+      describe: 'Include only files matching the include globbing pattern',
       default: '**',
-      ...stringList
+      type: 'string'
     })
     .option('e', {
       alias: 'exclude',
-      describe: 'Exclude any file matching one or more of exclude patterns (comma separated)',
-      default: [],
-      ...stringList
+      describe: 'Exclude any file matching the exclude globbing pattern',
+      default: '',
+      type: 'string'
     })
     .option('x', {
       group: 'serialization',
@@ -316,6 +331,13 @@ export function builder (yargs) {
       describe: 'Force a final newline at the end of an XMLResource (requires eXist >=6.1.0)',
       ...xmlBooleanSetting
     })
+    .option('H', {
+      group: 'serialization',
+      alias: 'serialize-as-html',
+      describe: 'Serialize resources that match the globbing pattern as HTML',
+      default: '*.html',
+      type: 'string'
+    })
     .option('v', {
       alias: 'verbose',
       describe: 'Log every file and resource that was created',
@@ -342,7 +364,11 @@ export async function handler (argv) {
     return 0
   }
 
-  const { threads, mintime, source } = argv
+  const { threads, mintime, source, include, exclude, serializeAsHtml } = argv
+
+  const matchesIncludeGlob = getGlobMatcher(include)
+  const matchesExcludeGlob = getGlobMatcher(exclude)
+  const matchesHtmlGlob = getGlobMatcher(serializeAsHtml)
 
   if (typeof mintime !== 'number' || mintime < 0) {
     throw Error('Invalid value for option "mintime"; must be an integer equal or greater than zero.')
@@ -357,5 +383,5 @@ export async function handler (argv) {
   const version = await db.server.version()
   argv.version = version
 
-  return await downloadCollectionOrResource(db, source, target, argv)
+  return await downloadCollectionOrResource(db, source, target, { ...argv, matchesIncludeGlob, matchesExcludeGlob, matchesHtmlGlob })
 }

spec/fixtures/five.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Sample HTML Document</title>
+    </head>
+    <body>
+        <h1>Welcome to My Sample HTML Document</h1>
+        <p>This is a paragraph of text in the body of the HTML document.</p>
+        <x-custom-tag></x-custom-tag>
+    </body>
+    <script><![CDATA[ console.log(1<2); ]]></script>
+</html>

spec/tests/get.js

@@ -173,7 +173,7 @@ test('with test collection', async (t) => {
     st.plan(9)
     const verboseLines = stderr.split('\n')
     st.equal(verboseLines[0], 'Connecting to https://localhost:8443 as admin', verboseLines[0])
-    st.ok(verboseLines[1].startsWith('Downloading: /db/get-test to'), verboseLines[1])
+    st.ok(verboseLines[1].startsWith('Downloading /db/get-test to'), verboseLines[1])
     st.equal(verboseLines[2], 'Downloading up to 4 resources at a time', verboseLines[2])
     st.equal(verboseLines[3], '', verboseLines[3])
     st.equal(verboseLines.length, 4, 'all expected lines in verbose output')
@@ -222,5 +222,29 @@ test('with test collection', async (t) => {
     await run('rm', ['-rf', additionalTestDirectory])
   })
 
+  const globTestDirectory = 'glob-test'
+  t.test('Downloading only txt files matching glob but exclude a1.txt', async (st) => {
+    await run('mkdir', [globTestDirectory])
+    const { stderr, stdout } = await run(
+      'xst',
+      ['get', testCollection, globTestDirectory, '--include', '*.txt', '--exclude', 'a1.txt'],
+      asAdmin
+    )
+    if (stderr) {
+      st.fail(stderr)
+      return st.end()
+    }
+    st.plan(2)
+
+    st.notOk(stdout, 'no output')
+    st.deepEqual(
+      readdirSync(`${globTestDirectory}/${testCollectionName}`),
+      ['a.txt', 'a20.txt', 'empty-subcollection', 'subcollection'],
+      'only .txt files (except a1.txt) were downloaded'
+    )
+    await run('rm', ['-rf', globTestDirectory])
+    st.end()
+  })
+
   t.teardown(tearDown)
 })

spec/tests/list.js

@@ -257,6 +257,7 @@ test('with fixtures uploaded', async (t) => {
 /db/list-test/fixtures/binding.json
 /db/list-test/fixtures/broken-test-app.xar
 /db/list-test/fixtures/connection.xstrc
+/db/list-test/fixtures/five.html
 /db/list-test/fixtures/test-app.xar
 /db/list-test/fixtures/test-lib.xar
 /db/list-test/fixtures/test.xml
@@ -293,6 +294,7 @@ test('with fixtures uploaded', async (t) => {
 /db/list-test/fixtures/test-app.xar
 /db/list-test/fixtures/test-lib.xar
 /db/list-test/fixtures/broken-test-app.xar
+/db/list-test/fixtures/five.html
 /db/list-test/fixtures/test.xml
 /db/list-test/fixtures/web-no-rest.xml
 /db/list-test/tests

utility/glob.js

@@ -8,7 +8,8 @@ export function toRegExpPattern (glob) {
     .replace(/\\/g, '\\\\') // escape backslashes
     .replace(/\./g, '\\.') // make . literals
     .replace(/\?/g, '.') // transform ?
-    .replace(/\*/g, '.*?') // transform *
+    .replace(/\*\*/g, '.*?') // transform **
+    .replace(/\*/g, '[^/]*?') // transform *
 
   return `^${converted}$`
 }
@@ -19,6 +20,13 @@ export function toRegExpPattern (glob) {
  * @returns {(item:ListResultItem) => Boolean}
  */
 export function getGlobMatcher (glob) {
-  const regex = new RegExp(toRegExpPattern(glob), 'i')
+  if (glob == null || glob.length === 0 || glob === '') {
+    return () => false
+  }
+  if (glob === '**') {
+    return () => true
+  }
+  const pattern = toRegExpPattern(glob)
+  const regex = new RegExp(pattern, 'i')
   return (item) => regex.test(item.name)
 }