Create initial simple test vinyl adapter, verifying that it works to load a local file as a gulp Vinyl file and writes with standard gulp.dest()

Author: donpedro

.vscode/launch.json

@@ -46,6 +46,20 @@
             "console": "internalConsole",
             "env": {"DEBUG_LEVEL":"debug"}
         },
+        {        
+            "name": "TestAdapter Debug",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
+            "stopOnEntry": false,
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "testAdapter"],
+            "cwd": "${workspaceRoot}",
+            "runtimeArgs": [
+                "--nolazy", "-r", "ts-node/register"
+            ],
+            "console": "internalConsole",
+            "env": {"DEBUG_LEVEL":"debug"}
+        },        
         {
             "name": "csvParseWithoutGulp",
             "type": "node",

debug/gulpfile.ts

@@ -1,5 +1,5 @@
 let gulp = require('gulp')
-import { tapCsv } from '../src/plugin'
+import { src as mysqlSrc } from '../src/plugin'
 
 import * as loglevel from 'loglevel'
 const log = loglevel.getLogger('gulpfile')
@@ -35,7 +35,7 @@ function runTapCsv(callback: any) {
     .on('data', function (file:Vinyl) {
       log.info('Starting processing on ' + file.basename)
     })    
-    .pipe(tapCsv({raw:true/*, info:true */}))
+    // .pipe(tapCsv({raw:true/*, info:true */}))
     .pipe(rename({
       extname: ".ndjson",
     }))      
@@ -63,5 +63,41 @@ export function csvParseWithoutGulp(callback: any) {
 
 }
 
+
+
+export function testAdapter(callback: any) {
+  log.info('gulp task starting for ' + PLUGIN_NAME)
+
+  try {
+
+
+  return mysqlSrc('../testdata/cars.csv',{buffer:gulpBufferMode})
+    .pipe(errorHandler(function(err:any) {
+      log.error('Error: ' + err)
+      callback(err)
+    }))
+    .on('data', function (file:Vinyl) {
+      log.info('Starting processing on ' + file.basename)
+    })    
+    // .pipe(tapCsv({raw:true/*, info:true */}))
+    // .pipe(rename({
+    //   extname: ".ndjson",
+    // }))      
+    .pipe(gulp.dest('../testdata/processed'))
+    .on('data', function (file:Vinyl) {
+      log.info('Finished processing on ' + file.basename)
+    })    
+    .on('end', function () {
+      log.info('gulp task complete')
+      callback()
+    })
+
+  }
+  catch (err) {
+    log.error(err)
+  }
+
+}
+
 exports.default = gulp.series(runTapCsv)
 exports.runTapCsvBuffer = gulp.series(switchToBuffer, runTapCsv)

package-lock.json

@@ -21,7 +21,6 @@
   ],
   "homepage": "https://gulpetl.com",
   "dependencies": {
-    "csv-parse": "^4.4.1",
     "from2": "^2.3.0",
     "gulp-error-handle": "^1.0.1",
     "loglevel": "^1.6.1",

package.json

@@ -7,16 +7,40 @@ import * as loglevel from 'loglevel'
 const log = loglevel.getLogger(PLUGIN_NAME) // get a logger instance based on the project name
 log.setLevel((process.env.DEBUG_LEVEL || 'warn') as log.LogLevelDesc)
 
-const parse = require('csv-parse')
+const from2 = require('from2');
+import * as path from 'path'
+
 
 /** wrap incoming recordObject in a Singer RECORD Message object*/
 function createRecord(recordObject:Object, streamName: string) : any {
   return {type:"RECORD", stream:streamName, record:recordObject}
 }
 
+// dead-simple stream-mode-only gulp vinyl adapter: loads a file as a stream and wraps it in a vinyl file, then returns vinyl file in a readable stream
+export function src(filePath:string, optionsIgnoredForNow:any) {
+  let fileStream
+  let vinylFile
+  try {
+    fileStream = require('fs').createReadStream(filePath)
+    // let asdf = (undefined as any).wontWork // create an error for testing
+    vinylFile = new Vinyl({
+      base: path.dirname(filePath),    
+      path:filePath,
+      contents:fileStream
+    });
+  }
+  catch (err) {
+    throw new PluginError(PLUGIN_NAME, err);
+  }
+
+  return from2.obj([vinylFile])
+}
+
+
 /* This is a gulp-etl plugin. It is compliant with best practices for Gulp plugins (see
 https://github.com/gulpjs/gulp/blob/master/docs/writing-a-plugin/guidelines.md#what-does-a-good-plugin-look-like ),
 and like all gulp-etl plugins it accepts a configObj as its first parameter */
+/*
 export function tapCsv(configObj: any) {
   if (!configObj) configObj = {}
   if (!configObj.columns) configObj.columns = true // we don't allow false for columns; it results in arrays instead of objects for each record
@@ -133,4 +157,6 @@ export function tapCsv(configObj: any) {
   })
 
   return strm
-}
+}
+
+*/

src/plugin.ts

@@ -0,0 +1,5 @@
+carModel,price,color
+"Audi",10000,"blue"
+"BMW",15000,"red"
+"Mercedes",20000,"yellow"
+"Porsche",30000,"green"