1.0.0: Initial commit

- buffer mode is now working; depends on gulp-buffer

Author: donpedro

.vscode/launch.json

@@ -38,37 +38,13 @@
             "request": "launch",
             "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
             "stopOnEntry": false,
-            "args": ["--gulpfile", "./debug/gulpfile.ts", "runTapCsvBuffer"],
+            "args": ["--gulpfile", "./debug/gulpfile.ts", "runMysqlAdapterBuffer"],
             "cwd": "${workspaceRoot}",
             "runtimeArgs": [
                 "--nolazy", "-r", "ts-node/register"
             ],
             "console": "internalConsole",
             "env": {"DEBUG_LEVEL":"debug"}
         },     
-        {
-            "name": "csvParseWithoutGulp",
-            "type": "node",
-            "request": "launch",
-            "program": "${workspaceRoot}/node_modules/gulp/bin/gulp.js",
-            "stopOnEntry": false,
-            "args": ["--gulpfile", "./debug/gulpfile.ts", "csvParseWithoutGulp"],
-            "cwd": "${workspaceRoot}",
-            "runtimeArgs": [
-                "--nolazy", "-r", "ts-node/register"
-            ],
-            "console": "internalConsole",
-            "env": {"DEBUG_LEVEL":"info"}
-        },
-        {
-            "name": "aws-doParse: debug TS w/o transpile using ts-node",
-            "type": "node",
-            "request": "launch",
-            "args": ["src/aws/dev-handler.ts"],
-            "runtimeArgs": ["-r", "ts-node/register"],
-            "cwd": "${workspaceRoot}",
-            "protocol": "inspector",
-            "internalConsoleOptions": "openOnSessionStart",
-        },
     ]
 }

README.md

@@ -1,6 +1,6 @@
-# gulp-etl-tap-mysql #
+# gulp-etl-mysql-adapter #
 
-This plugin connects to **MySQL** databases, running SQL queries and extracting the results to **gulp-etl** **Message Stream** files. It is a **gulp-etl** wrapper for [mysql](https://www.npmjs.com/package/mysql).
+This plugin connects to **MySQL** databases, running SQL queries and extracting the resulting rows to **gulp-etl** **Message Stream** files. It is a **gulp-etl** wrapper for [mysql](https://www.npmjs.com/package/mysql).
 
 This is a **[gulp-etl](https://gulpetl.com/)** tap, but unlike most of the other gulp-etl modules it is not a [gulp](https://gulpjs.com/) **[plugin](https://gulpjs.com/docs/en/getting-started/using-plugins)**; it is actually a **[vinyl adapter](https://gulpjs.com/docs/en/api/concepts#vinyl-adapters)**--it features a replacement for **[gulp.src()](https://gulpjs.com/docs/en/api/src)**. **gulp-etl** plugins and adapters work with [ndjson](http://ndjson.org/) data streams/files which we call **Message Streams** and which are compliant with the [Singer specification](https://github.com/singer-io/getting-started/blob/master/docs/SPEC.md#output). In the **gulp-etl** ecosystem, **taps** tap into an outside format or system (in this case, a MySQL database) and convert their contents/output to a Message Stream, and **targets** convert/output Message Streams to an outside format or system. In this way, these modules can be stacked to convert from one format or system to another, either directly or with tranformations or other parsing in between. Message Streams look like this:
 
@@ -16,7 +16,7 @@ This is a **[gulp-etl](https://gulpetl.com/)** tap, but unlike most of the other
 ### Usage
 **gulp** adapters take two parameters: a glob, which is used to locate file(s) in the file system, and an optional config object with settings specific to the adapter. For example: ```src('*.txt', {buffer:false})```
 
-Since this adapter doesn't pull from an existing file, the "glob" parameter is a virtual filename (with optional path info) which is assigned to the data file extracted from the server, e.g. ```mysqlData.ndjson```. And the configObj should look like this: 
+Since this adapter doesn't pull from an existing file, the "glob" parameter is a "pretend" filename (with optional path info) which is assigned to the data file extracted from the server, e.g. ```mysqlData.ndjson```. And the configObj should look like this: 
 ##### configObj / mysql-settings.json
 ```
 {
@@ -37,14 +37,14 @@ You *could* embed this information in your gulpfile, but we recommend storing it
 /* Run select query on the server and save the results in a local CSV file */
 
 var gulp = require('gulp')
-var tapMysql = require('gulp-etl-tap-mysql')
+var mysqlAdapter = require('gulp-etl-mysql-adapter')
 var targetCsv = require('gulp-etl-target-csv').targetCsv
 
 // contains secure info; store in parent folder of this project, outside of repo
 let configObj = require('../../mysql-settings.json')
 
 exports.default = function() {
-    return tapMysql.src('mysqlResults.ndjson',configObj)
+    return mysqlAdapter.src('mysqlResults.ndjson',configObj)
     .pipe(targetCsv({ columns:true }))
     .pipe(gulp.dest('output/'));    
 }

debug/gulpfile.ts

@@ -3,77 +3,54 @@ import * as tapMysql from '../src/plugin'
 
 import * as loglevel from 'loglevel'
 const log = loglevel.getLogger('gulpfile')
-log.setLevel((process.env.DEBUG_LEVEL || 'warn') as log.LogLevelDesc)
+log.setLevel((process.env.DEBUG_LEVEL || 'warn') as loglevel.LogLevelDesc)
 // if needed, you can control the plugin's logging level separately from 'gulpfile' logging above
 // const pluginLog = loglevel.getLogger(PLUGIN_NAME)
 // pluginLog.setLevel('debug')
 
-import * as rename from 'gulp-rename'
 const errorHandler = require('gulp-error-handle'); // handle all errors in one handler, but still stop the stream if there are errors
 
 const pkginfo = require('pkginfo')(module); // project package.json info into module.exports
 const PLUGIN_NAME = module.exports.name;
 
 import Vinyl = require('vinyl') 
 
-let gulpBufferMode = false;
 
-function switchToBuffer(callback: any) {
-  gulpBufferMode = true;
+// contains secure info; store in parent folder of this project, outside of repo
+let configObj = require('../../mysql-settings.json')
 
-  callback();
+/* 
+let configObj = 
+// mysql-settings.json should look like this: 
+{
+  "sql": "SELECT * FROM customers LIMIT 2;",
+  "connection": {
+    "host"     : "example.org",
+    "user"     : "bob",
+    "password" : "secret",
+    "database" : "schemaName"
+  }
 }
 
-export function csvParseWithoutGulp(callback: any) {
+*/
 
-  const parse = require('csv-parse')
+function switchToStream(callback: any) {
+  // buffer is true by default; we switch to stream mode by setting buffer to false here
+  configObj.buffer = false;
 
-  var parser = parse({delimiter: ',', columns:true});
-  
-  require('fs').createReadStream('../testdata/cars.csv').pipe(parser)
-  .on("data",(data:any)=>{
-    console.log(data)
-  });
-  
+  callback();
 }
 
-
-
-function testAdapter(callback: any) {
+function runMysqlAdapter(callback: any) {
   log.info('gulp task starting for ' + PLUGIN_NAME)
 
   try {
 
-    // contains secure info; store in parent folder of this project, outside of repo
-    let configObj = require('../../mysql-settings.json')
-
-    /* 
-    let configObj = 
-    // mysql-settings.json should look like this: 
-    {
-      "sql": "SELECT * FROM customers LIMIT 2;",
-      "connection": {
-        "host"     : "example.org",
-        "user"     : "bob",
-        "password" : "secret",
-        "database" : "schemaName"
-      }
-    }
-    
-    */
-
   return tapMysql.src('mysqlResults',configObj)
     .pipe(errorHandler(function(err:any) {
       log.error('Error: ' + err)
       callback(err)
     }))
-    .on('data', function (file:Vinyl) {
-      log.info('Starting processing on ' + file.basename)
-    })    
-    // .pipe(tapCsv({raw:true/*, info:true */}))
-    // .pipe(rename({
-    //   extname: ".ndjson",
-    // }))      
     .pipe(gulp.dest('../testdata/processed'))
     .on('data', function (file:Vinyl) {
       log.info('Finished processing on ' + file.basename)
@@ -90,5 +67,5 @@ function testAdapter(callback: any) {
 
 }
 
-exports.default = gulp.series(testAdapter)
-exports.runTapCsvBuffer = gulp.series(switchToBuffer, testAdapter)
+exports.default = gulp.series(switchToStream, runMysqlAdapter)
+exports.runMysqlAdapterBuffer = runMysqlAdapter