Skip to content

excessive logging overhead #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

excessive logging overhead #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
75fe8c7
excessive console logging for batch insert
sirvio Mar 29, 2017
880dc07
node(red) logging configuration
sirvio Mar 29, 2017
35caf47
0.2.24
sirvio Mar 29, 2017
14dd5c8
target is console
sirvio Mar 29, 2017
814746b
trace goes to stderr
sirvio Mar 29, 2017
3d8c62c
0.2.25
sirvio Mar 29, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 24 additions & 23 deletions 98-sqldb-dashdb-cf.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ module.exports = function(RED) {
var connString = "DRIVER={DB2};DATABASE=" + dashDBconfig.db + ";UID=" + dashDBconfig.username + ";PWD=" + dashDBconfig.password + ";HOSTNAME=" + dashDBconfig.hostname + ";port=" + dashDBconfig.port;

try {
console.log("dashDB output node: Opening db connection...");
console.info("dashDB output node: Opening db connection...");
db.openSync(connString);
console.log("dashDB output node: Connection open");
console.info("dashDB output node: Connection open");
}
catch (e) {
node.error(e.message);
Expand All @@ -111,26 +111,26 @@ module.exports = function(RED) {
columnListWithQuotes += "\"" + columnList[i] + "\"";

}
console.log("dashDB output node: columnList: " + columnListWithQuotes);
//console.trace("dashDB output node: columnList: " + columnListWithQuotes);

node.on("close", function() {
console.log("dashDB: Closing db connection...");
console.info("dashDB: Closing db connection...");
db.closeSync();
console.log("dashDB output node: Connection closed");
console.info("dashDB output node: Connection closed");
});

var questionMarks = genQuestionMarks(columnList);

var insertStatement = "insert into \""+node.table+"\" (" + columnListWithQuotes + ") values("+questionMarks+")";
console.log("dashDB output node: Preparing insert statement: " + insertStatement);
//console.trace("dashDB output node: Preparing insert statement: " + insertStatement);

node.on("input", function(msg) {
db.prepare(insertStatement, function (err, stmt) {
if (err) {
node.error("dashDB output node: " + err);
}
else {
console.log("dashDB output node: Prepare successful");
console.info("dashDB output node: Prepare successful");
processInput(node,msg,db,stmt,columnList,"dashDB");
}
});
Expand All @@ -143,7 +143,7 @@ function getColumns (node,db,table,service) {
//Remove the schema, if it exists, hopefully the table name is unique - need to improve this
var removeSchema = table.split(".");
if (removeSchema.length > 1) { table = removeSchema[1]; }
console.log(service+": Fetching column names for table " + table + "...");
console.info(service+": Fetching column names for table " + table + "...");
var sysibmColumns;
try {
sysibmColumns = db.querySync("select name from sysibm.syscolumns where tbname = '"+table+"' and generated = ''");
Expand All @@ -165,19 +165,19 @@ function getColumns (node,db,table,service) {
}

function processInput (node,msg,db,stmt,columnList,service) {
console.log(service+": Input event received");
console.log(service+": columnList: "+columnList);
//console.trace(service+": Input event received");
//console.trace(service+": columnList: "+columnList);
var valueToInsert;
var batchInsert;
var valueList;
var insertIterations;
if (msg.payload instanceof Array) {
console.log(service+": msg.payload is an array, need to iterate...");
//console.trace(service+": msg.payload is an array, need to iterate...");
batchInsert = true;
insertIterations = msg.payload.length;
}
else {
console.log(service+": msg.payload not an array");
//console.trace(service+": msg.payload not an array");
batchInsert = false;
insertIterations = 1;
}
Expand All @@ -199,13 +199,14 @@ function processInput (node,msg,db,stmt,columnList,service) {
}
else {node.error(service+": Column "+columnList[j]+" is missing from the payload or has an undefined value"); return;}
}
console.log("Values to execute:");
console.log(valueList);
// Excessive logging is unnecessary overhead for ETL type batch jobs
//console.trace("Values to execute:");
//console.trace(valueList);
stmt.execute(valueList, function (err, result) {
if (err) {
node.error(service+": Insert failed: "+err);
} else {
console.log(service+": Insert successful!");
//console.trace(service+": Insert successful!");
result.closeSync();
}
});
Expand Down Expand Up @@ -305,18 +306,18 @@ function dashDBQueryNode(n) {
var connString = "DRIVER={DB2};DATABASE=" + dashDBconfig.db + ";UID=" + dashDBconfig.username + ";PWD=" + dashDBconfig.password + ";HOSTNAME=" + dashDBconfig.hostname + ";port=" + dashDBconfig.port;

try {
console.log("dashDB query node: Opening db connection...");
console.info("dashDB query node: Opening db connection...");
db.openSync(connString);
console.log("dashDB query node: Connection open");
console.info("dashDB query node: Connection open");
}
catch (e) {
node.error(e.message);
}

node.on("close", function() {
console.log("dashDB query node: Closing db connection...");
console.info("dashDB query node: Closing db connection...");
db.closeSync();
console.log("dashDB query node: Connection closed");
console.info("dashDB query node: Connection closed");
});


Expand All @@ -334,9 +335,9 @@ function dashDBQueryNode(n) {
var parameterValues=[];
if (params != "" && params != null) {
var path = pathToArray(params.toString());
console.log("Input node: pathToArray: " + path);
console.info("Input node: pathToArray: " + path);
parameterValues = extractValues(msg, path);
console.log("Input node: parameterValues: " + parameterValues);
//console.trace("Input node: parameterValues: " + parameterValues);
}
db.query(queryToUse,parameterValues,function (err, rows, moreResultSets) {
queryresult = null;
Expand All @@ -345,8 +346,8 @@ function dashDBQueryNode(n) {
msg.error = err;
} else {
msg.error = null;
console.log("Fetching rows: " + rows);
console.log("value 1: " + JSON.stringify(rows[0]));
//console.trace("Fetching rows: " + rows);
//console.trace("value 1: " + JSON.stringify(rows[0]));
if (rows.length == 1) {queryresult = rows[0];}
else {
queryresult = [];
Expand Down
61 changes: 33 additions & 28 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,35 @@
{
"name" : "node-red-nodes-cf-sqldb-dashdb",
"version" : "0.2.23",
"description" : "Node-RED nodes to access SQLDB and dashDB services on Bluemix",
"dependencies" : {
"cfenv": "x",
"ibm_db" : "1.0.x"
},
"repository" : {
"type" : "git",
"url" : "https://github.com/smchamberlin/node-red-nodes-cf-sqldb-dashdb.git"
},
"license" : "Apache",
"keywords" : [ "node-red", "sqldb", "dashdb", "bluemix" ],
"node-red" : {
"nodes" : {
"sqldb-dashdb-cf" : "98-sqldb-dashdb-cf.js"
}
},
"author" : {
"name" : "Steven Chamberlin",
"email": "[email protected]"
},
"contributors" : [
{
"name": "Nicholas Vargas",
"email": "[email protected]"
}
]
"name": "node-red-nodes-cf-sqldb-dashdb",
"version": "0.2.25",
"description": "Node-RED nodes to access SQLDB and dashDB services on Bluemix",
"dependencies": {
"cfenv": "x",
"ibm_db": "1.0.x"
},
"repository": {
"type": "git",
"url": "https://github.com/smchamberlin/node-red-nodes-cf-sqldb-dashdb.git"
},
"license": "Apache",
"keywords": [
"node-red",
"sqldb",
"dashdb",
"bluemix"
],
"node-red": {
"nodes": {
"sqldb-dashdb-cf": "98-sqldb-dashdb-cf.js"
}
},
"author": {
"name": "Steven Chamberlin",
"email": "[email protected]"
},
"contributors": [
{
"name": "Nicholas Vargas",
"email": "[email protected]"
}
]
}