Skip to content

Commit 6c3be79

Browse files
georgeajitgeorgeajit
georgeajit
authored and
georgeajit
committed
#621 - New writeAll functional tests
1 parent c77fd6a commit 6c3be79

File tree

1 file changed

+329
-0
lines changed

1 file changed

+329
-0
lines changed

‎test-complete/nodejs-data-movement.js‎

Lines changed: 329 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,329 @@
1+
/*
2+
* Copyright (c) 2021 MarkLogic Corporation
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
var should = require('should');
17+
18+
var fs = require('fs');
19+
const path = require('path')
20+
21+
var marklogic = require('../');
22+
var q = marklogic.queryBuilder;
23+
24+
var testconfig = require('../etc/test-config-qa.js');
25+
26+
const stream = require('stream');
27+
28+
var transformuris = [];
29+
var batchuris = [];
30+
31+
var db = marklogic.createDatabaseClient(testconfig.restReaderConnection);
32+
var dbWriter = marklogic.createDatabaseClient(testconfig.restWriterConnection);
33+
var dbAdmin = marklogic.createDatabaseClient(testconfig.restAdminConnection);
34+
35+
describe('datamovement tests', function(){
36+
describe('writeAll-tests', function(){
37+
var transformName = 'dmsdk-timestamp';
38+
before(function (done) {
39+
var transformPath = __dirname + '/data/dmsdk/transform/write-transform.sjs';
40+
41+
this.timeout(10000);
42+
dbAdmin.config.transforms.write({
43+
name:transformName, format:'javascript', source:fs.createReadStream(transformPath)
44+
}).result(function(response){
45+
db.config.transforms.list().result(function(response) {
46+
//console.log('Installed transforms: ');
47+
var installedTransforms = JSON.stringify(response, null, 2);
48+
//console.log(installedTransforms);
49+
expect(installedTransforms).to.have.string(transformName);
50+
}, function(error) {
51+
console.log(JSON.stringify(error, null, 2));
52+
}); done();
53+
}, done);
54+
}); // end of before
55+
56+
// write docs of different formats
57+
it('writeAll different formats', function(done){
58+
const selectFiles = [];
59+
60+
var multiDocreadable = new stream.Readable({objectMode: true});
61+
// Handle only .json, .xml and .txt files
62+
var allowedFiles = ['.json', '.xml', '.txt'];
63+
64+
const dirPath = path.join(__dirname, '/data/dmsdk/');
65+
66+
function includeFile(fName) {
67+
var fileExt = path.extname(fName);
68+
return allowedFiles.includes(fileExt);
69+
}
70+
71+
var files = fs.readdirSync(dirPath).filter(includeFile);
72+
files.forEach(file => {
73+
let fileStat = fs.statSync(dirPath + '/' + file).isDirectory();
74+
if(!fileStat) {
75+
selectFiles.push(file);
76+
}
77+
});
78+
//console.log(selectFiles);
79+
for (var file of selectFiles) {
80+
var fileTobeRead = dirPath + file;
81+
var data = fs.readFileSync(fileTobeRead, {encoding:'utf8'});
82+
var findCType = path.extname(fileTobeRead);
83+
var jsonFN1 = {
84+
uri: file,
85+
contentType: findCType === '.json' ? 'application/json': findCType === '.xml' ? 'application/xml' : 'application/text',
86+
collections: ['qatestsText'],
87+
content: data
88+
};
89+
//console.log('Contents ' + jsonFN1.content);
90+
multiDocreadable.push(jsonFN1);
91+
}
92+
multiDocreadable.push(null);
93+
94+
multiDocreadable.pipe(dbWriter.documents.writeAll({
95+
onBatchSuccess: ((progressSoFar, documents) => {
96+
try {
97+
//console.log('Progress ' + progressSoFar.docsWrittenSuccessfully);
98+
progressSoFar.docsWrittenSuccessfully.should.be.greaterThanOrEqual(1);
99+
progressSoFar.docsFailedToBeWritten.should.be.equal(0);
100+
progressSoFar.timeElapsed.should.be.greaterThanOrEqual(0);
101+
documents.length.should.equal(5);
102+
} catch(err){
103+
done(err);
104+
}
105+
}),
106+
onCompletion: ((summary) => {
107+
summary.docsWrittenSuccessfully.should.be.equal(5);
108+
summary.docsFailedToBeWritten.should.be.equal(0);
109+
summary.timeElapsed.should.be.greaterThanOrEqual(0);
110+
})
111+
})); // End of pipe to writeAll
112+
setTimeout(()=>{done();}, 3000);
113+
}); // End of test case
114+
115+
// write binary doc
116+
it('writeAll binary format', function(done){
117+
var multiDocreadable = new stream.Readable({objectMode: true});
118+
var filename = __dirname+'/data/121-GIF-Image-GIF-gif_sample1.gif';
119+
var fileContents ='';
120+
121+
var data = fs.readFileSync(filename);
122+
123+
const jsonFN1 = {
124+
uri: '/data/121-GIF-Image-GIF-gif_sample1.gif',
125+
contentType: 'image/gif',
126+
collections: ['qatestsBinary'],
127+
directory:'/dmsdktest/',
128+
content: data
129+
};
130+
multiDocreadable.push(jsonFN1);
131+
multiDocreadable.push(null);
132+
//console.log('Contents ' + jsonFN1.content);
133+
multiDocreadable.pipe(dbWriter.documents.writeAll({
134+
onBatchSuccess: ((progressSoFar, documents) => {
135+
try {
136+
progressSoFar.docsWrittenSuccessfully.should.be.greaterThanOrEqual(1);
137+
progressSoFar.docsFailedToBeWritten.should.be.equal(0);
138+
progressSoFar.timeElapsed.should.be.greaterThanOrEqual(0);
139+
} catch(err){
140+
done(err);
141+
}
142+
}),
143+
onCompletion: ((summary) => {
144+
summary.docsWrittenSuccessfully.should.be.equal(1);
145+
summary.docsFailedToBeWritten.should.be.equal(0);
146+
summary.timeElapsed.should.be.greaterThanOrEqual(0);
147+
})
148+
149+
}));
150+
setTimeout(()=>{done();}, 3000);
151+
});
152+
153+
// write docs with SJS transforms
154+
it('writeAll with sjs transform', function(done){
155+
156+
var multiDocreadable = new stream.Readable({objectMode: true});
157+
158+
for(let i=0; i<10; i++) {
159+
const temp = {
160+
uri: '/qa/test/dmsdk/transforms/'+i+'.json',
161+
contentType: 'application/json',
162+
collections: ['qatestsTransform'],
163+
content: {['key '+i]:'value '+i}
164+
};
165+
multiDocreadable.push(temp);
166+
transformuris.push(temp.uri);
167+
}
168+
multiDocreadable.push(null);
169+
170+
multiDocreadable.pipe(dbWriter.documents.writeAll({
171+
transform: [transformName, {title: 'new title', myInt: 2, myBool: true}],
172+
onBatchSuccess: ((progressSoFar, documents) => {
173+
try {
174+
//console.log('Progress ' + progressSoFar.docsWrittenSuccessfully);
175+
progressSoFar.docsWrittenSuccessfully.should.be.greaterThanOrEqual(1);
176+
progressSoFar.docsFailedToBeWritten.should.be.equal(0);
177+
progressSoFar.timeElapsed.should.be.greaterThanOrEqual(0);
178+
documents.length.should.equal(10);
179+
} catch(err){
180+
done(err);
181+
}
182+
}),
183+
onCompletion: ((summary) => {
184+
summary.docsWrittenSuccessfully.should.be.equal(10);
185+
summary.docsFailedToBeWritten.should.be.equal(0);
186+
summary.timeElapsed.should.be.greaterThanOrEqual(0);
187+
})
188+
})); // End of pipe to writeAll transform
189+
setTimeout(()=>{done();}, 3000);
190+
}); // End of test case
191+
192+
it('Return transformed content during read', function(done){
193+
// Read back one document to make sure transform ran fine
194+
db.documents.read({
195+
uris: [transformuris[5]],
196+
categories: ['content', 'metadata']
197+
}).
198+
result(function(response) {
199+
//console.log(JSON.stringify(response, null, 2));
200+
var res1 = response[0].content['key 5'];
201+
var res2 = response[0].content['title'];
202+
var res3 = response[0].content['myBool'];
203+
var res4 = response[0].content['myInt'];
204+
res1.should.equal('value 5');
205+
res2.should.equal('new title');
206+
res3.should.equal('true');
207+
res4.should.equal('2');
208+
done();
209+
}, done);
210+
});
211+
212+
// Verify batch size during ingestion
213+
it('writeAll with batch size', function(done){
214+
215+
var multiDocreadable = new stream.Readable({objectMode: true});
216+
217+
for(let i=0; i<10; i++) {
218+
const temp = {
219+
uri: '/qa/test/dmsdk/batchsize/'+i+'.json',
220+
contentType: 'application/json',
221+
collections: ['qatestsBatchsize'],
222+
content: {['key '+i]:'value '+i}
223+
};
224+
multiDocreadable.push(temp);
225+
batchuris.push(temp.uri);
226+
}
227+
multiDocreadable.push(null);
228+
multiDocreadable.pipe(dbWriter.documents.writeAll({
229+
230+
batchSize:3,
231+
onBatchSuccess: ((progressSoFar, documents) => {
232+
try {
233+
progressSoFar.docsWrittenSuccessfully.should.be.greaterThanOrEqual(1);
234+
progressSoFar.docsFailedToBeWritten.should.be.equal(0);
235+
progressSoFar.timeElapsed.should.be.greaterThanOrEqual(0);
236+
} catch(err){
237+
done(err);
238+
}
239+
}),
240+
onCompletion: ((summary) => {
241+
summary.docsWrittenSuccessfully.should.be.equal(10);
242+
summary.docsFailedToBeWritten.should.be.equal(0);
243+
summary.timeElapsed.should.be.greaterThanOrEqual(0);
244+
})
245+
})); // End of pipe to writeAll transform
246+
setTimeout(()=>{done();}, 3000);
247+
}); // End of test case
248+
249+
it('Return batch size content during read', function(done){
250+
// Read back one document to make sure batch sizes ran fine
251+
db.documents.read({
252+
uris: [batchuris[3]],
253+
categories: ['content', 'metadata']
254+
}).
255+
result(function(response) {
256+
var res = response[0].content['key 3'];
257+
//console.log(JSON.stringify(res, null, 2));
258+
res.should.equal('value 3');
259+
260+
done();
261+
}, done);
262+
});
263+
264+
// Cleanup database contents for writeALL tests based on collections
265+
it('should remove the collection', function(done){
266+
this.timeout(10000);
267+
dbWriter.documents.removeAll({collection:'qatestsBinary'}).
268+
result(function(result) {
269+
return db.documents.probe('/data/121-GIF-Image-GIF-gif_sample1.gif').result();
270+
}, done).
271+
then(function(document) {
272+
document.exists.should.eql(false);
273+
done();
274+
}, done);
275+
});
276+
277+
278+
// Cleanup database contents for writeALL tests based on collections
279+
// removeAll({all: true}) will not work due to temporal docs present.
280+
it('Binary collection delete', function(done){
281+
this.timeout(10000);
282+
dbWriter.documents.removeAll({collection:'qatestsBinary'}).
283+
result(function(result) {
284+
return db.documents.probe('/data/121-GIF-Image-GIF-gif_sample1.gif').result();
285+
}, done).
286+
then(function(document) {
287+
document.exists.should.eql(false);
288+
done();
289+
}, done);
290+
});
291+
292+
it('Batch size collection removal', function(done){
293+
this.timeout(10000);
294+
dbWriter.documents.removeAll({collection:'qatestsBatchsize'}).
295+
result(function(result) {
296+
return db.documents.probe('/qa/test/dmsdk/batchsize/4.json').result();
297+
}, done).
298+
then(function(document) {
299+
document.exists.should.eql(false);
300+
done();
301+
}, done);
302+
});
303+
304+
it('Collection removal - text', function(done){
305+
this.timeout(10000);
306+
dbWriter.documents.removeAll({collection:'qatestsText'}).
307+
result(function(result) {
308+
return db.documents.probe('bbq1.xml').result();
309+
}, done).
310+
then(function(document) {
311+
document.exists.should.eql(false);
312+
done();
313+
}, done);
314+
});
315+
316+
it('Transform docs deleted', function(done){
317+
this.timeout(10000);
318+
dbWriter.documents.removeAll({collection:'qatestsTransform'}).
319+
result(function(result) {
320+
return db.documents.probe('/qa/test/dmsdk/transforms/9.json').result();
321+
}, done).
322+
then(function(document) {
323+
document.exists.should.eql(false);
324+
done();
325+
}, done);
326+
});
327+
328+
}); // End of writeAll-tests
329+
}); // End of datamovement tests

0 commit comments

Comments
 (0)