Skip to content
This repository was archived by the owner on Feb 21, 2026. It is now read-only.

Commit 9ecb8bd

Browse files
rgbkrkrgbkrk
committed
refactor integration script into separate modules
1 parent fafc4de commit 9ecb8bd

File tree

6 files changed

+195
-131
lines changed

6 files changed

+195
-131
lines changed

‎ci/integration.js‎

Lines changed: 18 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,27 @@
1-
const child_process = require("child_process");
2-
const { randomBytes } = require("crypto");
31
const path = require("path");
42
const fs = require("fs");
53

6-
global.XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
7-
8-
const { ajax } = require("rxjs/ajax");
9-
10-
const rxJupyter = require("rx-jupyter");
11-
124
const _ = require("lodash");
135

14-
console.log("running bookstore integration tests");
15-
16-
async function genToken(byteLength = 32) {
17-
return new Promise((resolve, reject) => {
18-
randomBytes(byteLength, (err, buffer) => {
19-
if (err) {
20-
reject(err);
21-
return;
22-
}
23-
24-
resolve(buffer.toString("hex"));
25-
});
26-
});
27-
}
6+
const s3 = require("./s3");
7+
const { JupyterServer } = require("./jupyter");
288

29-
const sleep = timeout =>
30-
new Promise((resolve, reject) => setTimeout(resolve, timeout));
9+
const { sleep } = require("./sleep");
3110

3211
// Catch all rogue promise rejections to fail CI
3312
process.on("unhandledRejection", error => {
3413
console.log("unhandledRejection", error.message);
14+
console.error(error.stack);
3515
process.exit(2);
3616
});
3717

38-
var Minio = require("minio");
18+
console.log("running bookstore integration tests");
3919

4020
const main = async () => {
41-
const jupyterToken = await genToken();
42-
const jupyterPort = 9988;
43-
const jupyterEndpoint = `http://127.0.0.1:${jupyterPort}`;
44-
4521
const bucketName = "bookstore";
46-
// Optional according to minio docs, likely here for AWS compat
47-
const regionName = "us-east-1";
22+
23+
const jupyterServer = new JupyterServer();
24+
await jupyterServer.start();
4825

4926
const s3Config = {
5027
endPoint: "127.0.0.1",
@@ -56,69 +33,11 @@ const main = async () => {
5633

5734
// Instantiate the minio client with the endpoint
5835
// and access keys as shown below.
59-
var minioClient = new Minio.Client(s3Config);
60-
61-
const madeBucket = await new Promise((resolve, reject) => {
62-
minioClient.makeBucket(bucketName, regionName, err => {
63-
if (err) {
64-
reject(err);
65-
return;
66-
}
67-
resolve();
68-
});
69-
});
70-
console.log(`Created bucket ${bucketName}`);
36+
var s3Client = new s3.Client(s3Config);
7137

72-
const jupyter = child_process.spawn(
73-
"jupyter",
74-
[
75-
"notebook",
76-
"--no-browser",
77-
`--NotebookApp.token=${jupyterToken}`,
78-
`--NotebookApp.disable_check_xsrf=True`,
79-
`--port=${jupyterPort}`,
80-
`--ip=127.0.0.1`
81-
],
82-
{ cwd: __dirname }
83-
);
38+
await s3Client.makeBucket(bucketName);
8439

85-
////// Refactor me later, streams are a bit messy with async await
86-
// Check to see that jupyter is up
87-
let jupyterUp = false;
88-
89-
jupyter.stdout.on("data", data => {
90-
const s = data.toString();
91-
console.log(s);
92-
});
93-
jupyter.stderr.on("data", data => {
94-
const s = data.toString();
95-
96-
console.error(s);
97-
if (s.includes("Jupyter Notebook is running at")) {
98-
jupyterUp = true;
99-
}
100-
});
101-
jupyter.stdout.on("end", data => console.log("DONE WITH JUPYTER"));
102-
103-
jupyter.on("exit", code => {
104-
if (code != 0) {
105-
// Jupyter exited badly
106-
console.error("jupyter errored", code);
107-
process.exit(code);
108-
}
109-
});
110-
111-
await sleep(3000);
112-
113-
if (!jupyterUp) {
114-
console.log("jupyter has not come up after 3 seconds, waiting 3 more");
115-
await sleep(3000);
116-
117-
if (!jupyterUp) {
118-
console.log("jupyter has not come up after 6 seconds, bailing");
119-
process.exit(1);
120-
}
121-
}
40+
console.log(`Created bucket ${bucketName}`);
12241

12342
const originalNotebook = {
12443
cells: [
@@ -153,47 +72,18 @@ const main = async () => {
15372
nbformat_minor: 2
15473
};
15574

156-
const xhr = await ajax({
157-
url: `${jupyterEndpoint}/api/contents/ci-local-writeout.ipynb`,
158-
responseType: "json",
159-
createXHR: () => new XMLHttpRequest(),
160-
method: "PUT",
161-
body: {
162-
type: "notebook",
163-
content: originalNotebook
164-
},
165-
headers: {
166-
"Content-Type": "application/json",
167-
Authorization: `token ${jupyterToken}`
168-
}
169-
}).toPromise();
75+
jupyterServer.writeNotebook("ci-local-writeout.ipynb", originalNotebook);
17076

17177
// Wait for minio to have the notebook
17278
// Future iterations of this script should poll to get the notebook
17379
await sleep(1000);
17480

175-
jupyter.kill();
81+
jupyterServer.shutdown();
17682

177-
//// Check the notebook we placed on S3
178-
const rawNotebook = await new Promise((resolve, reject) =>
179-
minioClient.getObject(
180-
bucketName,
181-
"ci-workspace/ci-local-writeout.ipynb",
182-
(err, dataStream) => {
183-
if (err) {
184-
console.error("wat");
185-
reject(err);
186-
return;
187-
}
188-
189-
const chunks = [];
190-
dataStream.on("data", chunk => chunks.push(chunk));
191-
dataStream.on("error", reject);
192-
dataStream.on("end", () => {
193-
resolve(Buffer.concat(chunks).toString("utf8"));
194-
});
195-
}
196-
)
83+
/***** Check notebook from S3 *****/
84+
const rawNotebook = await s3Client.getObject(
85+
bucketName,
86+
"ci-workspace/ci-local-writeout.ipynb"
19787
);
19888

19989
const notebook = JSON.parse(rawNotebook);
@@ -208,7 +98,7 @@ const main = async () => {
20898

20999
console.log("Notebook on S3 matches what we sent");
210100

211-
//// Check the notebook we placed on Disk
101+
/***** Check notebook from Disk *****/
212102
const diskNotebook = await new Promise((resolve, reject) =>
213103
fs.readFile(
214104
path.join(__dirname, "ci-local-writeout.ipynb"),

‎ci/jupyter.js‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
const child_process = require("child_process");
2+
const { genToken } = require("./token");
3+
const { sleep } = require("./sleep");
4+
5+
// "Polyfill" XMLHttpRequest for rxjs' ajax to use
6+
global.XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
7+
const { ajax } = require("rxjs/ajax");
8+
9+
class JupyterServer {
10+
constructor(config = {}) {
11+
this.port = config.port || 9988;
12+
this.ip = config.ip || "127.0.0.1";
13+
this.scheme = config.scheme || "http";
14+
this.token = null;
15+
16+
// Launch the server from the directory of this script by default
17+
this.cwd = config.cwd || __dirname;
18+
19+
this.process = null;
20+
this.up = false;
21+
}
22+
23+
async start() {
24+
if (!this.token) {
25+
this.token = await genToken();
26+
}
27+
28+
this.process = child_process.spawn(
29+
"jupyter",
30+
[
31+
"notebook",
32+
"--no-browser",
33+
`--NotebookApp.token=${this.token}`,
34+
`--NotebookApp.disable_check_xsrf=True`,
35+
`--port=${this.port}`,
36+
`--ip=${this.ip}`
37+
],
38+
{ cwd: this.cwd }
39+
);
40+
41+
////// Refactor me later, streams are a bit messy with async await
42+
////// Let's use spawn-rx in the future and make some clean rxjs with timeouts
43+
this.process.stdout.on("data", data => {
44+
const s = data.toString();
45+
console.log(s);
46+
});
47+
this.process.stderr.on("data", data => {
48+
const s = data.toString();
49+
50+
console.error(s);
51+
if (s.includes("Jupyter Notebook is running at")) {
52+
this.up = true;
53+
}
54+
});
55+
this.process.stdout.on("end", data =>
56+
console.log("jupyter server terminated")
57+
);
58+
59+
await sleep(3000);
60+
61+
if (!this.up) {
62+
console.log("jupyter has not come up after 3 seconds, waiting 3 more");
63+
await sleep(3000);
64+
65+
if (!this.up) {
66+
throw new Error("jupyter has not come up after 6 seconds, bailing");
67+
}
68+
}
69+
}
70+
71+
async writeNotebook(path, notebook) {
72+
// Once https://github.com/nteract/nteract/pull/3651 is merged, we can use
73+
// rx-jupyter for writing a notebook to the contents API
74+
const xhr = await ajax({
75+
url: `${this.endpoint}/api/contents/${path}`,
76+
responseType: "json",
77+
createXHR: () => new XMLHttpRequest(),
78+
method: "PUT",
79+
body: {
80+
type: "notebook",
81+
content: notebook
82+
},
83+
headers: {
84+
"Content-Type": "application/json",
85+
Authorization: `token ${this.token}`
86+
}
87+
}).toPromise();
88+
89+
return xhr;
90+
}
91+
92+
shutdown() {
93+
this.process.kill();
94+
}
95+
96+
get endpoint() {
97+
return `${this.scheme}://${this.ip}:${this.port}`;
98+
}
99+
}
100+
101+
module.exports = {
102+
JupyterServer
103+
};

‎ci/s3.js‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Optional according to minio docs, included for AWS compat
2+
const regionName = "us-east-1";
3+
4+
const Minio = require("minio");
5+
6+
function makeBucket(minioClient, bucketName) {
7+
return new Promise((resolve, reject) => {
8+
minioClient.makeBucket(bucketName, regionName, err => {
9+
if (err) {
10+
reject(err);
11+
return;
12+
}
13+
resolve();
14+
});
15+
});
16+
}
17+
18+
function getObject(minioClient, bucketName, objectName) {
19+
return new Promise((resolve, reject) =>
20+
minioClient.getObject(bucketName, objectName, (err, dataStream) => {
21+
if (err) {
22+
reject(err);
23+
return;
24+
}
25+
26+
const chunks = [];
27+
dataStream.on("data", chunk => chunks.push(chunk));
28+
dataStream.on("error", reject);
29+
dataStream.on("end", () => {
30+
resolve(Buffer.concat(chunks).toString("utf8"));
31+
});
32+
})
33+
);
34+
}
35+
36+
class Client {
37+
constructor(s3Config) {
38+
this.minioClient = new Minio.Client(s3Config);
39+
}
40+
41+
makeBucket(bucketName) {
42+
return makeBucket(this.minioClient, bucketName);
43+
}
44+
45+
getObject(bucketName, objectName) {
46+
return getObject(this.minioClient, bucketName, objectName);
47+
}
48+
}
49+
50+
module.exports = {
51+
Client
52+
};

‎ci/sleep.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
module.exports = {
2+
sleep: timeout =>
3+
new Promise((resolve, reject) => setTimeout(resolve, timeout))
4+
};

‎ci/token.js‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
const { randomBytes } = require("crypto");
2+
3+
function genToken(byteLength = 32) {
4+
return new Promise((resolve, reject) => {
5+
randomBytes(byteLength, (err, buffer) => {
6+
if (err) {
7+
reject(err);
8+
return;
9+
}
10+
11+
resolve(buffer.toString("hex"));
12+
});
13+
});
14+
}
15+
16+
module.exports = {
17+
genToken
18+
};

‎package.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,7 @@
2323
"dependencies": {
2424
"lodash": "^4.17.11",
2525
"minio": "^7.0.1",
26-
"rx-jupyter": "^4.0.0-alpha.0",
2726
"rxjs": "^6.3.3",
28-
"spawn-rx": "^3.0.0",
29-
"xhr2": "^0.1.4",
3027
"xmlhttprequest": "^1.8.0"
3128
}
3229
}

0 commit comments

Comments
 (0)