Skip to content

Commit 2df9bee

Browse files
committed
Fix windows paths, support cacheControl, add unit tests
1 parent 19c6e4b commit 2df9bee

File tree

7 files changed

+202
-34
lines changed

7 files changed

+202
-34
lines changed

README.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-creden
5252
```
5353

5454
### API
55-
```ts
55+
```js
5656
import Uploader from 's3-batch-upload';
5757

5858
await new Uploader({
@@ -63,6 +63,12 @@ await new Uploader({
6363
glob: '*.jpg', // default is '*.*'
6464
concurrency: '200', // default is 100
6565
dryRun: true, // default is false
66+
cacheControl: 'max-age: 300', // can be a string, for all uploade resources
67+
cacheControl: { // or an object with globs as keys to match the input path
68+
'**/settings.json': 'max-age: 60', // 1 mins for settings, specific matches should go first
69+
'**/*.json': 'max-age: 300', // 5 mins for other jsons
70+
'**/*.*': 'max-age: 3600', // 1 hour for everthing else
71+
}
6672
}).upload();
6773
```
6874

package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "s3-batch-upload",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"description": "Super fast batched S3 folder uploads from CLI or API.",
55
"main": "./index.js",
66
"types": "./index.d.ts",
@@ -79,6 +79,7 @@
7979
"babel-eslint": "^8.0.3",
8080
"babel-plugin-istanbul": "^4.1.5",
8181
"chai": "^4.1.2",
82+
"chai-stream": "^0.0.0",
8283
"coveralls": "^2.11.6",
8384
"cross-env": "^5.1.1",
8485
"eslint": "^4.13.1",
@@ -92,6 +93,7 @@
9293
"jsdom": "^11.5.1",
9394
"jsdom-global": "^3.0.2",
9495
"lint-staged": "^6.0.0",
96+
"minimatch": "^3.0.4",
9597
"mocha": "^4.0.1",
9698
"npm-run-all": "^4.1.2",
9799
"nyc": "^11.3.0",

src/lib/Uploader.ts

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import streamBatch from './batch';
22
import S3 from 'aws-sdk/clients/s3';
33

44
const glob = require('glob');
5+
const minimatch = require('minimatch');
56
const path = require('path');
67
const AWS = require('aws-sdk');
78
const ProgressBar = require('progress');
@@ -19,6 +20,8 @@ export type Options = {
1920
glob?: string;
2021
concurrency?: number;
2122
dryRun?: boolean;
23+
cacheControl?: string | { [key: string]: string; };
24+
s3Client?: S3;
2225
};
2326

2427
const defaultOptions = {
@@ -50,7 +53,7 @@ export default class Uploader {
5053
throw new Error('No bucket defined!');
5154
}
5255

53-
this.s3 = new AWS.S3();
56+
this.s3 = this.options.s3Client || new AWS.S3();
5457
}
5558

5659
public upload(): Promise<void> {
@@ -110,15 +113,18 @@ export default class Uploader {
110113
});
111114
}
112115

113-
private uploadFile(localFilePath:string, remotePath:string): Promise<void> {
116+
uploadFile(localFilePath:string, remotePath:string): Promise<void> {
117+
const body = fs.createReadStream(localFilePath);
118+
119+
const params = {
120+
Bucket: this.options.bucket,
121+
Key: remotePath.replace(/\\/, '/'),
122+
Body: body,
123+
ContentType: mime.getType(localFilePath),
124+
CacheControl: this.getCacheControlValue(localFilePath),
125+
};
126+
114127
return new Promise((resolve) => {
115-
const body = fs.createReadStream(localFilePath);
116-
const params = {
117-
Bucket: this.options.bucket,
118-
Key: remotePath,
119-
Body: body,
120-
ContentType: mime.getType(localFilePath),
121-
};
122128
if (!this.options.dryRun) {
123129
this.s3.upload(params, err => {
124130
// tslint:disable-next-line no-console
@@ -130,6 +136,22 @@ export default class Uploader {
130136
}
131137
})
132138
}
139+
140+
getCacheControlValue(file:string ): string {
141+
if (this.options.cacheControl) {
142+
// return single option for all files
143+
if (typeof this.options.cacheControl === 'string') {
144+
return this.options.cacheControl;
145+
}
146+
147+
// find match in glob patterns
148+
const match = Object.keys(this.options.cacheControl).find(key => minimatch(file, key));
149+
return match && this.options.cacheControl[match] || '';
150+
}
151+
152+
// return default value
153+
return '';
154+
}
133155
}
134156

135157

src/lib/cli.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@ import Uploader from './Uploader';
55

66
yargs
77
.usage('Usage: $0 <command> [options]')
8-
.command(['$0'], 'Upload files to s3', () => {}, (argv) => {
8+
.command(['$0', 'upload'], 'Upload files to s3', () => {}, (argv) => {
99
new Uploader(argv).upload();
1010
})
1111
.example('$0 -b bucket-name -p ./files -r /data', 'Upload files from a local folder to a s3 bucket path')
12+
.example('$0 ... -a "max-age: 300"', 'Set cache-control for all files')
13+
.example('$0 ... -a \'{ "**/*.json": "max-age: 300", "**/*.*": "3600" }\'', 'Upload files from a local folder to a s3 bucket path')
1214
.example('$0 -d ...', 'Dry run upload')
1315
.option('d', {
1416
alias: 'dry-run',
@@ -51,6 +53,13 @@ yargs
5153
type: 'string',
5254
nargs: 1,
5355
})
56+
.option('a', {
57+
alias: 'cache-control',
58+
default: '',
59+
describe: 'Cache control for uploaded files, can be string for single value or list of glob settings',
60+
type: 'string',
61+
nargs: 1,
62+
})
5463
// NOTE: For more info, see https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-json-file.html
5564
.option('c', {
5665
alias: 'config',

test/Example.spec.ts

Lines changed: 0 additions & 21 deletions
This file was deleted.

test/Uploader.spec.ts

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
import { expect, use } from 'chai';
2+
import { spy } from 'sinon';
3+
import sinonChai from 'sinon-chai';
4+
import chaiStream from 'chai-stream';
5+
6+
import Uploader from '../src/lib/Uploader';
7+
use(sinonChai);
8+
use(chaiStream);
9+
10+
let uploader:Uploader;
11+
12+
describe('Uploader', () => {
13+
describe('uploadFile', () => {
14+
it('should upload', async function() {
15+
this.timeout(5000);
16+
17+
const s3 = {
18+
upload(_, cb) {
19+
cb(null);
20+
}
21+
};
22+
spy(s3, "upload");
23+
24+
uploader = new Uploader({
25+
localPath: 'test/files',
26+
remotePath: 'fake',
27+
bucket: 'fake',
28+
glob: '**/pano_001_1k_0_0_0.png',
29+
s3Client: <any>s3,
30+
});
31+
32+
await uploader.upload();
33+
34+
const { Body, ...args} = (<any>s3.upload).lastCall.args[0];
35+
36+
37+
expect(args).to.deep.equal({
38+
Bucket: 'fake',
39+
Key: 'fake/panoramas/family-guy/pano_001_1k_0_0_0.png',
40+
ContentType: 'image/png',
41+
CacheControl: '',
42+
});
43+
44+
(<any>expect(Body).to.be.a).ReadableStream;
45+
46+
(<any>s3.upload).restore();
47+
});
48+
49+
it('should fix windows paths', async function() {
50+
this.timeout(5000);
51+
52+
const s3 = {
53+
upload(_, cb) {
54+
cb(null);
55+
}
56+
};
57+
spy(s3, "upload");
58+
59+
uploader = new Uploader({
60+
localPath: 'test/files',
61+
remotePath: 'fake',
62+
bucket: 'fake',
63+
glob: '**/pano_001_1k_0_0_0.png',
64+
s3Client: <any>s3,
65+
});
66+
67+
await uploader.uploadFile('files/panoramas/family-guy/pano_001_1k_0_0_0.png', 'foo\\bar.png');
68+
69+
const { Body, ...args} = (<any>s3.upload).lastCall.args[0];
70+
71+
72+
expect(args).to.deep.equal({
73+
Bucket: 'fake',
74+
Key: 'foo/bar.png',
75+
ContentType: 'image/png',
76+
CacheControl: '',
77+
});
78+
79+
(<any>expect(Body).to.be.a).ReadableStream;
80+
81+
(<any>s3.upload).restore();
82+
});
83+
});
84+
85+
describe('getCacheControlValue', () => {
86+
describe('with no config value', () => {
87+
it('should return default value', () => {
88+
uploader = new Uploader({
89+
localPath: '',
90+
remotePath: '',
91+
bucket: 'a',
92+
});
93+
94+
expect(uploader.getCacheControlValue('foo.bar')).to.equal('');
95+
});
96+
});
97+
98+
describe('with static config value', () => {
99+
it('should return config value', () => {
100+
uploader = new Uploader({
101+
localPath: '',
102+
remotePath: '',
103+
bucket: 'a',
104+
cacheControl: '1'
105+
});
106+
107+
expect(uploader.getCacheControlValue('foo.bar')).to.equal('1');
108+
});
109+
});
110+
111+
describe('with glob config value', () => {
112+
it('should return config value', () => {
113+
uploader = new Uploader({
114+
localPath: '',
115+
remotePath: '',
116+
bucket: 'a',
117+
cacheControl: {
118+
'**/*.json': '10',
119+
'**/*.*': '100',
120+
}
121+
});
122+
123+
expect(uploader.getCacheControlValue('files/foo.json')).to.equal('10');
124+
expect(uploader.getCacheControlValue('files/foo.jpg')).to.equal('100');
125+
expect(uploader.getCacheControlValue('foo.jpg')).to.equal('100');
126+
});
127+
});
128+
129+
});
130+
});

yarn.lock

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1095,6 +1095,14 @@ center-align@^0.1.1:
10951095
align-text "^0.1.3"
10961096
lazy-cache "^1.0.3"
10971097

1098+
chai-stream@^0.0.0:
1099+
version "0.0.0"
1100+
resolved "https://registry.yarnpkg.com/chai-stream/-/chai-stream-0.0.0.tgz#ca825f33cf516481c9e1ddc943f6de4dc5f539c0"
1101+
integrity sha1-yoJfM89RZIHJ4d3JQ/beTcX1OcA=
1102+
dependencies:
1103+
es6-promise "^3.0.2"
1104+
highland "^2.5.1"
1105+
10981106
chai@^4.1.2:
10991107
version "4.1.2"
11001108
resolved "https://registry.yarnpkg.com/chai/-/chai-4.1.2.tgz#0f64584ba642f0f2ace2806279f4f06ca23ad73c"
@@ -1546,6 +1554,11 @@ es6-object-assign@^1.0.3:
15461554
version "1.1.0"
15471555
resolved "https://registry.yarnpkg.com/es6-object-assign/-/es6-object-assign-1.1.0.tgz#c2c3582656247c39ea107cb1e6652b6f9f24523c"
15481556

1557+
es6-promise@^3.0.2:
1558+
version "3.3.1"
1559+
resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613"
1560+
integrity sha1-oIzd6EzNvzTQJ6FFG8kdS80ophM=
1561+
15491562
escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
15501563
version "1.0.5"
15511564
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
@@ -2224,6 +2237,13 @@ he@1.1.1:
22242237
version "1.1.1"
22252238
resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
22262239

2240+
highland@^2.5.1:
2241+
version "2.13.0"
2242+
resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.0.tgz#a4394d8dcb970cd071a79a20f0762b906258dc19"
2243+
integrity sha512-zGZBcgAHPY2Zf9VG9S5IrlcC7CH9ELioXVtp9T5bU2a4fP2zIsA+Y8pV/n/h2lMwbWMHTX0I0xN0ODJ3Pd3aBQ==
2244+
dependencies:
2245+
util-deprecate "^1.0.2"
2246+
22272247
highlight.js@^9.0.0:
22282248
version "9.12.0"
22292249
resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e"
@@ -4589,7 +4609,7 @@ url@0.10.3:
45894609
punycode "1.3.2"
45904610
querystring "0.2.0"
45914611

4592-
util-deprecate@~1.0.1:
4612+
util-deprecate@^1.0.2, util-deprecate@~1.0.1:
45934613
version "1.0.2"
45944614
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
45954615

0 commit comments

Comments
 (0)