Skip to content

Commit 68f10fd

Browse files
sivakumar-kailasamsivakumar-kailasam
committed
Use npm as the api docs source
1 parent 93b4921 commit 68f10fd

File tree

5 files changed

+820
-134
lines changed

5 files changed

+820
-134
lines changed

‎lib/fetch-yui-docs-from-s3.js

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
import S3 from 's3'
2+
import RSVP from 'rsvp'
3+
import path from 'path'
4+
import fs from 'fs-extra'
5+
import semverUtils from 'semver-utils'
6+
import semverExtra from 'semver-extra'
7+
import mkdirp from 'mkdir-promise'
8+
import { find as findObject, groupBy, trim, isEmpty } from 'lodash'
9+
10+
// These are read-only credentials to the builds.emberjs.com bucket only.
11+
const { AWS_ACCESS_KEY, AWS_SECRET_KEY } = process.env
12+
13+
const client = S3.createClient({
14+
s3Options: {
15+
accessKeyId: AWS_ACCESS_KEY,
16+
secretAccessKey: AWS_SECRET_KEY,
17+
},
18+
})
19+
20+
const options = {
21+
s3Params: {
22+
Bucket: 'builds.emberjs.com',
23+
Prefix: 'tags',
24+
},
25+
}
26+
27+
function getObjects() {
28+
return new RSVP.Promise((resolve, reject) => {
29+
let data = []
30+
31+
client
32+
.listObjects(options)
33+
.on('data', ({ Contents }) => {
34+
data = data.concat(Contents)
35+
})
36+
.on('end', () => resolve(data))
37+
.on('error', reject)
38+
})
39+
}
40+
41+
const getSrcFilePath = Key => {
42+
let name = path.basename(Key)
43+
let dir = path.basename(path.dirname(Key))
44+
return path.join('tmp', 's3-docs', dir, name)
45+
}
46+
47+
function downloadFile({ Key }) {
48+
let finalFile = getSrcFilePath(Key)
49+
50+
return mkdirp(path.dirname(finalFile)).then(() => {
51+
return new RSVP.Promise((resolve, reject) => {
52+
if (fs.existsSync(finalFile)) {
53+
return resolve(finalFile)
54+
} else {
55+
client
56+
.downloadFile({
57+
localFile: finalFile,
58+
s3Params: {
59+
Bucket: 'builds.emberjs.com',
60+
Key,
61+
},
62+
})
63+
.on('end', () => {
64+
console.log(`Downloaded ${finalFile}`)
65+
resolve(finalFile)
66+
})
67+
.on('error', err => {
68+
console.warn(`err! ${err}`)
69+
reject(err)
70+
})
71+
}
72+
})
73+
})
74+
}
75+
76+
function filterReleaseDocs({ Key }) {
77+
let key = Key.split('/')
78+
let tag = key[key.length - 2]
79+
let versionRegex = /v\d+\.\d+\.\d+$/
80+
return versionRegex.test(tag) && /-docs\.json/.test(key)
81+
}
82+
83+
export default function fetchYuiDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc) {
84+
return getObjects().then(docs => {
85+
let projectFiles = projects.map(p => `${p}-docs.json`)
86+
87+
let filteredDocs = docs
88+
.filter(filterReleaseDocs)
89+
.filter(
90+
({ Key }) =>
91+
projectFiles.includes(Key.split('/').pop()) && Key.includes(specificDocsVersion)
92+
)
93+
94+
if (!isEmpty(trim(specificDocsVersion))) {
95+
if (ignorePreviouslyIndexedDoc) {
96+
return RSVP.map(filteredDocs, ({ Key }) => {
97+
console.log(
98+
`Skipping download of ${Key} so that you can use your local file. We hope you know what you're doing 😅`
99+
)
100+
return getSrcFilePath(Key)
101+
})
102+
} else {
103+
return RSVP.map(filteredDocs, downloadFile)
104+
}
105+
}
106+
107+
let projectDocs = groupBy(filteredDocs, ({ Key }) => Key.split('/')[2].replace('.json', ''))
108+
109+
let docsToProcess = []
110+
Object.keys(projectDocs).forEach(projectName => {
111+
let docs = projectDocs[projectName].map(d => {
112+
return Object.assign({}, d, semverUtils.parse(d.Key.split('/')[1]))
113+
})
114+
let xDocs = groupBy(docs, ({ major, minor }) => `${major}.${minor}`)
115+
116+
docsToProcess = Object.keys(xDocs).map(key => {
117+
const latestVer = semverExtra.max(xDocs[key].map(({ version }) => version))
118+
return findObject(docs, ({ version }) => version === latestVer)
119+
})
120+
})
121+
122+
console.log(
123+
`Need to process ${docsToProcess.length} out of ${filteredDocs.length} published yui docs`
124+
)
125+
126+
return RSVP.map(docsToProcess, downloadFile)
127+
})
128+
}

‎lib/fetch-yui-docs.js

Lines changed: 65 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,128 +1,73 @@
1-
import S3 from 's3'
2-
import RSVP from 'rsvp'
3-
import path from 'path'
1+
import compareVersions from 'compare-versions'
2+
import download from 'download'
43
import fs from 'fs-extra'
5-
import semverUtils from 'semver-utils'
6-
import semverExtra from 'semver-extra'
7-
import mkdirp from 'mkdir-promise'
8-
import { find as findObject, groupBy, trim, isEmpty } from 'lodash'
9-
10-
// These are read-only credentials to the builds.emberjs.com bucket only.
11-
const { AWS_ACCESS_KEY, AWS_SECRET_KEY } = process.env
12-
13-
const client = S3.createClient({
14-
s3Options: {
15-
accessKeyId: AWS_ACCESS_KEY,
16-
secretAccessKey: AWS_SECRET_KEY,
17-
},
18-
})
19-
20-
const options = {
21-
s3Params: {
22-
Bucket: 'builds.emberjs.com',
23-
Prefix: 'tags',
24-
},
25-
}
26-
27-
function getObjects() {
28-
return new RSVP.Promise((resolve, reject) => {
29-
let data = []
30-
31-
client
32-
.listObjects(options)
33-
.on('data', ({ Contents }) => {
34-
data = data.concat(Contents)
35-
})
36-
.on('end', () => resolve(data))
37-
.on('error', reject)
38-
})
39-
}
40-
41-
const getSrcFilePath = Key => {
42-
let name = path.basename(Key)
43-
let dir = path.basename(path.dirname(Key))
44-
return path.join('tmp', 's3-docs', dir, name)
4+
import { isEmpty, trim } from 'lodash'
5+
import mkdirp from 'mkdirp'
6+
import pkgVersions from 'pkg-versions'
7+
import { all, resolve } from 'rsvp'
8+
9+
let filterValidVersions = (specificDocsVersion, ignorePreviouslyIndexedDoc) => version => {
10+
let isCompatibleVersion = compareVersions(version, '3.4.0') !== -1 && !version.includes('beta')
11+
12+
if (!isEmpty(trim(specificDocsVersion)) && !ignorePreviouslyIndexedDoc) {
13+
return isCompatibleVersion && version.includes(specificDocsVersion)
14+
} else {
15+
return isCompatibleVersion
16+
}
4517
}
4618

47-
function downloadFile({ Key }) {
48-
let finalFile = getSrcFilePath(Key)
49-
50-
return mkdirp(path.dirname(finalFile)).then(() => {
51-
return new RSVP.Promise((resolve, reject) => {
52-
if (fs.existsSync(finalFile)) {
53-
return resolve(finalFile)
54-
} else {
55-
client
56-
.downloadFile({
57-
localFile: finalFile,
58-
s3Params: {
59-
Bucket: 'builds.emberjs.com',
60-
Key,
61-
},
62-
})
63-
.on('end', () => {
64-
console.log(`Downloaded ${finalFile}`)
65-
resolve(finalFile)
66-
})
67-
.on('error', err => {
68-
console.warn(`err! ${err}`)
69-
reject(err)
70-
})
19+
async function fetchYuiDocsFromNpm(projects, projectName, filterValidVersionsForRequestedDocs) {
20+
if (!projects.includes(projectName)) {
21+
return resolve()
22+
}
23+
24+
let projectPkgMap = {
25+
ember: 'ember-source',
26+
'ember-data': 'ember-data',
27+
}
28+
let versions = await pkgVersions(projectPkgMap[projectName])
29+
let versionsToProcess = [...versions].filter(filterValidVersionsForRequestedDocs)
30+
31+
let getProjectFileUrl
32+
33+
if (projectName === 'ember') {
34+
getProjectFileUrl = v => `https://unpkg.com/ember-source@${v}/docs/data.json`
35+
} else {
36+
getProjectFileUrl = v => `https://unpkg.com/ember-data@${v}/dist/docs/data.json`
37+
}
38+
39+
return await all(
40+
versionsToProcess.map(async v => {
41+
let url = getProjectFileUrl(v)
42+
43+
try {
44+
let data = await download(url)
45+
let targetDir = `tmp/s3-docs/v${v}`
46+
let targetFile = `${targetDir}/${projectName}-docs.json`
47+
48+
mkdirp.sync(targetDir)
49+
fs.writeFileSync(targetFile, data)
50+
51+
return targetFile
52+
} catch (e) {
53+
console.error(`Failed to download ${url} ${e}`)
7154
}
7255
})
73-
})
74-
}
75-
76-
function filterReleaseDocs({ Key }) {
77-
let key = Key.split('/')
78-
let tag = key[key.length - 2]
79-
let versionRegex = /v\d+\.\d+\.\d+$/
80-
return versionRegex.test(tag) && /-docs\.json/.test(key)
56+
)
8157
}
8258

83-
export default function fetchYuiDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc) {
84-
return getObjects().then(docs => {
85-
let projectFiles = projects.map(p => `${p}-docs.json`)
86-
87-
let filteredDocs = docs
88-
.filter(filterReleaseDocs)
89-
.filter(
90-
({ Key }) =>
91-
projectFiles.includes(Key.split('/').pop()) && Key.includes(specificDocsVersion)
92-
)
93-
94-
if (!isEmpty(trim(specificDocsVersion))) {
95-
if (ignorePreviouslyIndexedDoc) {
96-
return RSVP.map(filteredDocs, ({ Key }) => {
97-
console.log(
98-
`Skipping download of ${Key} so that you can use your local file. We hope you know what you're doing 😅`
99-
)
100-
return getSrcFilePath(Key)
101-
})
102-
} else {
103-
return RSVP.map(filteredDocs, downloadFile)
104-
}
105-
}
106-
107-
let projectDocs = groupBy(filteredDocs, ({ Key }) => Key.split('/')[2].replace('.json', ''))
108-
109-
let docsToProcess = []
110-
Object.keys(projectDocs).forEach(projectName => {
111-
let docs = projectDocs[projectName].map(d => {
112-
return Object.assign({}, d, semverUtils.parse(d.Key.split('/')[1]))
113-
})
114-
let xDocs = groupBy(docs, ({ major, minor }) => `${major}.${minor}`)
115-
116-
docsToProcess = Object.keys(xDocs).map(key => {
117-
const latestVer = semverExtra.max(xDocs[key].map(({ version }) => version))
118-
return findObject(docs, ({ version }) => version === latestVer)
119-
})
120-
})
121-
122-
console.log(
123-
`Need to process ${docsToProcess.length} out of ${filteredDocs.length} published yui docs`
124-
)
125-
126-
return RSVP.map(docsToProcess, downloadFile)
127-
})
59+
export default async function fetchYuiDocs(
60+
projects,
61+
specificDocsVersion,
62+
ignorePreviouslyIndexedDoc
63+
) {
64+
let filterValidVersionsForRequestedDocs = filterValidVersions(
65+
specificDocsVersion,
66+
ignorePreviouslyIndexedDoc
67+
)
68+
69+
return [
70+
...(await fetchYuiDocsFromNpm(projects, 'ember', filterValidVersionsForRequestedDocs)),
71+
...(await fetchYuiDocsFromNpm(projects, 'ember-data', filterValidVersionsForRequestedDocs)),
72+
]
12873
}

‎lib/s3-sync.js

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ import { merge } from 'lodash'
99
// To increase s3's download & upload dir perf
1010
http.globalAgent.maxSockets = https.globalAgent.maxSockets = 30
1111

12-
const deleteRemoved = true
13-
1412
const { AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_SHOULD_PUBLISH } = process.env
1513

1614
const client = S3.createClient({
@@ -29,7 +27,6 @@ const jsonDocsDirDownloadOptions = {
2927
}
3028

3129
const jsonDocsDirUploadOptions = merge({}, jsonDocsDirDownloadOptions, {
32-
deleteRemoved,
3330
s3Params: {
3431
ACL: 'public-read',
3532
CacheControl: 'max-age=365000000, immutable',
@@ -45,7 +42,6 @@ let revDocsDirDownloadOptions = {
4542
}
4643

4744
const revDocsDirUploadOptions = merge({}, revDocsDirDownloadOptions, {
48-
deleteRemoved,
4945
s3Params: {
5046
ACL: 'public-read',
5147
},

‎package.json

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
},
2121
"dependencies": {
2222
"cheerio": "^1.0.0-rc.2",
23+
"compare-versions": "^3.3.1",
24+
"download": "^7.1.0",
2325
"esm": "^3.0.71",
2426
"fs-extra": "^7.0.0",
2527
"glob": "^7.1.2",
@@ -33,6 +35,7 @@
3335
"mkdir-promise": "^1.0.0",
3436
"mkdirp": "^0.5.1",
3537
"ora": "^3.0.0",
38+
"pkg-versions": "^1.1.0",
3639
"rev-file": "^3.0.0",
3740
"rimraf": "^2.6.2",
3841
"rsvp": "^4.8.3",
@@ -57,7 +60,9 @@
5760
"engines": {
5861
"node": ">= 8"
5962
},
60-
"cacheDirectories": ["tmp"],
63+
"cacheDirectories": [
64+
"tmp"
65+
],
6166
"husky": {
6267
"hooks": {
6368
"pre-commit": "pretty-quick --staged"

0 commit comments

Comments
 (0)