Process only the latest patch versions

Author: sivakumar-kailasam

lib/get-latest-patch-versions.js

@@ -0,0 +1,11 @@
+import { groupBy } from 'lodash'
+import semverCompare from 'semver-compare'
+import semverUtils from 'semver-utils'
+
+export function getLatestPatchVersions(versions) {
+	let groupedVersions = groupBy(
+		versions.sort(semverCompare).map(semverUtils.parse),
+		v => `${v.major}.${v.minor}`
+	)
+	return Object.values(groupedVersions).map(verGroup => verGroup.pop().version)
+}

lib/read-docs.js

@@ -1,32 +1,42 @@
 import fs from 'fs-extra'
 import glob from 'glob'
-import path from 'path'
-import RSVP from 'rsvp'
 import { dasherize } from 'inflected'
+import { get as deepGet } from 'lodash'
+import { getLatestPatchVersions } from './get-latest-patch-versions'
 
 export default function readDocs(
 	projects,
 	specificVersion = '',
 	ignorePreviouslyIndexedDoc = false,
 	runClean = false
 ) {
-	console.log('Reading project files')
+	return projects.reduce(async (prevPromise, projectName) => {
+		let prevValue = await prevPromise
+		let prevIndexedVersions = []
 
-	let docs = {}
-	projects.forEach(projectName => {
-		let fileName = `${dasherize(projectName)}-docs.json`
-		let folders = glob.sync(`tmp/s3-docs/v${specificVersion}*/${fileName}`)
 		let projectFile = `tmp/json-docs/${projectName}/projects/${projectName}.json`
-		let prevIndexedVersions = []
-		if (fs.existsSync(projectFile)) {
-			projectFile = JSON.parse(fs.readFileSync(projectFile))
-			projectFile.data.relationships['project-versions'].data.forEach(({ id }) => {
-				let version = id.replace(`${projectName}-`, '')
-				prevIndexedVersions.push(`tmp/s3-docs/v${version}/${projectName}-docs.json`)
-			})
+
+		console.log(projectFile, await fs.pathExists(projectFile))
+
+		if (await fs.pathExists(projectFile)) {
+			projectFile = await fs.readJson(projectFile)
+
+			prevIndexedVersions = deepGet(projectFile, 'data.relationships.project-versions.data').map(
+				({ id }) => id.replace(`${projectName}-`, '')
+			)
 		}
 
+		let folders = glob.sync(`tmp/s3-docs/v${specificVersion}*/${dasherize(projectName)}-docs.json`)
+
+		let availableSourceVersions = folders.map(x =>
+			x.replace('tmp/s3-docs/v', '').replace(`/${projectName}-docs.json`, '')
+		)
+
 		if (!runClean) {
+			availableSourceVersions = availableSourceVersions.filter(
+				version => !prevIndexedVersions.includes(version)
+			)
+
 			folders = folders.filter(f => {
 				if (!prevIndexedVersions.includes(f) || ignorePreviouslyIndexedDoc) {
 					return f
@@ -37,25 +47,23 @@ export default function readDocs(
 			})
 		}
 
-		docs[projectName] = folders.map(docs => {
-			let version = path.basename(path.dirname(docs)).replace('v', '')
-			let data
-
-			try {
-				data = JSON.parse(fs.readFileSync(docs, { encoding: 'utf8' }))
-			} catch (e) {
-				console.error(docs)
-				console.error(e.stack)
-				process.exit(1)
-			}
-			console.log('returning from readDocs')
-			return {
-				project: projectName,
-				version,
-				data,
-			}
-		})
-	})
-
-	return RSVP.resolve(docs)
+		availableSourceVersions = getLatestPatchVersions(availableSourceVersions)
+
+		let docs = await Promise.all(
+			availableSourceVersions.map(async version => {
+				let data
+
+				try {
+					data = await fs.readJSON(`tmp/s3-docs/v${version}/${projectName}-docs.json`)
+					return { project: projectName, version, data }
+				} catch (e) {
+					console.error(version)
+					console.error(e.stack)
+					process.exit(1)
+				}
+			})
+		)
+
+		return { ...prevValue, [projectName]: docs }
+	}, Promise.resolve({}))
 }