Add "clean" option to start to force process all versions (#62)

Author: toddjordan

README.md

@@ -43,6 +43,7 @@ You need an additional flag `AWS_SHOULD_PUBLISH=true` for publishing the docs.
 - First run `yarn start` so that you have all the docs from s3 on your local. This is important so that we don't loose other versions of docs that are already out there when the index files are generated
 - Then go to `./tmp/s3-docs/<the_version_you_want_to_replace>` and override the file there with the yuidoc file that you want to be processed. Ensure that the file name is same as the one that's already there.
 - Then run `yarn start --project=ember-data --version=3.2.0 --ignorePreviouslyIndexedDoc`. Make sure you enter the entire version(including patch version).
+- To run against all versions of ember and ember-data regardless of indexed version, run `node --max_old_space_size=8192 index.js --clean`
 
 ## Generating API Documentation and Testing API Docs Locally
 
@@ -56,7 +57,6 @@ app with documentation pulled from a local copy of ember.js.
     - [ember-api-docs](https://github.com/ember-learn/ember-api-docs)
 1.  Set up the project according to the instructions above in `Running the app`.
 1.  From the `ember-jsonapi-docs` directory, run `./generate-local.sh yui ember 2.18.0`. This command runs the Ember documentation build, generates jsonapi output, and copies it to the `ember-api-docs` directory. To build ember data documentation, run `./generate-local.sh yui ember-data 2.17.2`.
-    - \_If you encounter an error like `ember-2.18.0 has already been indexed in json-docs`, then use a new unique version number like `2.18.1`, or whatever is appropriate.
     - If your `rev-index/ember-X.X.X.json` file fails to generate, make sure you have all dependencies installed for the ember.js repo
     - If you are debugging failed builds, periodically clear out the contents of the `tmp` directory, and run the script again. Past failed runs can cause subsequent runs to fail in unexpected ways.\_
 1.  Run the API app with the newly generated local data by running `API_HOST=http://localhost:4200 ember s` in the `ember-api-docs` directory.

generate-local.sh

@@ -32,7 +32,7 @@ fi
 
 cd ../ember-jsonapi-docs
 echo "🏃 💨  Running ember-jsonapi-docs for version $VERSION 🏃 💨 "
-yarn start -- --project $PROJECT --version $VERSION
+yarn start --project $PROJECT --version $VERSION --ignorePreviouslyIndexedDoc
 echo "🚚 💨  Copying rev-index json file to ember-api-docs app... 🚚 💨 "
 rm -f ../ember-api-docs/public/rev-index/$PROJECT-$VERSION.json
 mkdir -p ../ember-api-docs/public/rev-index

index.js

@@ -9,6 +9,7 @@ let specificDocsVersion = argv.version ? argv.version : ''
 
 let ignorePreviouslyIndexedDoc =
 	projects.length !== 0 && specificDocsVersion !== '' && argv.ignorePreviouslyIndexedDoc
+let runClean = !!argv.clean
 
 const { apiDocsProcessor } = require('./main.js')
-apiDocsProcessor(projects, specificDocsVersion, ignorePreviouslyIndexedDoc)
+apiDocsProcessor(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean)

lib/read-docs.js

@@ -7,7 +7,8 @@ import { dasherize } from 'inflected'
 export default function readDocs(
 	projects,
 	specificVersion = '',
-	ignorePreviouslyIndexedDoc = false
+	ignorePreviouslyIndexedDoc = false,
+	runClean = false
 ) {
 	console.log('Reading project files')
 
@@ -25,14 +26,16 @@ export default function readDocs(
 			})
 		}
 
-		folders = folders.filter(f => {
-			if (!prevIndexedVersions.includes(f) || ignorePreviouslyIndexedDoc) {
-				return f
-			} else {
-				let version = f.replace('tmp/s3-docs/v', '').replace(`/${projectName}-docs.json`, '')
-				console.log(`${projectName}-${version} has already been indexed in json-docs`)
-			}
-		})
+		if (!runClean) {
+			folders = folders.filter(f => {
+				if (!prevIndexedVersions.includes(f) || ignorePreviouslyIndexedDoc) {
+					return f
+				} else {
+					let version = f.replace('tmp/s3-docs/v', '').replace(`/${projectName}-docs.json`, '')
+					console.log(`${projectName}-${version} has already been indexed in json-docs`)
+				}
+			})
+		}
 
 		docs[projectName] = folders.map(docs => {
 			let version = path.basename(path.dirname(docs)).replace('v', '')

main.js

@@ -13,7 +13,12 @@ import saveDoc from './lib/save-document'
 import revProjVersionFiles from './lib/rev-docs'
 import { downloadExistingDocsToLocal, uploadToS3 } from './lib/s3-sync'
 
-export function apiDocsProcessor(projects, specificDocsVersion, ignorePreviouslyIndexedDoc) {
+export function apiDocsProcessor(
+	projects,
+	specificDocsVersion,
+	ignorePreviouslyIndexedDoc,
+	runClean
+) {
 	RSVP.on('error', reason => {
 		console.log(reason)
 		process.exit(1)
@@ -23,8 +28,8 @@ export function apiDocsProcessor(projects, specificDocsVersion, ignorePreviously
 	console.log(`Downloading docs for ${projects.join(' & ')}${docsVersionMsg}`)
 
 	downloadExistingDocsToLocal()
-		.then(() => fetchYuiDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc))
-		.then(() => readDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc))
+		.then(() => fetchYuiDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc || runClean))
+		.then(() => readDocs(projects, specificDocsVersion, ignorePreviouslyIndexedDoc, runClean))
 		.then(docs => {
 			return RSVP.map(projects, projectName => {
 				return RSVP.map(docs[projectName], doc => {