Merge pull request #83 from sat-utils/misc_fixes

Publish 0.1.0

Author: matthewhanson

.circleci/config.yml

@@ -35,8 +35,11 @@ jobs:
           name: Deploy example to aws
           command: |
             if [[  "$CIRCLE_BRANCH" == 'master' ]]; then
-              cd example
-              yarn
-              ./node_modules/.bin/kes cf deploy --template node_modules/@sat-utils/api/template --deployment dev --region us-east-1 --role arn:aws:iam::552819999234:role/sat-api-deployer-role-DeployerRole-JBBKUABAELDR
+              #cd example
+              #yarn
+              #./node_modules/.bin/kes cf deploy --template node_modules/@sat-utils/api/template --deployment dev --region us-east-1 --role arn:aws:iam::552819999234:role/sat-api-deployer-role-DeployerRole-JBBKUABAELDR
+              echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
+              VERSION=$(jq --raw-output .version lerna.json)
+              node_modules/.bin/lerna publish --skip-git --repo-version $VERSION --yes --force-publish=*
             fi
 

CHANGES.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v0.1.0] - 2018-09-18
+
+### Fixed
+- Fixed broken ingests for Landsat and Sentinel
+
 ## [v0.0.2]
 
 ### Added
@@ -37,6 +42,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Refactor several modules: metadata, landsat, sentinel
 - Refactor and improve splitting
 
-[Unreleased]: https://github.com/sat-utils/sat-api/compare/v0.0.2...HEAD
+[Unreleased]: https://github.com/sat-utils/sat-api/compare/v0.1.0...HEAD
+[v0.1.0]: https://github.com/sat-utils/sat-api/compare/v0.0.2...v0.1.0
 [v0.0.2]: https://github.com/sat-utils/sat-api/compare/legacy-v2.0.0...v0.0.2
 [legacy-v2.0.0]: https://github.com/sat-utils/sat-api/tree/legacy

README.md

@@ -34,7 +34,7 @@ Sat-api includes a number of NPM packages (in the packages/ directory) that are
 
     $ yarn docs-serve
 
-On Linux, if you get the message "Error: watch *path*/book.json ENOSPC", issue the following command (require sudo access).
+On Linux, if you get the message "Error: watch *path*/book.json ENOSPC", issue the following command (requires sudo access).
 
     $ echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
 

example/.kes/config.yml

@@ -14,9 +14,9 @@ dev:
     volumeSize: 80
 
 prod:
-  stackName: sat-api-v1
+  stackName: sat-api
   system_bucket: sat-api 
   es:
     instanceCount: 2
     instanceType: m3.medium.elasticsearch
-    volumeSize: 80
+    volumeSize: 80

lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "2.11.0",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -1,6 +1,4 @@
 {
-  "name": "sat-api",
-  "version": "1.0.0",
   "description": "One API to search public Satellites metadata",
   "main": "index.js",
   "repository": "https://github.com/sat-utils/sat-api",

packages/api-lib/libs/api.js

@@ -15,7 +15,7 @@ function Search(event, esClient) {
     params = event.body
   }
 
-  this.headers = event.headers
+  this.endpoint = event.endpoint
 
   this.merge = false
   if (_.has(params, 'merge')) {
@@ -34,17 +34,13 @@ function Search(event, esClient) {
   this.client = esClient
 
   this.queries = queries(this.params)
-
-  console.log(`Queries: ${JSON.stringify(this.queries)}`)
 }
 
 // search for items using collection and items
-Search.prototype.search_items = (callback) => {
+Search.prototype.search_items = function (callback) {
   // check collection first
   this.search_collections((err, resp) => {
     const collections = resp.features.map((c) => c.properties['c:id'])
-    console.log('matched collections', collections)
-    console.log('queries before', JSON.stringify(this.queries))
     let qs
     if (collections.length === 0) {
       qs = { bool: { must_not: { exists: { field: 'c:id' } } } }
@@ -62,7 +58,7 @@ Search.prototype.search_items = (callback) => {
 }
 
 
-Search.prototype.search_collections = (callback) => {
+Search.prototype.search_collections = function (callback) {
   // hacky way to get all collections
   const sz = this.size
   const frm = this.frm
@@ -92,7 +88,7 @@ Search.prototype.search_collections = (callback) => {
 }
 
 
-Search.prototype.search = (index, callback) => {
+Search.prototype.search = function (index, callback) {
   const self = this
 
   const searchParams = {
@@ -129,20 +125,15 @@ Search.prototype.search = (index, callback) => {
       props = _.omit(props, ['bbox', 'geometry', 'assets', 'links', 'eo:bands'])
       const links = body.hits.hits[i]._source.links || []
       // add self and collection links
-      const host = (
-        'X-Forwarded-Host' in self.headers ?
-          self.headers['X-Forwarded-Host'] : self.headers.Host
-      )
-      const apiUrl = `${self.headers['X-Forwarded-Proto']}://${host}`
       let prefix = '/search/stac'
       if (index === 'collections') {
         prefix = '/collections'
-        links.self = { rel: 'self', href: `${apiUrl}${prefix}?c:id=${props.collection}` }
+        links.self = { rel: 'self', href: `${self.endpoint}${prefix}?c:id=${props.collection}` }
       }
       else {
-        links.self = { rel: 'self', href: `${apiUrl}${prefix}?id=${props.id}` }
+        links.self = { rel: 'self', href: `${self.endpoint}${prefix}?id=${props.id}` }
         if (_.has(props, 'c:id')) {
-          links.collection = { href: `${apiUrl}/collections/${props['c:id']}/definition` }
+          links.collection = { href: `${self.endpoint}/collections/${props['c:id']}/definition` }
         }
       }
       return {

packages/api-lib/libs/es.js

@@ -134,7 +134,7 @@ function streamToEs(stream, transform, client, index) {
   let nRecords = 0
   let nTransformed = 0
 
-  const toEs = through2({ objectMode: true, consume: true }, (data, encoding, next) => {
+  const toEs = through2({ objectMode: true, consume: true }, function (data, encoding, next) {
     const record = {
       index,
       type: 'doc',

packages/api-lib/libs/ingest-csv.js

@@ -3,8 +3,6 @@
 const csv = require('fast-csv')
 const AWS = require('aws-sdk')
 const zlib = require('zlib')
-const pLimit = require('p-limit')
-const lodash = require('lodash')
 const es = require('./es')
 
 let esClient
@@ -34,10 +32,9 @@ function invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, retries) {
       else {
         console.log(`launched ${JSON.stringify(params)}`)
       }
-    }).promise()
+    })
   }
-
-  return Promise.resolve()
+  return 0
 }
 
 // split a CSV to multiple files and trigger lambdas
@@ -128,7 +125,6 @@ function split({
   newStream.on('error', (e) => cb(e))
 
   return newStream.on('end', () => {
-    const limit = pLimit(3) //set concurrent call to 3 at a time
     // write the last records
     if (lineCounter > 0) {
       fileCounter += 1
@@ -158,16 +154,13 @@ function split({
         `${extra} extra (Files ${startFile}-${maxEndFile})`
       )
 
-      const promises = lodash.range(numLambdas).map((i) => limit(() => {
-        endFile = (i < extra) ? startFile + batchSize : (startFile + batchSize) - 1
+      for (let i = 0; i < numLambdas; i += 1) {
+        endFile = (i < extra) ? (startFile + batchSize) : ((startFile + batchSize) - 1)
+        invokeLambda(bucket, key, startFile, Math.min(endFile, maxEndFile), arn)
         startFile = endFile + 1
-        return invokeLambda(bucket, key, startFile - 1, Math.min(endFile, maxEndFile), arn)
-      }))
-
-      return Promise.all(promises).then(() => cb()).catch(cb)
+      }
     }
     cb()
-    return Promise.resolve()
   })
 }
 

packages/api-lib/libs/queries.js

@@ -87,7 +87,7 @@ const geometryQuery = (field, geometry) => {
 
 
 const intersects = (inGeojson, queries) => {
-  let geojson
+  let geojson = inGeojson
   // if we receive an object, assume it's GeoJSON, if not, try and parse
   if (typeof geojson === 'string') {
     try {