Cache Shapes.txt

Author: consindo

.gitignore

@@ -45,6 +45,7 @@ cache/*.txt
 cache/*/*.txt
 cache/stops/*/*.txt
 cache/*.zip
+cache/*/*
 cache/maps/*.png
 cache/maps/*.webp
 dist/generated/*

package-lock.json

@@ -77,6 +77,7 @@
     "react-transition-group": "^2.2.0",
     "request": "^2.74.0",
     "resolve-url-loader": "^2.1.0",
+    "rimraf": "^2.6.2",
     "sendgrid": "^4.3.0",
     "stream-transform": "^0.1.2",
     "style-loader": "^0.18.2",

package.json

@@ -29,6 +29,7 @@ const config = {
 			name: 'calendar_dates.txt',
 			table: 'calendar_dates'
 		},
-	]
+	],
+	shapeFile: 'shapes.txt'
 }
 module.exports = config

server/agencies/at.js

@@ -1,5 +1,10 @@
+const fs = require('fs')
+const path = require('path')
+const rimraf = require('rimraf')
+
 const connection = require('./db/connection.js')
 const gtfsImport = require('./db/gtfs-import.js')
+const createShapes = require('./db/create-shapes.js')
 
 const metlink = require('./agencies/metlink.js')
 const at = require('./agencies/at.js')
@@ -19,5 +24,18 @@ async function doShit() {
       console.log(err)
     })
   })
+
+  const outputDir = '../cache/metlink.zipunarchived/shapes'
+  // cleans up old import if exists
+  if (fs.existsSync(outputDir)) {
+    await new Promise((resolve, reject) => {
+      rimraf(outputDir, resolve)
+    })
+  }
+  fs.mkdirSync(outputDir)
+
+  let metlinkShapes = new createShapes()
+  await metlinkShapes.create('../cache/metlink.zipunarchived/shapes.txt', outputDir, ['20170828_20170808-090059'])
+  await metlinkShapes.upload('nz-wlg-20170828-20170808-090059', path.resolve(outputDir, '20170828_20170808-090059'))
 }
 connection.isReady.then(doShit)

server/cache-new.js

@@ -1,10 +1,11 @@
-var azure = require('azure-storage');
+var azure = require('azure-storage')
 var request = require('request')
 var moment = require('moment-timezone')
 var fs = require('fs')
 const extract = require('extract-zip')
 const csvparse = require('csv-parse')
 const transform = require('stream-transform')
+const rimraf = require('rimraf')
 
 const Queue = require('./queue.js')
 const path = require('path')
@@ -16,6 +17,7 @@ var tableSvc = azure.createTableService()
 const sql = require('mssql')
 const connection = require('./db/connection.js')
 const gtfsImport = require('./db/gtfs-import.js')
+const createShapes = require('./db/create-shapes.js')
 
 const zipLocation = path.resolve(__dirname, '../cache/at.zip')
 
@@ -183,6 +185,28 @@ async function importAt() {
     await importer.upload(zipLocation + 'unarchived', file, at.prefix, version, ignoreVersions)
   }
 
+  let atShapes = new createShapes()
+  const inputDir = path.resolve(zipLocation + 'unarchived', 'shapes.txt')
+  const outputDir = path.resolve(zipLocation + 'unarchived', 'shapes')
+
+  // cleans up old import if exists
+  if (fs.existsSync(outputDir)) {
+    await new Promise((resolve, reject) => {
+      rimraf(outputDir, resolve)
+    })
+  }
+  fs.mkdirSync(outputDir)
+
+  // creates the new datas
+  await atShapes.create(inputDir, outputDir, version)
+
+  const shapeFiles = fs.readdirSync(outputDir)
+  for (let singleVersion of shapeFiles) {
+    const containerName = (at.prefix + '-' + singleVersion).replace('.', '-').replace('_', '-')
+    await atShapes.upload(containerName, path.resolve(outputDir, singleVersion))
+  }
+  console.log('Uploaded All Shapes')
+
   // It was a success
   for (let v in version) {
     const vRequest = connection.get().request()

server/cache.js

@@ -8,7 +8,7 @@ const azure = require('azure-storage')
 const blobSvc = azure.createBlobService()
 
 class createShapes {
-  create(inputFile, outputDirectory) {
+  create(inputFile, outputDirectory, versions) {
     return new Promise((resolve, reject) => {
       const input = fs.createReadStream(inputFile)
       const parser = csvparse({delimiter: ','})
@@ -46,7 +46,17 @@ class createShapes {
       }).on('finish', () => {
         console.log('Created Shapes. Writing to disk...')
         Object.keys(output).forEach((key) => {
-          fs.writeFileSync(path.resolve(outputDirectory, `${key}.json`), JSON.stringify(output[key]))
+          let subfolder = (versions.length === 1 ? versions[0] : versions[0] + '-extra')
+          versions.forEach((version) => {
+            if (key.match(version)) {
+              subfolder = version
+            }
+          })
+          let dir = path.resolve(outputDirectory, subfolder)
+          if (!fs.existsSync(dir)){
+            fs.mkdirSync(dir)
+          }
+          fs.writeFileSync(path.resolve(dir, `${key}.json`), JSON.stringify(output[key]))
         })
         console.log('Written to disk!')
         resolve()
@@ -87,6 +97,9 @@ class createShapes {
         }
         console.log(container.green +':', 'Blob Container Created.')
         fs.readdir(directory, function(err, files) {
+          if (err) {
+            console.error(err)
+          }
           uploadSingle(files, 0, uploadSingle)
         })    
       })

server/db/create-shapes.js

@@ -46,6 +46,7 @@ router.get('/:prefix/line/:line', line.getLine)
 router.get('/:prefix/stops/trip/:trip_id', line.getStopsFromTrip)
 router.get('/:prefix/stops/shape/:shape_id', line.getStopsFromShape)
 router.get('/:prefix/shape/:shape_id', line.getShape)
+router.get('/:prefix/shapejson/:shape_id', line.getShapeJSON)
 router.post('/:prefix/realtime', realtime.getTripsEndpoint)
 router.post('/:prefix/realtimebypass', realtime.endpointBypass)
 router.post('/:prefix/vehicle_location', realtime.getVehicleLocation)

server/db/manual-shape-create.js

@@ -65,21 +65,6 @@ function cacheOperatorsAndShapes(prefix = 'nz-akl') {
       }
       getOperator(index + 1)
     }).catch(err => console.warn(err))
-
-    // caches the shape 
-    line._getLine(todo[index], function(err, data) {
-      if (err) {
-        console.warn(err)
-      }
-      shapeCount++
-      if (typeof(data[0]) !== 'undefined') {
-        shapesToCache.push({shape_id: data[0].shape_id})
-      }
-      if (todo.length === shapeCount) {
-        console.log('nz-akl:'.green, 'Collected List of Shapes To Cache')
-        line.cacheShapes(shapesToCache)
-      }
-    }, 'nz-akl')
   }
   getOperator(0)
 }
@@ -126,7 +111,7 @@ var line = {
   _getLine(lineId, cb, prefix = 'nz-akl') {
     const sqlRequest = connection.get().request()
     sqlRequest.input('prefix', sql.VarChar(50), prefix)
-    sqlRequest.input('version', sql.VarChar(50), cache.currentVersion())
+    sqlRequest.input('version', sql.VarChar(50), cache.currentVersion(prefix))
     sqlRequest.input('route_short_name', sql.VarChar(50), lineId)
     sqlRequest.query(`
       SELECT 
@@ -210,61 +195,18 @@ var line = {
       })
     })
   },
-  getShapeFromAt(arrayOfShapeId, cb) {
-    if (arrayOfShapeId.length === 0) {
-      return
-    }
-
-    var newOpts = JSON.parse(JSON.stringify(shapeWKBOptions))
-    let shape_id = arrayOfShapeId[0]
-    newOpts.url += shape_id
-    request(newOpts, function(err, response, body) {
-      if (err) {
-        console.warn(err)
-        console.log(`${shape_id} : Failed to get Shape`)
-        return line.getShapeFromAt(arrayOfShapeId.slice(1), cb)
+  getShapeJSON: function(req, res) {
+    let prefix = req.params.prefix
+    const containerName = encodeURIComponent((prefix+'-'+cache.currentVersion(prefix)).replace('_','-').replace('.','-'))
+    const shape_id = req.params.shape_id
+    const fileName = encodeURIComponent(shape_id+'.json')
+    blobSvc.getBlobToStream(containerName, fileName, res, function(blobError) {
+      if (blobError) {
+        res.status(404)
       }
-
-      // if AT doesn't send back a shape
-      let wkb = JSON.parse(body).response
-      if (wkb.length < 1) {
-        console.log(`${shape_id} : Shape not found!`)
-        return nextItem()
-      }
-
-      let nextItem = function() {
-        if (arrayOfShapeId.length === 1) {
-          // only calls back the last shape, for immediate return
-          if (cb) {
-            cb(wkb)
-          }
-          return
-        } else {
-          return line.getShapeFromAt(arrayOfShapeId.slice(1), cb)
-        }
-      }
-
-      blobSvc.createBlockBlobFromText('shapewkb', shape_id, wkb[0].the_geom, function(blobErr, blobResult, blobResponse) {
-        if (blobErr) {
-          console.warn(blobErr)
-          return nextItem()
-        }
-        nextItem()
-
-        // informs table storage that there is an item there
-        var task = {
-            PartitionKey: {'_': 'shapewkb'},
-            RowKey: {'_': shape_id},
-            date: {'_': new Date(), '$':'Edm.DateTime'}
-          }
-        tableSvc.insertOrReplaceEntity('meta', task, function (error) {
-          if (error) {
-            return console.warn(error)
-          }
-          console.log(`${shape_id} : Shape Saved from AT`)
-        })
-      })
-    })  
+      res.end()
+      return
+    })
   },
 
   exceptionCheck: function(route, bestMatchMode = false) {
@@ -352,41 +294,6 @@ var line = {
       req.params.trip_id =  trip_id
       line.getStopsFromTrip(req, res)
     })
-  },
-
-  cacheShapes: function(trips) {
-    // makes a priority list
-    let allShapes = {}
-    trips.forEach(function(trip) {
-      let shape = trip.shape_id
-      if (shape in allShapes) {
-        allShapes[shape] +=1
-      } else {
-        allShapes[shape] = 1
-      }
-    })
-    // flattens to priority array
-    let sorted = Object.keys(allShapes).sort(function(a, b) {
-      return allShapes[b] - allShapes[a]
-    }).map(function(sortedKey) {
-      return sortedKey
-    })
-    let promises = []
-    let requests = []
-    sorted.forEach(function(shape) {
-      promises.push(new Promise(function(resolve, reject) {
-        tableSvc.retrieveEntity('meta', 'shapewkb', shape, function(err, result, response) {
-          if (err) {
-            requests.push(shape)
-          }
-          resolve()
-        })
-      }))
-    })
-    // after it sees if they exist, fire the cache
-    Promise.all(promises).then(function() {
-      line.getShapeFromAt(requests)
-    })
   }
 }