diff options
author | evansiroky <evan.siroky@yahoo.com> | 2019-01-05 15:39:28 -0800 |
---|---|---|
committer | evansiroky <evan.siroky@yahoo.com> | 2019-01-05 15:39:28 -0800 |
commit | 5348a6fd04c857165b592a302c26a7ece02c9e46 (patch) | |
tree | 232269701de08f15f736edbb85e1f78ad3cbaf10 | |
parent | 45bbec0aa7ee5f60354b9803c4ff79d4d63689a9 (diff) | |
download | timezone-boundary-builder-5348a6fd04c857165b592a302c26a7ece02c9e46.tar.gz |
Refactor progress reporting to cover all build phases
-rw-r--r-- | index.js | 61 | ||||
-rw-r--r-- | progressStats.js | 49 |
2 files changed, 79 insertions, 31 deletions
@@ -12,13 +12,12 @@ var jsts = require('jsts') var rimraf = require('rimraf') var overpass = require('query-overpass') +const ProgressStats = require('./progressStats') + var osmBoundarySources = require('./osmBoundarySources.json') var zoneCfg = require('./timezones.json') var expectedZoneOverlaps = require('./expectedZoneOverlaps.json') -const ProgressStats = require('./progressStats') -var progressStats = new ProgressStats(Object.keys(osmBoundarySources).length) - // allow building of only a specified zones var filteredIndex = process.argv.indexOf('--filtered-zones') if (filteredIndex > -1 && process.argv[filteredIndex + 1]) { @@ -140,6 +139,11 @@ var geomToGeoJsonString = function (geom) { return JSON.stringify(geoJsonWriter.write(geom)) } +const downloadProgress = new ProgressStats( + 'Downloading', + Object.keys(osmBoundarySources).length +) + var downloadOsmBoundary = function (boundaryId, boundaryCallback) { var cfg = osmBoundarySources[boundaryId] var query = '[out:json][timeout:60];(' @@ -162,12 +166,11 @@ var downloadOsmBoundary = function (boundaryId, boundaryCallback) { query += ';);out body;>;out meta qt;' - console.log(debug) + downloadProgress.beginTask(debug, true) asynclib.auto({ downloadFromOverpass: function (cb) { - progressStats.logNext() - console.log('downloading from overpass; overall progress ' + progressStats.getPercentage() + '% done - ' + progressStats.getTimeLeft(5) + ' left') + console.log('downloading from overpass') fetchIfNeeded(boundaryFilename, boundaryCallback, cb, function () { var overpassResponseHandler = function (err, data) { if (err) { @@ -348,8 +351,13 @@ var postProcessZone = function (geom, returnAsObject) { return returnAsObject ? newGeojson : JSON.stringify(newGeojson) } +const buildingProgress = new ProgressStats( + 'Building', + Object.keys(zoneCfg).length +) + var makeTimezoneBoundary = function (tzid, callback) { - console.log('makeTimezoneBoundary for', tzid) + buildingProgress.beginTask('makeTimezoneBoundary for', tzid) var ops = zoneCfg[tzid] var geom @@ -414,6 +422,12 @@ var formatBounds = function (bounds) { } var validateTimezoneBoundaries = function () { + const numZones = Object.keys(zoneCfg).length + const validationProgress = new ProgressStats( + 'Validation', + numZones * (numZones + 1) / 2 + ) + console.log('do validation... this may take a few minutes') var allZonesOk = true var zones = Object.keys(zoneCfg) @@ -424,6 +438,9 @@ var validateTimezoneBoundaries = function () { zoneGeom = getDistZoneGeom(tzid) for (var j = i + 1; j < zones.length; j++) { + if (Math.round(validationProgress.getPercentage()) % 10 === 0) { + validationProgress.printStats('Validating zones', true) + } compareTzid = zones[j] var compareZoneGeom = getDistZoneGeom(compareTzid) @@ -516,6 +533,7 @@ var validateTimezoneBoundaries = function () { allZonesOk = false } } + validationProgress.logNext() } } @@ -623,25 +641,25 @@ var combineAndWriteZones = function (callback) { ], callback) } -asynclib.auto({ +const autoScript = { makeDownloadsDir: function (cb) { - console.log('creating downloads dir') + overallProgress.beginTask('Creating downloads dir') safeMkdir('./downloads', cb) }, makeDistDir: function (cb) { - console.log('createing dist dir') + overallProgress.beginTask('Creating dist dir') safeMkdir('./dist', cb) }, getOsmBoundaries: ['makeDownloadsDir', function (results, cb) { - console.log('downloading osm boundaries') + overallProgress.beginTask('Downloading osm boundaries') asynclib.eachSeries(Object.keys(osmBoundarySources), downloadOsmBoundary, cb) }], createZones: ['makeDistDir', 'getOsmBoundaries', function (results, cb) { - console.log('createZones') + overallProgress.beginTask('Creating timezone boundaries') asynclib.each(Object.keys(zoneCfg), makeTimezoneBoundary, cb) }], validateZones: ['createZones', function (results, cb) { - console.log('validating zones') + overallProgress.beginTask('Validating timezone boundaries') loadDistZonesIntoMemory() if (process.argv.indexOf('no-validation') > -1) { console.warn('WARNING: Skipping validation!') @@ -651,22 +669,23 @@ asynclib.auto({ } }], addOceans: ['validateZones', function (results, cb) { + overallProgress.beginTask('Adding oceans') addOceans(cb) }], mergeZones: ['addOceans', function (results, cb) { - console.log('merge zones') + overallProgress.beginTask('Merging zones') combineAndWriteZones(cb) }], zipGeoJson: ['mergeZones', function (results, cb) { - console.log('zip geojson') + overallProgress.beginTask('Zipping geojson') exec('zip dist/timezones.geojson.zip dist/combined.json', cb) }], zipGeoJsonWithOceans: ['mergeZones', function (results, cb) { - console.log('zip geojson with oceans') + overallProgress.beginTask('Zipping geojson with oceans') exec('zip dist/timezones-with-oceans.geojson.zip dist/combined-with-oceans.json', cb) }], makeShapefile: ['mergeZones', function (results, cb) { - console.log('convert from geojson to shapefile') + overallProgress.beginTask('Converting from geojson to shapefile') rimraf.sync('dist/combined-shapefile.*') exec( 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile.shp dist/combined.json', @@ -677,7 +696,7 @@ asynclib.auto({ ) }], makeShapefileWithOceans: ['mergeZones', function (results, cb) { - console.log('convert from geojson with oceans to shapefile') + overallProgress.beginTask('Converting from geojson with oceans to shapefile') rimraf.sync('dist/combined-shapefile-with-oceans.*') exec( 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile-with-oceans.shp dist/combined-with-oceans.json', @@ -687,7 +706,11 @@ asynclib.auto({ } ) }] -}, function (err, results) { +} + +const overallProgress = new ProgressStats('Overall', Object.keys(autoScript).length) + +asynclib.auto(autoScript, function (err, results) { console.log('done') if (err) { console.log('error!', err) diff --git a/progressStats.js b/progressStats.js index 64b50e1..a55853f 100644 --- a/progressStats.js +++ b/progressStats.js @@ -1,25 +1,50 @@ class ProgressStats { - constructor(totalDownloads) { - this.totalDownloads = totalDownloads - this.downloadCounter = 0 + constructor(trackerName, totalTasks) { + this.trackerName = trackerName + this.totalTasks = totalTasks + this.taskCounter = 0 + this.referenceLength = 5 this.timestamps = [] } logNext() { - this.downloadCounter++ + this.taskCounter++ this.timestamps.push({ - "index": this.downloadCounter, + "index": this.taskCounter, "timestamp": Date.now() }) } /** + * Begin a new task. Print the current progress and then increment the number of tasks. + * @param {string} A short message about the current task progress + * @param {[boolean]} logTimeLeft whether or not to log the time left. + */ + beginTask (message, logTimeLeft) { + this.printStats(message, logTimeLeft) + this.logNext() + } + + /** + * Print the current progress. + * @param {string} A short message about the current task progress + * @param {[boolean]} logTimeLeft whether or not to log the time left. + */ + printStats (message, logTimeLeft) { + message = `${message}; ${this.trackerName} progress: ${this.getPercentage()}% done` + if (logTimeLeft) { + message = `${message} - ${this.getTimeLeft()} left` + } + console.log(message) + } + + /** * calculates the percentage of finished downloads * @returns {string} */ getPercentage() { - var current = (this.downloadCounter / this.totalDownloads) + var current = (this.taskCounter / this.totalTasks) return Math.round(current * 1000.0) / 10.0 } @@ -29,8 +54,8 @@ class ProgressStats { * * @returns {string} */ - getTimeLeft(referenceLength) { - if(this.downloadCounter <= referenceLength){ + getTimeLeft () { + if(this.taskCounter <= this.referenceLength) { //number of reference downloads must exist before left time can be predicted return "? minutes" } @@ -41,12 +66,12 @@ class ProgressStats { } var indexOfStepsBefore = this.timestamps.findIndex((t) => { - return t.index == (this.downloadCounter - referenceLength) + return t.index === (this.taskCounter - this.referenceLength) }) var lastSteps = this.timestamps[indexOfStepsBefore]; var millisOflastSteps = Date.now() - lastSteps.timestamp - var downloadsLeft = this.totalDownloads - this.downloadCounter - var millisecondsLeft = (millisOflastSteps / referenceLength) * downloadsLeft + var downloadsLeft = this.totalTasks - this.taskCounter + var millisecondsLeft = (millisOflastSteps / this.referenceLength) * downloadsLeft return this.formatMilliseconds(millisecondsLeft) } @@ -73,4 +98,4 @@ class ProgressStats { } -module.exports = ProgressStats
\ No newline at end of file +module.exports = ProgressStats |