Add better debugging to help troubleshoot common errors

Also add list of lookup libraries.

Fixes #27
Fixes #30

4 files changed, 61 insertions, 6 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2d8774..622745e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+## Unreleased
+
+### Other Changes
+
+* Better debugging
+  * Printing Overpass query for when no data is received.
+  * Using geojsonhint and writing problematic geojson to a pretty-printed file.
+  * Save validation overlaps to file.
+  * Add links to troubleshooting wiki for common errors.
+* Update README
+  * Add list of geogrphical lookup libraries
+  * Add troubleshooting wiki
+
 ## 2017c
 
 ### Zone Changes
diff --git a/README.md b/README.md
index b756e71..aa168e3 100644
--- a/README.md
+++ b/README.md
@@ -14,9 +14,23 @@ The underlying data is download from [OpenStreetMap](http://www.openstreetmap.or
 
 To maintain consistency with the timezone database, this project will only create a new release after the timezone database creates a new release.  If there are no new timezones created or deleted in a timezone database release, then this project will only create a release if there have been changes performed to the boundary definitions of an existing zone within this project.
 
+## Lookup Libraries
+
+A few common languages already have libraries with an API that can be used to lookup the timezone name at a particular GPS coordinate.  Here are some libraries that use the data produced by timezone-boundary-builder:
+
+| Library | Language |
+| -- | -- |
+| [ZoneDetect](https://github.com/BertoldVdb/ZoneDetect) | C |
+| [node-geo-tz](https://github.com/evansiroky/node-geo-tz/) | JavaScript (node.js only) |
+| [tz-lookup](https://github.com/darkskyapp/tz-lookup/) | JavaScript (node.js and in browser) |
+| [timezonefinder](https://github.com/MrMinimal64/timezonefinder) | Python |
+| [GeoTimezone](https://github.com/mj1856/GeoTimeZone) | .NET |
+
+Another common way to use the data for lookup purposes is to load the shapefile into a spatially-aware database.  See this [blog post](https://simonwillison.net/2017/Dec/12/location-time-zone-api/) for an example of how that can be done.
+
 ## Running the script
 
-It is possible to run the script to generate the timezones.  Due to the ever-changing nature of OpenStreetMap, the script can break when unexpected data is received.
+If the data in the releases are not sufficiently recent or you want to build the latest from master, it is possible to run the script to generate the timezones.  However, due to the ever-changing nature of OpenStreetMap, the script should be considered unstable.  The script frequently breaks when unexpected data is received or changes in OpenStreetMap cause validation issues.  Please see the [troubleshooting guide](https://github.com/evansiroky/timezone-boundary-builder/wiki/Troubleshooting) for help with common errors.
 
 **Run the script to generate timezones for all timezones.**
 
diff --git a/index.js b/index.js
index 7782b52..2192e11 100644
--- a/index.js
+++ b/index.js
@@ -2,6 +2,7 @@ var exec = require('child_process').exec
 var fs = require('fs')
 
 var area = require('@mapbox/geojson-area')
+var geojsonhint = require('@mapbox/geojsonhint')
 var helpers = require('@turf/helpers')
 var multiPolygon = helpers.multiPolygon
 var polygon = helpers.polygon
@@ -185,10 +186,17 @@ var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
     },
     validateOverpassResult: ['downloadFromOverpass', function (results, cb) {
       var data = results.downloadFromOverpass
-      if (!data.features || data.features.length === 0) {
+      if (!data.features) {
         var err = new Error('Invalid geojson for boundary: ' + boundaryId)
         return cb(err)
       }
+      if (data.features.length === 0) {
+        console.error('No data for the following query:')
+        console.error(query)
+        console.error('To read more about this error, please visit https://git.io/vxKQL')
+        var err = new Error('No data found for from overpass query')
+        return cb(err)
+      }
       cb()
     }],
     saveSingleMultiPolygon: ['validateOverpassResult', function (results, cb) {
@@ -200,14 +208,26 @@ var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
         var curOsmGeom = data.features[i].geometry
         if (curOsmGeom.type === 'Polygon' || curOsmGeom.type === 'MultiPolygon') {
           console.log('combining border')
+          let errors = geojsonhint.hint(curOsmGeom)
+          if (errors && errors.length > 0) {
+            const stringifiedGeojson = JSON.stringify(curOsmGeom, null, 2)
+            errors = geojsonhint.hint(stringifiedGeojson)
+            console.error('Invalid geojson received in Overpass Result')
+            console.error('Overpass query: ' + query)
+            const problemFilename = boundaryId + '_convert_to_geom_error.json'
+            fs.writeFileSync(problemFilename, stringifiedGeojson)
+            console.error('saved problem file to ' + problemFilename)
+            console.error('To read more about this error, please visit https://git.io/vxKQq')
+            return cb(errors)
+          }
           try {
             var curGeom = geoJsonToGeom(curOsmGeom)
           } catch (e) {
             console.error('error converting overpass result to geojson')
             console.error(e)
-            fs.writeFileSync(boundaryId + '_convert_to_geom_error.json', JSON.stringify(curOsmGeom))
+
             fs.writeFileSync(boundaryId + '_convert_to_geom_error-all-features.json', JSON.stringify(data))
-            throw e
+            return cb(e)
           }
           if (!combined) {
             combined = curGeom
@@ -221,7 +241,7 @@ var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
       } catch (e) {
         console.error('error writing combined border to geojson')
         fs.writeFileSync(boundaryId + '_combined_border_convert_to_geom_error.json', JSON.stringify(data))
-        throw e
+        return cb(e)
       }
     }]
   }, boundaryCallback)
@@ -392,7 +412,14 @@ var validateTimezoneBoundaries = function () {
         var intersectedArea = intersectedGeom.getArea()
 
         if (intersectedArea > 0.0001) {
-          console.log('Validation error: ' + tzid + ' intersects ' + compareTzid + ' area: ' + intersectedArea)
+          console.error('Validation error: ' + tzid + ' intersects ' + compareTzid + ' area: ' + intersectedArea)
+          const debugFilename = tzid.replace('/', '-') + '-' + compareTzid.replace('/', '-') + '-overlap.json'
+          fs.writeFileSync(
+            debugFilename,
+            JSON.stringify(geoJsonWriter.write(intersectedGeom))
+          )
+          console.error('wrote overlap area as file ' + debugFilename)
+          console.error('To read more about this error, please visit https://git.io/vx6nx')
           allZonesOk = false
         }
       }
diff --git a/package.json b/package.json
index 3483e5f..8b01e83 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
   "license": "MIT",
   "dependencies": {
     "@mapbox/geojson-area": "^0.2.2",
+    "@mapbox/geojsonhint": "^2.1.0",
     "@turf/helpers": "^3.10.3",
     "async": "^2.1.5",
     "jsts": "^1.3.0",