Merge pull request #66 from openaddresses/support-lat-lon-in-esri

Support specifying lat and lon field names on Esri services

openaddr/cache.py

@@ -362,6 +362,7 @@ def download(self, source_urls, workdir, source_config):
         mkdirsp(download_path)
 
         query_fields = EsriRestDownloadTask.field_names_to_request(source_config)
+        conform = source_config.data_source.get('conform') or {}
 
         for source_url in source_urls:
             size = 0
@@ -400,12 +401,30 @@ def download(self, source_urls, workdir, source_config):
                         geom = feature.get('geometry') or {}
                         row = feature.get('properties') or {}
 
+                        # If the feature doesn't have a geometry, see if the conform has lat and lon fields specified
+                        # and try to build a point geometry from them
+                        if not geom and conform.get('lat') and conform.get('lon'):
+                            lat_field_name = conform['lat']
+                            lon_field_name = conform['lon']
+
+                            # Don't support functions to build the geometry yet
+                            if not isinstance(lat_field_name, str) or not isinstance(lon_field_name, str):
+                                raise TypeError("lat and lon don't support functions yet")
+
+                            try:
+                                geom = {
+                                    'type': 'Point',
+                                    'coordinates': [float(row.get(lon_field_name)), float(row.get(lat_field_name))]
+                                }
+                            except (TypeError, ValueError):
+                                raise TypeError("Couldn't build geometry from lat and lon fields")
+
                         if not geom:
                             raise TypeError("No geometry parsed")
                         if any((isinstance(g, float) and math.isnan(g)) for g in traverse(geom)):
                             raise TypeError("Geometry has NaN coordinates")
 
-                        shp = shape(feature['geometry'])
+                        shp = shape(geom)
                         row[GEOM_FIELDNAME] = shp.wkt
 
                         writer.writerow({fn: row.get(fn) for fn in field_names})

openaddr/tests/cache.py

@@ -1,9 +1,12 @@
 from __future__ import absolute_import, division, print_function
 
+import csv
+
 from .. import SourceConfig
 from urllib.parse import urlparse, parse_qs
 from os.path import join, dirname
 
+import json
 import shutil
 import mimetypes
 
@@ -67,6 +70,36 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.workdir)
 
+    def response_content(self, url, request):
+        ''' Fake HTTP responses for use with HTTMock in tests.
+        '''
+        scheme, host, path, _, query, _ = urlparse(url.geturl())
+        tests_dirname = dirname(__file__)
+        data_dirname = join(tests_dirname, 'data')
+        local_path = None
+
+        if (host, path) == ('web2.kcsgis.com', '/kcsgis/rest/services/Cullman/VAM_Cullman_FS/FeatureServer/4'):
+            qs = parse_qs(query)
+
+            if qs.get('f') == ['json']:
+                local_path = join(data_dirname, 'us-al-cullman-metadata.json')
+
+        if (host, path) == ('web2.kcsgis.com', '/kcsgis/rest/services/Cullman/VAM_Cullman_FS/FeatureServer/4/query'):
+            qs = parse_qs(query)
+            body_qs = parse_qs(request.body)
+
+            if qs.get('returnCountOnly') == ['true']:
+                local_path = join(data_dirname, 'us-al-cullman-count-only.json')
+            if request.method == 'POST' and body_qs.get('resultOffset') == ['0']:
+                local_path = join(data_dirname, 'us-al-cullman-0.json')
+
+        if local_path:
+            type, _ = mimetypes.guess_type(local_path)
+            with open(local_path, 'rb') as file:
+                return httmock.response(200, file.read(), headers={'Content-Type': type})
+
+        raise NotImplementedError(url.geturl())
+
     def test_download_with_conform(self):
         """ ESRI Caching Will Request With The Minimum Fields Required """
         conforms = (
@@ -294,3 +327,31 @@ def test_field_names_to_request(self):
         }), "addresses", "default")
         fields10 = EsriRestDownloadTask.field_names_to_request(conform10)
         self.assertEqual(fields10, ['Street'])
+
+    def test_handle_feature_server_with_lat_lon_in_conform(self):
+        '''
+        '''
+        task = EsriRestDownloadTask('us-fl-palmbeach')
+        c = SourceConfig(dict({
+            "schema": 2,
+            "layers": {
+                "addresses": [{
+                    "name": "default",
+                    "conform": {
+                        "lat": "LAT",
+                        "lon": "LON"
+                    }
+                }]
+            }
+        }), "addresses", "default")
+        with httmock.HTTMock(self.response_content):
+            output_path = task.download(["https://web2.kcsgis.com/kcsgis/rest/services/Cullman/VAM_Cullman_FS/FeatureServer/4"], self.workdir, c)
+            self.assertEqual(len(output_path), 1)
+
+            # Load the downloaded CSV and check the geometry
+            with open(output_path[0], 'r') as file:
+                reader = csv.DictReader(file)
+                all_data = list(reader)
+                self.assertEqual(len(all_data),  5)
+                self.assertTrue('oa:geom' in all_data[0])
+                self.assertEqual(all_data[0]['oa:geom'], 'POINT (-86.82960553 34.18671398)')

openaddr/tests/data/us-al-cullman-0.json

@@ -0,0 +1 @@
+{"objectIdFieldName":"OBJECTID","globalIdFieldName":"GlobalID","fields":[{"name":"OBJECTID","alias":"OBJECTID","type":"esriFieldTypeOID"},{"name":"StNum","alias":"StNum","type":"esriFieldTypeInteger"},{"name":"StPre","alias":"StPre","type":"esriFieldTypeString","length":12},{"name":"STNAME","alias":"STNAME","type":"esriFieldTypeString","length":101},{"name":"StType","alias":"StType","type":"esriFieldTypeString","length":40},{"name":"StPost","alias":"StPost","type":"esriFieldTypeString","length":12},{"name":"City","alias":"City","type":"esriFieldTypeString","length":25},{"name":"StUnit","alias":"StUnit","type":"esriFieldTypeString","length":65},{"name":"ZipCode","alias":"ZipCode","type":"esriFieldTypeInteger"},{"name":"SourceAccuracy","alias":"SourceAccuracy","type":"esriFieldTypeString","length":1073741822},{"name":"LAT","alias":"LAT","type":"esriFieldTypeDouble"},{"name":"LON","alias":"LON","type":"esriFieldTypeDouble"},{"name":"FULLNAME","alias":"FULLNAME","type":"esriFieldTypeString","length":100},{"name":"MAFID","alias":"MAFID","type":"esriFieldTypeInteger"},{"name":"ACTION_","alias":"ACTION_","type":"esriFieldTypeString","length":1},{"name":"STATEFP","alias":"STATEFP","type":"esriFieldTypeString","length":2},{"name":"COUNTYFP","alias":"COUNTYFP","type":"esriFieldTypeString","length":3},{"name":"TRACT","alias":"TRACT","type":"esriFieldTypeString","length":6},{"name":"BLOCK","alias":"BLOCK","type":"esriFieldTypeString","length":6},{"name":"GEOID","alias":"GEOID","type":"esriFieldTypeString","length":17},{"name":"GQ_FLAG","alias":"GQ_FLAG","type":"esriFieldTypeString","length":1},{"name":"GQ_NAME","alias":"GQ_NAME","type":"esriFieldTypeString","length":100},{"name":"FACILITY_NAME","alias":"FACILITY_NAME","type":"esriFieldTypeString","length":100},{"name":"LOCATION_DESCRIPTION","alias":"LOCATION_DESCRIPTION","type":"esriFieldTypeString","length":100},{"name":"NONCITYSTYLE_ADDRESS","alias":"NONCITYSTYLE_ADDRESS","type":"esriFieldTypeString","length":50},{"name":"NONCITYSTYLE_ZIP","alias":"NONCITYSTYLE_ZIP","type":"esriFieldTypeInteger"},{"name":"MAPSPOT","alias":"MAPSPOT","type":"esriFieldTypeSmallInteger"},{"name":"ADDRESS_USE","alias":"ADDRESS_USE","type":"esriFieldTypeString","length":1},{"name":"CITY_STYLE","alias":"CITY_STYLE","type":"esriFieldTypeString","length":1},{"name":"GlobalID","alias":"GlobalID","type":"esriFieldTypeGlobalID","length":38},{"name":"created_user","alias":"created_user","type":"esriFieldTypeString","length":255},{"name":"created_date","alias":"created_date","type":"esriFieldTypeDate","length":8},{"name":"last_edited_user","alias":"last_edited_user","type":"esriFieldTypeString","length":255},{"name":"last_edited_date","alias":"last_edited_date","type":"esriFieldTypeDate","length":8},{"name":"MAPID","alias":"MAPID","type":"esriFieldTypeInteger"},{"name":"EditType","alias":"EditType","type":"esriFieldTypeString","length":2},{"name":"KickoutDate","alias":"KickoutDate","type":"esriFieldTypeDate","length":8},{"name":"KickoutID","alias":"KickoutID","type":"esriFieldTypeSmallInteger"},{"name":"UserID","alias":"UserID","type":"esriFieldTypeSmallInteger"},{"name":"LocationType","alias":"LocationType","type":"esriFieldTypeString","length":20}],"features":[{"attributes":{"OBJECTID":64746,"StNum":888,"StPre":"N","STNAME":"CRAIGTEST","StType":"LN","StPost":"S","City":"CLINTON","StUnit":"UNIT B","ZipCode":35007,"SourceAccuracy":"OFFICE","LAT":34.18671398,"LON":-86.829605529999995,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":"964900","BLOCK":"4018","GEOID":"010439649004018","GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"GlobalID":"{A05ACB53-D3F5-4459-838D-3E9486BA3E15}","created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":47656,"EditType":"D","KickoutDate":null,"KickoutID":null,"UserID":0,"LocationType":"RESIDENTIAL"}},{"attributes":{"OBJECTID":64747,"StNum":1010,"StPre":"N","STNAME":"CRAIGTEST","StType":"LN","StPost":"S","City":"CLINTON","StUnit":"UNIT B","ZipCode":35007,"SourceAccuracy":"OFFICE","LAT":34.186989099999998,"LON":-86.830270720000001,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":"964900","BLOCK":"4003","GEOID":"010439649004003","GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"GlobalID":"{17EC4274-0B9D-43C3-9F9A-63DF94E13ED4}","created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":47657,"EditType":"D","KickoutDate":null,"KickoutID":null,"UserID":0,"LocationType":"RESIDENTIAL"}},{"attributes":{"OBJECTID":64748,"StNum":527,"StPre":"","STNAME":"COUNTY ROAD 1464","StType":"","StPost":"NE","City":"CULLMAN","StUnit":"","ZipCode":35058,"SourceAccuracy":"OFFICE","LAT":34.206364659999998,"LON":-86.804870919999999,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":"964900","BLOCK":"1026","GEOID":"010439649001026","GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"GlobalID":"{7688C747-EFBB-455B-B900-88D77F1D7AC6}","created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":47658,"EditType":"A","KickoutDate":null,"KickoutID":0,"UserID":0,"LocationType":"RESIDENTIAL"}},{"attributes":{"OBJECTID":64749,"StNum":55,"StPre":"","STNAME":"COUNTY RD 1571","StType":"","StPost":"","City":"CULLMAN","StUnit":"","ZipCode":35058,"SourceAccuracy":"OFFICE","LAT":34.214512130000003,"LON":-86.828527780000002,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":"964900","BLOCK":"1058","GEOID":"010439649001058","GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"GlobalID":"{F8FC5609-0071-41DE-94BF-F53F1ADA4D4B}","created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":47659,"EditType":"A","KickoutDate":null,"KickoutID":0,"UserID":0,"LocationType":"RESIDENTIAL"}},{"attributes":{"OBJECTID":64750,"StNum":295,"StPre":"","STNAME":"COUNTY RD 1273","StType":"","StPost":"","City":"CULLMAN","StUnit":"","ZipCode":35057,"SourceAccuracy":"OFFICE","LAT":34.234429499999997,"LON":-86.894148459999997,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":"964400","BLOCK":"1072","GEOID":"010439644001072","GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"GlobalID":"{7DAC2835-7B24-43B0-BEFC-6F10B35903D4}","created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":47660,"EditType":"A","KickoutDate":null,"KickoutID":0,"UserID":0,"LocationType":"RESIDENTIAL"}}],"exceededTransferLimit":false}

openaddr/tests/data/us-al-cullman-count-only.json

@@ -0,0 +1 @@
+{"count":5}

openaddr/tests/data/us-al-cullman-metadata.json

@@ -0,0 +1 @@
+{"currentVersion":10.51,"id":4,"name":"Cullman Addresses","type":"Table","description":"","copyrightText":"","editFieldsInfo":null,"ownershipBasedAccessControlForFeatures":null,"syncCanReturnChanges":false,"relationships":[],"isDataVersioned":false,"supportsRollbackOnFailureParameter":true,"archivingInfo":{"supportsQueryWithHistoricMoment":false,"startArchivingMoment":-1},"supportsStatistics":true,"supportsAdvancedQueries":true,"supportsValidateSQL":true,"supportsCalculate":true,"advancedQueryCapabilities":{"supportsPagination":true,"supportsTrueCurve":true,"supportsQueryWithDistance":true,"supportsReturningQueryExtent":true,"supportsStatistics":true,"supportsOrderBy":true,"supportsDistinct":true,"supportsSqlExpression":true},"hasAttachments":false,"supportsApplyEditsWithGlobalIds":false,"htmlPopupType":"esriServerHTMLPopupTypeNone","objectIdField":"OBJECTID","globalIdField":"GlobalID","displayField":"STNAME","typeIdField":"","subtypeField":"","fields":[{"name":"OBJECTID","type":"esriFieldTypeOID","alias":"OBJECTID","domain":null,"editable":false,"nullable":false},{"name":"StNum","type":"esriFieldTypeInteger","alias":"StNum","domain":null,"editable":true,"nullable":true},{"name":"StPre","type":"esriFieldTypeString","alias":"StPre","domain":null,"editable":true,"nullable":true,"length":12},{"name":"STNAME","type":"esriFieldTypeString","alias":"STNAME","domain":null,"editable":true,"nullable":true,"length":101},{"name":"StType","type":"esriFieldTypeString","alias":"StType","domain":null,"editable":true,"nullable":true,"length":40},{"name":"StPost","type":"esriFieldTypeString","alias":"StPost","domain":null,"editable":true,"nullable":true,"length":12},{"name":"City","type":"esriFieldTypeString","alias":"City","domain":null,"editable":true,"nullable":false,"length":25},{"name":"StUnit","type":"esriFieldTypeString","alias":"StUnit","domain":null,"editable":true,"nullable":true,"length":65},{"name":"ZipCode","type":"esriFieldTypeInteger","alias":"ZipCode","domain":null,"editable":true,"nullable":true},{"name":"SourceAccuracy","type":"esriFieldTypeString","alias":"SourceAccuracy","domain":null,"editable":true,"nullable":true,"length":1073741822},{"name":"LAT","type":"esriFieldTypeDouble","alias":"LAT","domain":null,"editable":true,"nullable":true},{"name":"LON","type":"esriFieldTypeDouble","alias":"LON","domain":null,"editable":true,"nullable":true},{"name":"FULLNAME","type":"esriFieldTypeString","alias":"FULLNAME","domain":null,"editable":true,"nullable":true,"length":100},{"name":"MAFID","type":"esriFieldTypeInteger","alias":"MAFID","domain":null,"editable":true,"nullable":true},{"name":"ACTION_","type":"esriFieldTypeString","alias":"ACTION_","domain":null,"editable":true,"nullable":true,"length":1},{"name":"STATEFP","type":"esriFieldTypeString","alias":"STATEFP","domain":null,"editable":true,"nullable":true,"length":2},{"name":"COUNTYFP","type":"esriFieldTypeString","alias":"COUNTYFP","domain":null,"editable":true,"nullable":true,"length":3},{"name":"TRACT","type":"esriFieldTypeString","alias":"TRACT","domain":null,"editable":true,"nullable":true,"length":6},{"name":"BLOCK","type":"esriFieldTypeString","alias":"BLOCK","domain":null,"editable":true,"nullable":true,"length":6},{"name":"GEOID","type":"esriFieldTypeString","alias":"GEOID","domain":null,"editable":true,"nullable":true,"length":17},{"name":"GQ_FLAG","type":"esriFieldTypeString","alias":"GQ_FLAG","domain":null,"editable":true,"nullable":true,"length":1},{"name":"GQ_NAME","type":"esriFieldTypeString","alias":"GQ_NAME","domain":null,"editable":true,"nullable":true,"length":100},{"name":"FACILITY_NAME","type":"esriFieldTypeString","alias":"FACILITY_NAME","domain":null,"editable":true,"nullable":true,"length":100},{"name":"LOCATION_DESCRIPTION","type":"esriFieldTypeString","alias":"LOCATION_DESCRIPTION","domain":null,"editable":true,"nullable":true,"length":100},{"name":"NONCITYSTYLE_ADDRESS","type":"esriFieldTypeString","alias":"NONCITYSTYLE_ADDRESS","domain":null,"editable":true,"nullable":true,"length":50},{"name":"NONCITYSTYLE_ZIP","type":"esriFieldTypeInteger","alias":"NONCITYSTYLE_ZIP","domain":null,"editable":true,"nullable":true},{"name":"MAPSPOT","type":"esriFieldTypeSmallInteger","alias":"MAPSPOT","domain":null,"editable":true,"nullable":true},{"name":"ADDRESS_USE","type":"esriFieldTypeString","alias":"ADDRESS_USE","domain":null,"editable":true,"nullable":true,"length":1},{"name":"CITY_STYLE","type":"esriFieldTypeString","alias":"CITY_STYLE","domain":null,"editable":true,"nullable":true,"length":1},{"name":"GlobalID","type":"esriFieldTypeGlobalID","alias":"GlobalID","domain":null,"editable":false,"nullable":false,"length":38},{"name":"created_user","type":"esriFieldTypeString","alias":"created_user","domain":null,"editable":true,"nullable":true,"length":255},{"name":"created_date","type":"esriFieldTypeDate","alias":"created_date","domain":null,"editable":true,"nullable":true,"length":8},{"name":"last_edited_user","type":"esriFieldTypeString","alias":"last_edited_user","domain":null,"editable":true,"nullable":true,"length":255},{"name":"last_edited_date","type":"esriFieldTypeDate","alias":"last_edited_date","domain":null,"editable":true,"nullable":true,"length":8},{"name":"MAPID","type":"esriFieldTypeInteger","alias":"MAPID","domain":null,"editable":true,"nullable":false},{"name":"EditType","type":"esriFieldTypeString","alias":"EditType","domain":null,"editable":true,"nullable":true,"length":2},{"name":"KickoutDate","type":"esriFieldTypeDate","alias":"KickoutDate","domain":null,"editable":true,"nullable":true,"length":8},{"name":"KickoutID","type":"esriFieldTypeSmallInteger","alias":"KickoutID","domain":null,"editable":true,"nullable":true},{"name":"UserID","type":"esriFieldTypeSmallInteger","alias":"UserID","domain":null,"editable":true,"nullable":true},{"name":"LocationType","type":"esriFieldTypeString","alias":"LocationType","domain":null,"editable":true,"nullable":true,"length":20}],"indexes":[{"name":"R23_SDE_ROWID_UK","fields":"OBJECTID","isAscending":true,"isUnique":true,"description":""},{"name":"UUID_23","fields":"GlobalID","isAscending":true,"isUnique":false,"description":""}],"dateFieldsTimeReference":{"timeZone":"UTC","respectsDaylightSaving":false},"types":[],"templates":[{"name":"Cullman Addresses","description":"","prototype":{"attributes":{"UserID":0,"LocationType":null,"StNum":null,"StPre":null,"STNAME":null,"StType":null,"StPost":null,"City":" ","StUnit":null,"ZipCode":null,"SourceAccuracy":null,"LAT":null,"LON":null,"FULLNAME":null,"MAFID":null,"ACTION_":null,"STATEFP":null,"COUNTYFP":null,"TRACT":null,"BLOCK":null,"GEOID":null,"GQ_FLAG":null,"GQ_NAME":null,"FACILITY_NAME":null,"LOCATION_DESCRIPTION":null,"NONCITYSTYLE_ADDRESS":null,"NONCITYSTYLE_ZIP":null,"MAPSPOT":null,"ADDRESS_USE":null,"CITY_STYLE":null,"KickoutID":0,"created_user":null,"created_date":null,"last_edited_user":null,"last_edited_date":null,"MAPID":0,"EditType":"''","KickoutDate":null}},"drawingTool":"esriFeatureEditToolNone"}],"maxRecordCount":100000,"supportedQueryFormats":"JSON, AMF, geoJSON","capabilities":"Create,Delete,Query,Update,Uploads,Editing","useStandardizedQueries":true}