Skip to content
Snippets Groups Projects
Commit b4085e08 authored by Alessandro Cerioni's avatar Alessandro Cerioni
Browse files

More robust handling of geometry. Handling of boolean and date types.

parent f6d3819f
No related branches found
No related tags found
No related merge requests found
...@@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ): ...@@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ):
else: else:
output = catalog.copy() output = catalog.copy()
# selected_schema = "abr_arbres_alignement" # selected_schema = "rdata"
# selected_table = selected_schema + ".abrarbre" # selected_table = selected_schema + ".sit_sitra.sittourisme"
# found = False # found = False
logging.info('Getting schemas...') logging.info('Getting schemas...')
...@@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ): ...@@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ):
count = 'unknown no. of' count = 'unknown no. of'
#print(count) #print(count)
db_schema_table = '%s.%s' % (pg.dbname, table) db_schema_table = '%s/%s' % (pg.dbname, table)
t2 = time.time() t2 = time.time()
logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count)) logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count))
logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1))) logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1)))
......
...@@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ): ...@@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ):
out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop]) out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop])
elif out_types[lookup_key] == 'float': elif out_types[lookup_key] == 'float':
out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop]) out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop])
elif out_types[lookup_key] == 'datetime': elif out_types[lookup_key] in ['date', 'datetime']:
out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ') out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ')
elif out_types[lookup_key] == 'bool':
out_flattened_properties[prop] = convert_to_boolean(in_flattened_properties[prop])
else: else:
logging.critical('type %s not supported', out_types[prop]) logging.critical('type %s not supported', out_types[prop])
sys.exit(1) sys.exit(1)
......
...@@ -88,21 +88,29 @@ class Remote(object): ...@@ -88,21 +88,29 @@ class Remote(object):
selected = select(fields) selected = select(fields)
for entry in self.engine.execute(selected): for entry in self.engine.execute(selected):
items = entry.items() items = entry.items()
#print(items)
if geom is not None:
try:
geometry = json.loads(items.pop()[1])
except TypeError:
geom = None
properties = dict(items) properties = dict(items)
geometry = None
try:
geometry = json.loads(properties['ST_AsGeoJSON_1'])
del properties['ST_AsGeoJSON_1']
except:
pass
# #print(items)
# if geom is not None:
# try:
# geometry = json.loads(items.pop()[1])
# except TypeError:
# geom = None
#properties = dict(items)
document = { document = {
'type': 'Feature', 'type': 'Feature',
'properties': properties 'properties': properties
} }
if geom is not None: if geometry is not None:
document['geometry'] = geometry document['geometry'] = geometry
yield document yield document
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment