From b4085e08fe5735df485db2bde90701e24df24eaa Mon Sep 17 00:00:00 2001
From: Alessandro Cerioni <acerioni@grandlyon.com>
Date: Sun, 3 Mar 2019 16:30:36 +0100
Subject: [PATCH] More robust handling of geometry. Handling of boolean and
date types.
---
5-pg-field-type-detector.py | 6 +++---
6-doc-processor.py | 4 +++-
utils/postgis_helper.py | 24 ++++++++++++++++--------
3 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/5-pg-field-type-detector.py b/5-pg-field-type-detector.py
index c1e848c..f3c85c6 100644
--- a/5-pg-field-type-detector.py
+++ b/5-pg-field-type-detector.py
@@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ):
else:
output = catalog.copy()
- # selected_schema = "abr_arbres_alignement"
- # selected_table = selected_schema + ".abrarbre"
+ # selected_schema = "rdata"
+ # selected_table = selected_schema + ".sit_sitra.sittourisme"
# found = False
logging.info('Getting schemas...')
@@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ):
count = 'unknown no. of'
#print(count)
- db_schema_table = '%s.%s' % (pg.dbname, table)
+ db_schema_table = '%s/%s' % (pg.dbname, table)
t2 = time.time()
logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count))
logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1)))
diff --git a/6-doc-processor.py b/6-doc-processor.py
index 2ce2c96..8409753 100644
--- a/6-doc-processor.py
+++ b/6-doc-processor.py
@@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ):
out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop])
elif out_types[lookup_key] == 'float':
out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop])
- elif out_types[lookup_key] == 'datetime':
+ elif out_types[lookup_key] in ['date', 'datetime']:
out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ')
+ elif out_types[lookup_key] == 'bool':
+ out_flattened_properties[prop] = convert_to_boolean(in_flattened_properties[prop])
else:
logging.critical('type %s not supported', out_types[prop])
sys.exit(1)
diff --git a/utils/postgis_helper.py b/utils/postgis_helper.py
index a0190a6..e558cba 100644
--- a/utils/postgis_helper.py
+++ b/utils/postgis_helper.py
@@ -88,21 +88,29 @@ class Remote(object):
selected = select(fields)
for entry in self.engine.execute(selected):
items = entry.items()
- #print(items)
- if geom is not None:
- try:
- geometry = json.loads(items.pop()[1])
- except TypeError:
- geom = None
-
properties = dict(items)
+ geometry = None
+ try:
+ geometry = json.loads(properties['ST_AsGeoJSON_1'])
+ del properties['ST_AsGeoJSON_1']
+ except:
+ pass
+
+ # #print(items)
+ # if geom is not None:
+ # try:
+ # geometry = json.loads(items.pop()[1])
+ # except TypeError:
+ # geom = None
+ #properties = dict(items)
+
document = {
'type': 'Feature',
'properties': properties
}
- if geom is not None:
+ if geometry is not None:
document['geometry'] = geometry
yield document
--
GitLab