From b4085e08fe5735df485db2bde90701e24df24eaa Mon Sep 17 00:00:00 2001 From: Alessandro Cerioni <acerioni@grandlyon.com> Date: Sun, 3 Mar 2019 16:30:36 +0100 Subject: [PATCH] More robust handling of geometry. Handling of boolean and date types. --- 5-pg-field-type-detector.py | 6 +++--- 6-doc-processor.py | 4 +++- utils/postgis_helper.py | 24 ++++++++++++++++-------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/5-pg-field-type-detector.py b/5-pg-field-type-detector.py index c1e848c..f3c85c6 100644 --- a/5-pg-field-type-detector.py +++ b/5-pg-field-type-detector.py @@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ): else: output = catalog.copy() - # selected_schema = "abr_arbres_alignement" - # selected_table = selected_schema + ".abrarbre" + # selected_schema = "rdata" + # selected_table = selected_schema + ".sit_sitra.sittourisme" # found = False logging.info('Getting schemas...') @@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ): count = 'unknown no. of' #print(count) - db_schema_table = '%s.%s' % (pg.dbname, table) + db_schema_table = '%s/%s' % (pg.dbname, table) t2 = time.time() logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count)) logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1))) diff --git a/6-doc-processor.py b/6-doc-processor.py index 2ce2c96..8409753 100644 --- a/6-doc-processor.py +++ b/6-doc-processor.py @@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ): out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop]) elif out_types[lookup_key] == 'float': out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop]) - elif out_types[lookup_key] == 'datetime': + elif out_types[lookup_key] in ['date', 'datetime']: out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ') + elif out_types[lookup_key] == 'bool': + out_flattened_properties[prop] = convert_to_boolean(in_flattened_properties[prop]) else: logging.critical('type %s not supported', out_types[prop]) sys.exit(1) diff --git a/utils/postgis_helper.py b/utils/postgis_helper.py index a0190a6..e558cba 100644 --- a/utils/postgis_helper.py +++ b/utils/postgis_helper.py @@ -88,21 +88,29 @@ class Remote(object): selected = select(fields) for entry in self.engine.execute(selected): items = entry.items() - #print(items) - if geom is not None: - try: - geometry = json.loads(items.pop()[1]) - except TypeError: - geom = None - properties = dict(items) + geometry = None + try: + geometry = json.loads(properties['ST_AsGeoJSON_1']) + del properties['ST_AsGeoJSON_1'] + except: + pass + + # #print(items) + # if geom is not None: + # try: + # geometry = json.loads(items.pop()[1]) + # except TypeError: + # geom = None + #properties = dict(items) + document = { 'type': 'Feature', 'properties': properties } - if geom is not None: + if geometry is not None: document['geometry'] = geometry yield document -- GitLab