diff --git a/5-pg-field-type-detector.py b/5-pg-field-type-detector.py index c1e848c09b0bfce3aaaa672b046b7ec70cd55a32..f3c85c6ce2344884cfb5c202380b584204ce4463 100644 --- a/5-pg-field-type-detector.py +++ b/5-pg-field-type-detector.py @@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ): else: output = catalog.copy() - # selected_schema = "abr_arbres_alignement" - # selected_table = selected_schema + ".abrarbre" + # selected_schema = "rdata" + # selected_table = selected_schema + ".sit_sitra.sittourisme" # found = False logging.info('Getting schemas...') @@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ): count = 'unknown no. of' #print(count) - db_schema_table = '%s.%s' % (pg.dbname, table) + db_schema_table = '%s/%s' % (pg.dbname, table) t2 = time.time() logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count)) logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1))) diff --git a/6-doc-processor.py b/6-doc-processor.py index 2ce2c967c09f33cee34096f814f02fb816431f20..84097537077668a3c4699e44e7e33f1b0683fe68 100644 --- a/6-doc-processor.py +++ b/6-doc-processor.py @@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ): out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop]) elif out_types[lookup_key] == 'float': out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop]) - elif out_types[lookup_key] == 'datetime': + elif out_types[lookup_key] in ['date', 'datetime']: out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ') + elif out_types[lookup_key] == 'bool': + out_flattened_properties[prop] = convert_to_boolean(in_flattened_properties[prop]) else: logging.critical('type %s not supported', out_types[prop]) sys.exit(1) diff --git a/utils/postgis_helper.py b/utils/postgis_helper.py index a0190a667ce307257f5f4e76380f38978aba5ffb..e558cbafcce6768d877186f7032b90cdf0ece85d 100644 --- a/utils/postgis_helper.py +++ b/utils/postgis_helper.py @@ -88,21 +88,29 @@ class Remote(object): selected = select(fields) for entry in self.engine.execute(selected): items = entry.items() - #print(items) - if geom is not None: - try: - geometry = json.loads(items.pop()[1]) - except TypeError: - geom = None - properties = dict(items) + geometry = None + try: + geometry = json.loads(properties['ST_AsGeoJSON_1']) + del properties['ST_AsGeoJSON_1'] + except: + pass + + # #print(items) + # if geom is not None: + # try: + # geometry = json.loads(items.pop()[1]) + # except TypeError: + # geom = None + #properties = dict(items) + document = { 'type': 'Feature', 'properties': properties } - if geom is not None: + if geometry is not None: document['geometry'] = geometry yield document