More robust handling of geometry. Handling of boolean and date types.

5-pg-field-type-detector.py

@@ -191,8 +191,8 @@ def generate_field_catalog( cfg, pg, catalog=None ):
     else:
         output = catalog.copy()
 
-    # selected_schema = "abr_arbres_alignement"
-    # selected_table = selected_schema + ".abrarbre"
+    # selected_schema = "rdata"
+    # selected_table = selected_schema + ".sit_sitra.sittourisme"
     # found = False
 
     logging.info('Getting schemas...')
@@ -217,7 +217,7 @@ def generate_field_catalog( cfg, pg, catalog=None ):
                 count = 'unknown no. of'
             #print(count)
 
-            db_schema_table = '%s.%s' % (pg.dbname, table)
+            db_schema_table = '%s/%s' % (pg.dbname, table)
             t2 = time.time()
             logging.info('Analyzing table %s (%s records)...' % (db_schema_table, count))
             logging.info('| %i records were analyzed so far (%.2f records/s)' % (output['analyzed_docs'], output['analyzed_docs']/(t2-t1)))

6-doc-processor.py

@@ -44,8 +44,10 @@ def fix_field_types( in_docs, out_types ):
                 out_flattened_properties[prop] = convert_to_int(in_flattened_properties[prop])
             elif out_types[lookup_key] == 'float':
                 out_flattened_properties[prop] = convert_to_float(in_flattened_properties[prop])
-            elif out_types[lookup_key] == 'datetime':
+            elif out_types[lookup_key] in ['date', 'datetime']:
                 out_flattened_properties[prop] = convert_to_datetime(in_flattened_properties[prop]).strftime('%Y-%m-%dT%H:%M:%SZ')
+            elif out_types[lookup_key] == 'bool':
+                out_flattened_properties[prop] = convert_to_boolean(in_flattened_properties[prop])
             else:
                 logging.critical('type %s not supported', out_types[prop])
                 sys.exit(1)

utils/postgis_helper.py

@@ -88,21 +88,29 @@ class Remote(object):
         selected = select(fields)
         for entry in self.engine.execute(selected):
             items = entry.items()
-            #print(items)
-            if geom is not None:
-                try:
-                    geometry = json.loads(items.pop()[1])
-                except TypeError:
-                    geom = None
-
             properties = dict(items)
 
+            geometry = None
+            try:
+                geometry = json.loads(properties['ST_AsGeoJSON_1'])
+                del properties['ST_AsGeoJSON_1']
+            except:
+                pass
+
+            # #print(items)
+            # if geom is not None:
+            #     try:
+            #         geometry = json.loads(items.pop()[1])
+            #     except TypeError:
+            #         geom = None
+            #properties = dict(items)
+
             document = {
                 'type': 'Feature',
                 'properties': properties
             }
 
-            if geom is not None:
+            if geometry is not None:
                 document['geometry'] = geometry
 
             yield document