Merge branch 'development' into 'master'

Fix #10877 considère les tableaux de str comme str au lieu de json

See merge request !12

tests/conftest.py

@@ -5,9 +5,13 @@ import msgpack
 from lib.serializers import encode_datetime, decode_datetime
 import re
 import os
+import json
 
 import pytest
 
+# Passer à True pour génerer des JSON à côté des MP (plus lisible)
+DEBUG_OUTPUT_JSON = False
+
 def _sanitize(name):
     """
     Replace certain characters (which might be problematic when contained in
@@ -19,8 +23,14 @@ def _sanitize(name):
 
 
 @pytest.fixture
-def load_object():
-    def load_file(file_path):
+def load_object(request):
+
+    module = request.module.__name__
+    node = _sanitize(request.node.name).replace("test_fix_field_types",
+                                                'test_get_entries_from_postgis')
+
+    def load_file(file_path_template):
+        file_path = file_path_template.format(node)
 
         with open(file_path, 'rb') as file_stream:
             expected_data = msgpack.unpackb(file_stream.read(), raw=False, object_hook=decode_datetime)
@@ -51,9 +61,13 @@ def verify_objects(request):
                 the_body = msgpack.packb(data, use_bin_type=True, default=encode_datetime)
                 file_stream.write(the_body)
 
+        if DEBUG_OUTPUT_JSON:
+            with open(file_path+".json", "w") as file_stream:
+                the_body = json.dump(data, file_stream, default=encode_datetime, indent=4)
+
         with open(file_path, 'rb') as file_stream:
             expected_data = msgpack.unpackb(file_stream.read(), raw=False, object_hook=decode_datetime)
 
             assert data == expected_data
 
-    yield check_data
+    return check_data

tests/test_doc_enricher.py

@@ -4,9 +4,12 @@ from workers.doc_enricher import get_entries_from_postgis
 
 
 @pytest.mark.vcr()
-def test_get_entries_from_postgis(verify_objects):
-    link = dict(url='/rdata',
-                name='apd_apidae.apdevenement_2_0_0')
+@pytest.mark.parametrize("url,name", [
+    ('/rdata', "apd_apidae.apdevenement"),
+    ('/grandlyon', 'eco_ecologie.ecoannuproducteur_latest')
+])
+def test_get_entries_from_postgis(verify_objects, url, name):
+    link = dict(url=url, name=name)
 
     cfg = dict()
     cfg['host'] = "147.135.219.0"

tests/test_doc_processor.py

-from workers.doc_processor import fix_field_types
 import os
 import json
 
+import pytest
+
+from workers.doc_processor import fix_field_types
+
 
-def test_fix_field_types(load_object, verify_objects):
+@pytest.mark.parametrize("url,name", [
+    ('/rdata', "apd_apidae.apdevenement"),
+    ('/grandlyon', 'eco_ecologie.ecoannuproducteur_latest')
+])
+def test_fix_field_types(load_object, verify_objects, url, name):
     link = dict()
-    link['url'] = "/rdata"
-    link['name'] = "apd_apidae.apdevenement_2_0_0"
+    link['url'] = url
+    link['name'] = name
 
     filename = os.path.join('tests/data/prod_working_directory/',
                             'field_catalog_by_dbschematable.json' )
@@ -15,7 +22,7 @@ def test_fix_field_types(load_object, verify_objects):
         with open(filename, 'r') as fp:
             catalog = json.load(fp)
 
-    docs = load_object('tests/data/test_doc_enricher.test_get_entries_from_postgis.0.mp')
+    docs = load_object('tests/data/test_doc_enricher.{}.0.mp')
     doc_page = [{'data-fr': feature} for feature in docs]
 
     out_docs = fix_field_types(doc_page, catalog, link)

workers/doc_processor.py

@@ -78,6 +78,16 @@ def fix_field_types( in_docs, field_catalog, link):
         def typed_name(name, types):
             type_of_name = types.get(name)
 
+            # Redmine 10877
+            # si on a ".0" dans le nom, c'est surement un tableau de str
+            # on le rangera dans _str au lieu de json pour ne pas perturber ElasticSearch
+            # (pour elastic, un teableau de str est un type str, pas un json)
+
+            type_of_name = type_of_name or types.get(f"{ name }.0")
+
+            # Si le type n'exite pas (address par exemple)
+            # c'est que la propriété est éclatée en address.code_poste, address.address
+            # donc on dit que c'est un json
             if not type_of_name:
                 return f'{name}_json'