Adding a session setup script. Disabling refresh of the destination index, in...

Adding a session setup script. Disabling refresh of the destination index, in order to speed up the reindexation.

0-setup-session.py

+import pika
+from utils.my_logging import logging
+import os, glob
+from pymongo import MongoClient
+
+if __name__ == '__main__':
+
+    import yaml
+
+    with open("config.yaml", 'r') as yamlfile:
+        cfg = yaml.load(yamlfile)
+
+    es = Elasticsearch([cfg['indexer']['url']], timeout=60)
+    es_index = cfg['indexer']['index']
+
+    es_body = { "settings" : {
+                    "number_of_shards" : 1,
+                    "number_of_replicas" : 0,
+                    "index.mapping.total_fields.limit": 10000,
+                    "refresh_interval": "30s"
+                    },
+                "mappings": {
+                    "_doc": {
+                        "dynamic_templates": [ # priority is given by order!
+                            {
+                                "uuid" : {
+                                    "path_match": "uuid",
+                                    "mapping": {
+                                        "type": "keyword",
+                                        }
+                                    }
+                            },
+                            {
+                                "default" : {
+                                    "path_match": "*",
+                                    "mapping": {
+                                        "enabled": "false"
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+    #es_body.update({"mappings": {"_doc": {"dynamic_date_formats": ["strict_date_optional_time"]}}})
+
+    try:
+        rep = es.indices.create(es_index, es_body)#, wait_for_active_shards=0)
+    except:
+        pass

es_template.py

@@ -6,6 +6,7 @@ template = {
         #"index.mapping.ignore_malformed": True,
         # "number_of_shards" : 48,
         # "number_of_replicas" : 0,
+        "refresh_interval" : -1,
         "max_ngram_diff": 100,
         "analysis": {
             "filter": {