From 2d70b833fc272f28a0d7cbcbbfa6bb9580c91db9 Mon Sep 17 00:00:00 2001
From: ddamiron <ddamiron@sii.fr>
Date: Tue, 2 Jul 2019 09:30:02 +0200
Subject: [PATCH] add loglevel in mongo log message

---
 lib/log_message.py            |  3 ++-
 lib/mongo_session.py          |  2 +-
 main.py                       | 21 ++++++++++++---------
 workers/doc-enricher.py       |  6 ++++--
 workers/doc-indexer.py        |  8 +++++---
 workers/doc-processor.py      |  6 ++++--
 workers/metadata-processor.py | 12 ++++++++----
 workers/reindexer.py          | 21 +++++++++++++--------
 workers/sample-generator.py   |  9 ++++++---
 9 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/lib/log_message.py b/lib/log_message.py
index 9a4d84b..8ad373c 100644
--- a/lib/log_message.py
+++ b/lib/log_message.py
@@ -4,7 +4,7 @@ import uuid as uuid_lib
 
 
 class LogMessage:
-    def __init__(self, session_id, uuid, step, status, uuid_prefix, info):
+    def __init__(self, session_id, uuid, step, status, uuid_prefix, info, loglevel):
         if(session_id.lower()) == 'new':
             self.session_id = str(uuid_lib.uuid4())
         else:
@@ -14,6 +14,7 @@ class LogMessage:
         self.status = status
         self.uuid_prefix = uuid_prefix
         self.info = info
+        self.loglevel = loglevel
         self.timestamp = self.generate_timestamp()
 
     @staticmethod
diff --git a/lib/mongo_session.py b/lib/mongo_session.py
index 27aa8bc..a61c1ae 100644
--- a/lib/mongo_session.py
+++ b/lib/mongo_session.py
@@ -52,7 +52,7 @@ class MongoSession:
                                                    "status": body_object["status"],
                                                    "uuid_prefix": body_object["uuid_prefix"],
                                                    "info": body_object["info"],
-                                                   })
+                                                   "loglevel": body_object["loglevel"]})
         except Exception as exc:
             print('[ERROR saving log]:', exc)
         ch.basic_ack(delivery_tag=method.delivery_tag)
diff --git a/main.py b/main.py
index 03cad9c..a10cb20 100644
--- a/main.py
+++ b/main.py
@@ -185,7 +185,8 @@ def delete_dataset_from_dest_index(the_rabbit, the_cfg, the_uuid):
                              step='main',
                              status='delete_dataset_from_dest_index',
                              uuid_prefix='meta',
-                             info='no info'
+                             info='no info',
+                             loglevel='INFO'
                              )
     the_rabbit.publish_log(log_message=log_message.__dict__)
     # ------------------------------------------------------------
@@ -233,12 +234,12 @@ def get_metadata_records_processor(the_rabbit,
 
         the_uuid = record['geonet:info']['uuid']
 
-        if is_locked(the_cfg['session']['working_directory'], the_uuid ):
-            logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid)
-            continue
-        else:
-            logging.info("Setting lock for dataset with uuid = %s" % the_uuid)
-            lock(the_cfg['session']['working_directory'], the_uuid)
+        # if is_locked(the_cfg['session']['working_directory'], the_uuid ):
+        #     logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid)
+        #     continue
+        # else:
+        #     logging.info("Setting lock for dataset with uuid = %s" % the_uuid)
+        #     lock(the_cfg['session']['working_directory'], the_uuid)
 
         # delete_dataset_from_indices(cfg, record['geonet:info']['uuid'])
         delete_dataset_from_dest_index(the_rabbit=the_rabbit,
@@ -277,7 +278,8 @@ def main(cfg):
                                  step='main',
                                  status='Starting...',
                                  uuid_prefix='meta',
-                                 info='no info'
+                                 info='no info',
+                                 loglevel='INFO'
                                  )
         cfg['session']['current_uuid'] = uuid_to_get
         with RabbitSession(cfg) as rabbit:
@@ -299,7 +301,8 @@ def main(cfg):
                                       step='main',
                                       status='Terminated',
                                       uuid_prefix='meta',
-                                      info='no info'
+                                      info='no info',
+                                      loglevel='INFO'
                                       )
             cfg['session']['current_uuid'] = uuid_to_get
             rabbit.publish_log(log_message=log_message2.__dict__)
diff --git a/workers/doc-enricher.py b/workers/doc-enricher.py
index f183ac6..9ec2d39 100644
--- a/workers/doc-enricher.py
+++ b/workers/doc-enricher.py
@@ -249,7 +249,8 @@ def enrich_docs( channel, method, properties, body ):
                                  step='doc-enricher',
                                  status=status_message,
                                  uuid_prefix='meta',
-                                 info='no info'
+                                 info='no info',
+                                 loglevel='INFO'
                                  )
 
         json_body = json.dumps(log_message.__dict__)
@@ -288,7 +289,8 @@ def enrich_docs( channel, method, properties, body ):
                                      step='doc-enricher',
                                      status='done',
                                      uuid_prefix='meta',
-                                     info='no info'
+                                     info='no info',
+                                     loglevel='INFO'
                                      )
 
             json_body = json.dumps(log_message.__dict__)
diff --git a/workers/doc-indexer.py b/workers/doc-indexer.py
index 193aa74..6e9e5e7 100644
--- a/workers/doc-indexer.py
+++ b/workers/doc-indexer.py
@@ -132,7 +132,8 @@ def index_docs(channel, method, properties, body):
                              step='doc-indexer',
                              status=status_message,
                              uuid_prefix='meta',
-                             info='no info'
+                             info='no info',
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -206,7 +207,8 @@ def index_docs(channel, method, properties, body):
                                  step='doc-indexer',
                                  status='done',
                                  uuid_prefix='meta',
-                                 info='no info'
+                                 info='no info',
+                                 loglevel='INFO'
                                  )
 
         json_body = json.dumps(log_message.__dict__)
@@ -262,7 +264,7 @@ if __name__ == '__main__':
     parser.add_argument('--exchange', dest='exchange', help='the RabbitMQ exchange', type=str, required=True)
     parser.add_argument('--queue', dest='queue', help='the RabbitMQ queue', type=str, required=True)
     parser.add_argument('--loglevel', dest='loglevel', help='the log level', default="INFO", type=str,
-                        choices=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR'])
+                        argu=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR'])
     parser.add_argument('--user', dest='user', help='the RabbitMQ user login', type=str, required=True)
     parser.add_argument('--password', dest='password', help='the RabbitMQ user password', type=str, required=True)
 
diff --git a/workers/doc-processor.py b/workers/doc-processor.py
index 5be41ba..b121f9e 100644
--- a/workers/doc-processor.py
+++ b/workers/doc-processor.py
@@ -121,7 +121,8 @@ def process_docs( channel, method, properties, body ):
                              step='doc-processor',
                              status=status_message,
                              uuid_prefix='meta',
-                             info='no info'
+                             info='no info',
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -163,7 +164,8 @@ def process_docs( channel, method, properties, body ):
                                  step='doc-processor',
                                  status='done',
                                  uuid_prefix='meta',
-                                 info='no info'
+                                 info='no info',
+                                 loglevel='INFO'
                                  )
 
         json_body = json.dumps(log_message.__dict__)
diff --git a/workers/metadata-processor.py b/workers/metadata-processor.py
index e9c39fc..b48e21d 100644
--- a/workers/metadata-processor.py
+++ b/workers/metadata-processor.py
@@ -315,7 +315,8 @@ def callback( channel, method, properties, body ):
                              step='metadata-processor',
                              status='Starting...',
                              uuid_prefix='meta',
-                             info='no info'
+                             info='no info',
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -387,7 +388,8 @@ def callback( channel, method, properties, body ):
                                          step='metadata-processor',
                                          status='sent task doc to enrich',
                                          uuid_prefix='meta',
-                                         info='no info'
+                                         info='no info',
+                                         loglevel='INFO'
                                          )
 
                 json_body = json.dumps(log_message.__dict__)
@@ -435,7 +437,8 @@ def callback( channel, method, properties, body ):
                                  step='metadata-processor',
                                  status='sent task doc to index',
                                  uuid_prefix='meta',
-                                 info='no info'
+                                 info='no info',
+                                 loglevel='INFO'
                                  )
 
         json_body = json.dumps(log_message.__dict__)
@@ -456,7 +459,8 @@ def callback( channel, method, properties, body ):
                              step='metadata-processor',
                              status='terminated',
                              uuid_prefix='meta',
-                             info='no info'
+                             info='no info',
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
diff --git a/workers/reindexer.py b/workers/reindexer.py
index f7f35ef..f695990 100644
--- a/workers/reindexer.py
+++ b/workers/reindexer.py
@@ -40,7 +40,8 @@ def create_sampling_task(cfg, channel, uuid):
                              step='reindexer',
                              status='create sampling task',
                              uuid_prefix='full',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -58,7 +59,7 @@ def create_sampling_task(cfg, channel, uuid):
                            routing_key=cfg['rabbitmq']['routing_key_6'],
                            body=the_body,
                            properties=pika.BasicProperties(delivery_mode = 2)
-                         )
+                           )
 
     #connection.close()
 
@@ -135,7 +136,7 @@ def on_msg_callback(channel, method, properties, body):
     # else:
     #     channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1)
     #     logging.error("The uuid ends neither with .full nor with .meta. What shall I do?")
-        # return
+    # return
 
 
     if count_es != count_ref:
@@ -157,7 +158,8 @@ def on_msg_callback(channel, method, properties, body):
                              step='reindexer',
                              status=message,
                              uuid_prefix='full',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -252,7 +254,8 @@ def on_msg_callback(channel, method, properties, body):
                                      step='reindexer',
                                      status=message,
                                      uuid_prefix='full',
-                                     info=uuid
+                                     info=uuid,
+                                     loglevel='INFO'
                                      )
 
             json_body = json.dumps(log_message.__dict__)
@@ -278,7 +281,8 @@ def on_msg_callback(channel, method, properties, body):
                                      step='reindexer',
                                      status=message,
                                      uuid_prefix='meta',
-                                     info=uuid
+                                     info=uuid,
+                                     loglevel='INFO'
                                      )
 
             json_body = json.dumps(log_message.__dict__)
@@ -308,7 +312,8 @@ def on_msg_callback(channel, method, properties, body):
                              step='reindexer',
                              status=message,
                              uuid_prefix='',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -344,7 +349,7 @@ def main(cfg):
 
     channel.basic_qos(prefetch_count=1)
     channel.basic_consume(on_message_callback=lambda ch, method, properties, body: on_msg_callback(ch, method, properties, body),
-                            queue=reindex_tasks_to_create_qn)
+                          queue=reindex_tasks_to_create_qn)
 
     channel.start_consuming()
 
diff --git a/workers/sample-generator.py b/workers/sample-generator.py
index 1ff6a03..9e5f2a8 100644
--- a/workers/sample-generator.py
+++ b/workers/sample-generator.py
@@ -188,7 +188,8 @@ def callback(channel, method, properties, body):
                              step='sampler',
                              status=message,
                              uuid_prefix='',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -238,7 +239,8 @@ def callback(channel, method, properties, body):
                              step='sampler',
                              status=message,
                              uuid_prefix='',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
@@ -278,7 +280,8 @@ def callback(channel, method, properties, body):
                              step='sampler',
                              status=message,
                              uuid_prefix='',
-                             info=uuid
+                             info=uuid,
+                             loglevel='INFO'
                              )
 
     json_body = json.dumps(log_message.__dict__)
-- 
GitLab