From 2d70b833fc272f28a0d7cbcbbfa6bb9580c91db9 Mon Sep 17 00:00:00 2001 From: ddamiron <ddamiron@sii.fr> Date: Tue, 2 Jul 2019 09:30:02 +0200 Subject: [PATCH] add loglevel in mongo log message --- lib/log_message.py | 3 ++- lib/mongo_session.py | 2 +- main.py | 21 ++++++++++++--------- workers/doc-enricher.py | 6 ++++-- workers/doc-indexer.py | 8 +++++--- workers/doc-processor.py | 6 ++++-- workers/metadata-processor.py | 12 ++++++++---- workers/reindexer.py | 21 +++++++++++++-------- workers/sample-generator.py | 9 ++++++--- 9 files changed, 55 insertions(+), 33 deletions(-) diff --git a/lib/log_message.py b/lib/log_message.py index 9a4d84b..8ad373c 100644 --- a/lib/log_message.py +++ b/lib/log_message.py @@ -4,7 +4,7 @@ import uuid as uuid_lib class LogMessage: - def __init__(self, session_id, uuid, step, status, uuid_prefix, info): + def __init__(self, session_id, uuid, step, status, uuid_prefix, info, loglevel): if(session_id.lower()) == 'new': self.session_id = str(uuid_lib.uuid4()) else: @@ -14,6 +14,7 @@ class LogMessage: self.status = status self.uuid_prefix = uuid_prefix self.info = info + self.loglevel = loglevel self.timestamp = self.generate_timestamp() @staticmethod diff --git a/lib/mongo_session.py b/lib/mongo_session.py index 27aa8bc..a61c1ae 100644 --- a/lib/mongo_session.py +++ b/lib/mongo_session.py @@ -52,7 +52,7 @@ class MongoSession: "status": body_object["status"], "uuid_prefix": body_object["uuid_prefix"], "info": body_object["info"], - }) + "loglevel": body_object["loglevel"]}) except Exception as exc: print('[ERROR saving log]:', exc) ch.basic_ack(delivery_tag=method.delivery_tag) diff --git a/main.py b/main.py index 03cad9c..a10cb20 100644 --- a/main.py +++ b/main.py @@ -185,7 +185,8 @@ def delete_dataset_from_dest_index(the_rabbit, the_cfg, the_uuid): step='main', status='delete_dataset_from_dest_index', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) the_rabbit.publish_log(log_message=log_message.__dict__) # ------------------------------------------------------------ @@ -233,12 +234,12 @@ def get_metadata_records_processor(the_rabbit, the_uuid = record['geonet:info']['uuid'] - if is_locked(the_cfg['session']['working_directory'], the_uuid ): - logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid) - continue - else: - logging.info("Setting lock for dataset with uuid = %s" % the_uuid) - lock(the_cfg['session']['working_directory'], the_uuid) + # if is_locked(the_cfg['session']['working_directory'], the_uuid ): + # logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid) + # continue + # else: + # logging.info("Setting lock for dataset with uuid = %s" % the_uuid) + # lock(the_cfg['session']['working_directory'], the_uuid) # delete_dataset_from_indices(cfg, record['geonet:info']['uuid']) delete_dataset_from_dest_index(the_rabbit=the_rabbit, @@ -277,7 +278,8 @@ def main(cfg): step='main', status='Starting...', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) cfg['session']['current_uuid'] = uuid_to_get with RabbitSession(cfg) as rabbit: @@ -299,7 +301,8 @@ def main(cfg): step='main', status='Terminated', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) cfg['session']['current_uuid'] = uuid_to_get rabbit.publish_log(log_message=log_message2.__dict__) diff --git a/workers/doc-enricher.py b/workers/doc-enricher.py index f183ac6..9ec2d39 100644 --- a/workers/doc-enricher.py +++ b/workers/doc-enricher.py @@ -249,7 +249,8 @@ def enrich_docs( channel, method, properties, body ): step='doc-enricher', status=status_message, uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -288,7 +289,8 @@ def enrich_docs( channel, method, properties, body ): step='doc-enricher', status='done', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) diff --git a/workers/doc-indexer.py b/workers/doc-indexer.py index 193aa74..6e9e5e7 100644 --- a/workers/doc-indexer.py +++ b/workers/doc-indexer.py @@ -132,7 +132,8 @@ def index_docs(channel, method, properties, body): step='doc-indexer', status=status_message, uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -206,7 +207,8 @@ def index_docs(channel, method, properties, body): step='doc-indexer', status='done', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -262,7 +264,7 @@ if __name__ == '__main__': parser.add_argument('--exchange', dest='exchange', help='the RabbitMQ exchange', type=str, required=True) parser.add_argument('--queue', dest='queue', help='the RabbitMQ queue', type=str, required=True) parser.add_argument('--loglevel', dest='loglevel', help='the log level', default="INFO", type=str, - choices=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR']) + argu=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR']) parser.add_argument('--user', dest='user', help='the RabbitMQ user login', type=str, required=True) parser.add_argument('--password', dest='password', help='the RabbitMQ user password', type=str, required=True) diff --git a/workers/doc-processor.py b/workers/doc-processor.py index 5be41ba..b121f9e 100644 --- a/workers/doc-processor.py +++ b/workers/doc-processor.py @@ -121,7 +121,8 @@ def process_docs( channel, method, properties, body ): step='doc-processor', status=status_message, uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -163,7 +164,8 @@ def process_docs( channel, method, properties, body ): step='doc-processor', status='done', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) diff --git a/workers/metadata-processor.py b/workers/metadata-processor.py index e9c39fc..b48e21d 100644 --- a/workers/metadata-processor.py +++ b/workers/metadata-processor.py @@ -315,7 +315,8 @@ def callback( channel, method, properties, body ): step='metadata-processor', status='Starting...', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -387,7 +388,8 @@ def callback( channel, method, properties, body ): step='metadata-processor', status='sent task doc to enrich', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -435,7 +437,8 @@ def callback( channel, method, properties, body ): step='metadata-processor', status='sent task doc to index', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -456,7 +459,8 @@ def callback( channel, method, properties, body ): step='metadata-processor', status='terminated', uuid_prefix='meta', - info='no info' + info='no info', + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) diff --git a/workers/reindexer.py b/workers/reindexer.py index f7f35ef..f695990 100644 --- a/workers/reindexer.py +++ b/workers/reindexer.py @@ -40,7 +40,8 @@ def create_sampling_task(cfg, channel, uuid): step='reindexer', status='create sampling task', uuid_prefix='full', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -58,7 +59,7 @@ def create_sampling_task(cfg, channel, uuid): routing_key=cfg['rabbitmq']['routing_key_6'], body=the_body, properties=pika.BasicProperties(delivery_mode = 2) - ) + ) #connection.close() @@ -135,7 +136,7 @@ def on_msg_callback(channel, method, properties, body): # else: # channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1) # logging.error("The uuid ends neither with .full nor with .meta. What shall I do?") - # return + # return if count_es != count_ref: @@ -157,7 +158,8 @@ def on_msg_callback(channel, method, properties, body): step='reindexer', status=message, uuid_prefix='full', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -252,7 +254,8 @@ def on_msg_callback(channel, method, properties, body): step='reindexer', status=message, uuid_prefix='full', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -278,7 +281,8 @@ def on_msg_callback(channel, method, properties, body): step='reindexer', status=message, uuid_prefix='meta', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -308,7 +312,8 @@ def on_msg_callback(channel, method, properties, body): step='reindexer', status=message, uuid_prefix='', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -344,7 +349,7 @@ def main(cfg): channel.basic_qos(prefetch_count=1) channel.basic_consume(on_message_callback=lambda ch, method, properties, body: on_msg_callback(ch, method, properties, body), - queue=reindex_tasks_to_create_qn) + queue=reindex_tasks_to_create_qn) channel.start_consuming() diff --git a/workers/sample-generator.py b/workers/sample-generator.py index 1ff6a03..9e5f2a8 100644 --- a/workers/sample-generator.py +++ b/workers/sample-generator.py @@ -188,7 +188,8 @@ def callback(channel, method, properties, body): step='sampler', status=message, uuid_prefix='', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -238,7 +239,8 @@ def callback(channel, method, properties, body): step='sampler', status=message, uuid_prefix='', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) @@ -278,7 +280,8 @@ def callback(channel, method, properties, body): step='sampler', status=message, uuid_prefix='', - info=uuid + info=uuid, + loglevel='INFO' ) json_body = json.dumps(log_message.__dict__) -- GitLab