From 2d70b833fc272f28a0d7cbcbbfa6bb9580c91db9 Mon Sep 17 00:00:00 2001
From: ddamiron <ddamiron@sii.fr>
Date: Tue, 2 Jul 2019 09:30:02 +0200
Subject: [PATCH] add loglevel in mongo log message
---
lib/log_message.py | 3 ++-
lib/mongo_session.py | 2 +-
main.py | 21 ++++++++++++---------
workers/doc-enricher.py | 6 ++++--
workers/doc-indexer.py | 8 +++++---
workers/doc-processor.py | 6 ++++--
workers/metadata-processor.py | 12 ++++++++----
workers/reindexer.py | 21 +++++++++++++--------
workers/sample-generator.py | 9 ++++++---
9 files changed, 55 insertions(+), 33 deletions(-)
diff --git a/lib/log_message.py b/lib/log_message.py
index 9a4d84b..8ad373c 100644
--- a/lib/log_message.py
+++ b/lib/log_message.py
@@ -4,7 +4,7 @@ import uuid as uuid_lib
class LogMessage:
- def __init__(self, session_id, uuid, step, status, uuid_prefix, info):
+ def __init__(self, session_id, uuid, step, status, uuid_prefix, info, loglevel):
if(session_id.lower()) == 'new':
self.session_id = str(uuid_lib.uuid4())
else:
@@ -14,6 +14,7 @@ class LogMessage:
self.status = status
self.uuid_prefix = uuid_prefix
self.info = info
+ self.loglevel = loglevel
self.timestamp = self.generate_timestamp()
@staticmethod
diff --git a/lib/mongo_session.py b/lib/mongo_session.py
index 27aa8bc..a61c1ae 100644
--- a/lib/mongo_session.py
+++ b/lib/mongo_session.py
@@ -52,7 +52,7 @@ class MongoSession:
"status": body_object["status"],
"uuid_prefix": body_object["uuid_prefix"],
"info": body_object["info"],
- })
+ "loglevel": body_object["loglevel"]})
except Exception as exc:
print('[ERROR saving log]:', exc)
ch.basic_ack(delivery_tag=method.delivery_tag)
diff --git a/main.py b/main.py
index 03cad9c..a10cb20 100644
--- a/main.py
+++ b/main.py
@@ -185,7 +185,8 @@ def delete_dataset_from_dest_index(the_rabbit, the_cfg, the_uuid):
step='main',
status='delete_dataset_from_dest_index',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
the_rabbit.publish_log(log_message=log_message.__dict__)
# ------------------------------------------------------------
@@ -233,12 +234,12 @@ def get_metadata_records_processor(the_rabbit,
the_uuid = record['geonet:info']['uuid']
- if is_locked(the_cfg['session']['working_directory'], the_uuid ):
- logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid)
- continue
- else:
- logging.info("Setting lock for dataset with uuid = %s" % the_uuid)
- lock(the_cfg['session']['working_directory'], the_uuid)
+ # if is_locked(the_cfg['session']['working_directory'], the_uuid ):
+ # logging.info("Dataset with uuid = %s is already being indexed: skipping." % the_uuid)
+ # continue
+ # else:
+ # logging.info("Setting lock for dataset with uuid = %s" % the_uuid)
+ # lock(the_cfg['session']['working_directory'], the_uuid)
# delete_dataset_from_indices(cfg, record['geonet:info']['uuid'])
delete_dataset_from_dest_index(the_rabbit=the_rabbit,
@@ -277,7 +278,8 @@ def main(cfg):
step='main',
status='Starting...',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
cfg['session']['current_uuid'] = uuid_to_get
with RabbitSession(cfg) as rabbit:
@@ -299,7 +301,8 @@ def main(cfg):
step='main',
status='Terminated',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
cfg['session']['current_uuid'] = uuid_to_get
rabbit.publish_log(log_message=log_message2.__dict__)
diff --git a/workers/doc-enricher.py b/workers/doc-enricher.py
index f183ac6..9ec2d39 100644
--- a/workers/doc-enricher.py
+++ b/workers/doc-enricher.py
@@ -249,7 +249,8 @@ def enrich_docs( channel, method, properties, body ):
step='doc-enricher',
status=status_message,
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -288,7 +289,8 @@ def enrich_docs( channel, method, properties, body ):
step='doc-enricher',
status='done',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
diff --git a/workers/doc-indexer.py b/workers/doc-indexer.py
index 193aa74..6e9e5e7 100644
--- a/workers/doc-indexer.py
+++ b/workers/doc-indexer.py
@@ -132,7 +132,8 @@ def index_docs(channel, method, properties, body):
step='doc-indexer',
status=status_message,
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -206,7 +207,8 @@ def index_docs(channel, method, properties, body):
step='doc-indexer',
status='done',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -262,7 +264,7 @@ if __name__ == '__main__':
parser.add_argument('--exchange', dest='exchange', help='the RabbitMQ exchange', type=str, required=True)
parser.add_argument('--queue', dest='queue', help='the RabbitMQ queue', type=str, required=True)
parser.add_argument('--loglevel', dest='loglevel', help='the log level', default="INFO", type=str,
- choices=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR'])
+ argu=['INFO', 'DEBUG', 'WARN', 'CRITICAL', 'ERROR'])
parser.add_argument('--user', dest='user', help='the RabbitMQ user login', type=str, required=True)
parser.add_argument('--password', dest='password', help='the RabbitMQ user password', type=str, required=True)
diff --git a/workers/doc-processor.py b/workers/doc-processor.py
index 5be41ba..b121f9e 100644
--- a/workers/doc-processor.py
+++ b/workers/doc-processor.py
@@ -121,7 +121,8 @@ def process_docs( channel, method, properties, body ):
step='doc-processor',
status=status_message,
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -163,7 +164,8 @@ def process_docs( channel, method, properties, body ):
step='doc-processor',
status='done',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
diff --git a/workers/metadata-processor.py b/workers/metadata-processor.py
index e9c39fc..b48e21d 100644
--- a/workers/metadata-processor.py
+++ b/workers/metadata-processor.py
@@ -315,7 +315,8 @@ def callback( channel, method, properties, body ):
step='metadata-processor',
status='Starting...',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -387,7 +388,8 @@ def callback( channel, method, properties, body ):
step='metadata-processor',
status='sent task doc to enrich',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -435,7 +437,8 @@ def callback( channel, method, properties, body ):
step='metadata-processor',
status='sent task doc to index',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -456,7 +459,8 @@ def callback( channel, method, properties, body ):
step='metadata-processor',
status='terminated',
uuid_prefix='meta',
- info='no info'
+ info='no info',
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
diff --git a/workers/reindexer.py b/workers/reindexer.py
index f7f35ef..f695990 100644
--- a/workers/reindexer.py
+++ b/workers/reindexer.py
@@ -40,7 +40,8 @@ def create_sampling_task(cfg, channel, uuid):
step='reindexer',
status='create sampling task',
uuid_prefix='full',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -58,7 +59,7 @@ def create_sampling_task(cfg, channel, uuid):
routing_key=cfg['rabbitmq']['routing_key_6'],
body=the_body,
properties=pika.BasicProperties(delivery_mode = 2)
- )
+ )
#connection.close()
@@ -135,7 +136,7 @@ def on_msg_callback(channel, method, properties, body):
# else:
# channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1)
# logging.error("The uuid ends neither with .full nor with .meta. What shall I do?")
- # return
+ # return
if count_es != count_ref:
@@ -157,7 +158,8 @@ def on_msg_callback(channel, method, properties, body):
step='reindexer',
status=message,
uuid_prefix='full',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -252,7 +254,8 @@ def on_msg_callback(channel, method, properties, body):
step='reindexer',
status=message,
uuid_prefix='full',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -278,7 +281,8 @@ def on_msg_callback(channel, method, properties, body):
step='reindexer',
status=message,
uuid_prefix='meta',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -308,7 +312,8 @@ def on_msg_callback(channel, method, properties, body):
step='reindexer',
status=message,
uuid_prefix='',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -344,7 +349,7 @@ def main(cfg):
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_message_callback=lambda ch, method, properties, body: on_msg_callback(ch, method, properties, body),
- queue=reindex_tasks_to_create_qn)
+ queue=reindex_tasks_to_create_qn)
channel.start_consuming()
diff --git a/workers/sample-generator.py b/workers/sample-generator.py
index 1ff6a03..9e5f2a8 100644
--- a/workers/sample-generator.py
+++ b/workers/sample-generator.py
@@ -188,7 +188,8 @@ def callback(channel, method, properties, body):
step='sampler',
status=message,
uuid_prefix='',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -238,7 +239,8 @@ def callback(channel, method, properties, body):
step='sampler',
status=message,
uuid_prefix='',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
@@ -278,7 +280,8 @@ def callback(channel, method, properties, body):
step='sampler',
status=message,
uuid_prefix='',
- info=uuid
+ info=uuid,
+ loglevel='INFO'
)
json_body = json.dumps(log_message.__dict__)
--
GitLab