From 18f6de96e8d6cebf689610a33b0f563763b50660 Mon Sep 17 00:00:00 2001 From: ddamiron <ddamiron@sii.fr> Date: Tue, 2 Jul 2019 15:59:56 +0200 Subject: [PATCH] update README add filterkey/value endpoint --- README.md | 14 ++++++++++++-- api.py | 10 +++++----- lib/mongo_session.py | 4 ++-- ...read_errors_logs.py => read_and_filter_logs.py} | 12 ++++++++---- 4 files changed, 27 insertions(+), 13 deletions(-) rename lib/{read_errors_logs.py => read_and_filter_logs.py} (74%) diff --git a/README.md b/README.md index ce4638f..307f51e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ A simplified overview of the entire workflow is provided by the attached [draw.i 8. Run `docker-compose -f docker-compose-tools.yml up metadata-getter` -TO connect to metabase: +## To connect to metabase: go to <your-url>:3001 use the interface to create an account @@ -33,7 +33,17 @@ connect to the mongo database setting the parameters: 5. Port: <mongo_port> 6. Database username: <mongo_user_login> 7. Database password: <mongo_user_password> -# TODO + +## using uuid endpoint to launch main method: +http://localhost:8000/uuid/<uuid> +example: +http://localhost:8000/uuid/e23c6d3e-40be-4d5e-bc15-4b7e7313942f + +## using status end-point for filtering mongo queries: +/status/<session_id>/<query_key>/<query_value> +example: +/status/bb0764fe-324e-4719-9214-dc4abc59fe50/step/doc-processor +## TODO * adding an HTTP API to trigger indexation * producing indexation reports diff --git a/api.py b/api.py index ba6c4f2..92fba98 100644 --- a/api.py +++ b/api.py @@ -23,7 +23,7 @@ except ImportError: from lib.read_status_logs import ReadLogger from lib.get_most_recent_status_logs import GetLatestLog -from lib.read_errors_logs import ReadErrorLogger +from lib.read_and_filter_logs import ReadFilterLogger # read 'n' parse the configuration with open("config.yaml", 'r') as yamlfile: @@ -117,10 +117,10 @@ def lateststatus(session_id): return Response(json.dumps(body, indent=4, sort_keys=True, default=str), mimetype='application/json') -@api.route('/status/<session_id>/error') -def taskerror(session_id): - my_task_errors = ReadErrorLogger(session_id=session_id, cfg=cfg) - body = my_task_errors.main() +@api.route('/status/<session_id>/<query_key>/<query_value>') +def taskfilter(session_id, query_key, query_value): + my_task_filter = ReadFilterLogger(session_id=session_id, cfg=cfg, query_key=query_key, query_value=query_value) + body = my_task_filter.main() return Response(json.dumps(body, indent=4, sort_keys=True, default=str), mimetype='application/json') diff --git a/lib/mongo_session.py b/lib/mongo_session.py index 2b807c6..44ae95a 100644 --- a/lib/mongo_session.py +++ b/lib/mongo_session.py @@ -69,11 +69,11 @@ class MongoSession: except Exception as exc: print('[ERROR reading log]:', exc) - def read_mongo_errors(self, this_session_id): + def read_and_filter_mongo(self, this_session_id, query_key, query_value): try: data = [] request_result = self.mongo_data_collection.find({"session_id": this_session_id, - "loglevel": "ERROR"}) + query_key: query_value}) for res in request_result: data.append(res) return data diff --git a/lib/read_errors_logs.py b/lib/read_and_filter_logs.py similarity index 74% rename from lib/read_errors_logs.py rename to lib/read_and_filter_logs.py index 7e9d977..5687bcd 100644 --- a/lib/read_errors_logs.py +++ b/lib/read_and_filter_logs.py @@ -24,14 +24,18 @@ newPath = os.path.join(parentDir) sys.path.append(newPath) -class ReadErrorLogger: - def __init__(self, session_id, cfg): +class ReadFilterLogger: + def __init__(self, session_id, cfg, query_key, query_value): self.session_id = session_id + self.query_key = query_key + self.query_value = query_value self.cfg = cfg def main(self): - response_body = MongoSession(self.cfg).read_mongo_errors(self.session_id) + response_body = MongoSession(self.cfg).read_and_filter_mongo(self.session_id, + query_key=self.query_key, + query_value=self.query_value) return response_body @@ -44,7 +48,7 @@ if __name__ == '__main__': cfg = load(yamlfile, Loader=Loader) try: session_id = '17-06-14:57-indexation' - my_process_logger = ReadErrorLogger(session_id=session_id, cfg=cfg) + my_process_logger = ReadFilterLogger(session_id=session_id, cfg=cfg) data = my_process_logger.main() print('data : ', data) except pika.exceptions.ChannelClosed: -- GitLab