diff --git a/README.md b/README.md index ce4638fa799c339f8df62d06ba879c1ae74b5dcc..307f51e64176a46ac06a5235486705eaaecfe8d8 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ A simplified overview of the entire workflow is provided by the attached [draw.i 8. Run `docker-compose -f docker-compose-tools.yml up metadata-getter` -TO connect to metabase: +## To connect to metabase: go to <your-url>:3001 use the interface to create an account @@ -33,7 +33,17 @@ connect to the mongo database setting the parameters: 5. Port: <mongo_port> 6. Database username: <mongo_user_login> 7. Database password: <mongo_user_password> -# TODO + +## using uuid endpoint to launch main method: +http://localhost:8000/uuid/<uuid> +example: +http://localhost:8000/uuid/e23c6d3e-40be-4d5e-bc15-4b7e7313942f + +## using status end-point for filtering mongo queries: +/status/<session_id>/<query_key>/<query_value> +example: +/status/bb0764fe-324e-4719-9214-dc4abc59fe50/step/doc-processor +## TODO * adding an HTTP API to trigger indexation * producing indexation reports diff --git a/api.py b/api.py index ba6c4f25eb0ea30346185604f7a37408f910acf6..92fba98ef551712ae6a5e249c33dacf1f9ba6640 100644 --- a/api.py +++ b/api.py @@ -23,7 +23,7 @@ except ImportError: from lib.read_status_logs import ReadLogger from lib.get_most_recent_status_logs import GetLatestLog -from lib.read_errors_logs import ReadErrorLogger +from lib.read_and_filter_logs import ReadFilterLogger # read 'n' parse the configuration with open("config.yaml", 'r') as yamlfile: @@ -117,10 +117,10 @@ def lateststatus(session_id): return Response(json.dumps(body, indent=4, sort_keys=True, default=str), mimetype='application/json') -@api.route('/status/<session_id>/error') -def taskerror(session_id): - my_task_errors = ReadErrorLogger(session_id=session_id, cfg=cfg) - body = my_task_errors.main() +@api.route('/status/<session_id>/<query_key>/<query_value>') +def taskfilter(session_id, query_key, query_value): + my_task_filter = ReadFilterLogger(session_id=session_id, cfg=cfg, query_key=query_key, query_value=query_value) + body = my_task_filter.main() return Response(json.dumps(body, indent=4, sort_keys=True, default=str), mimetype='application/json') diff --git a/lib/mongo_session.py b/lib/mongo_session.py index 2b807c6314b0e036dbdf9563115fb90cec9b2482..44ae95aab6d52970b9f13bede5d587ed5bba831a 100644 --- a/lib/mongo_session.py +++ b/lib/mongo_session.py @@ -69,11 +69,11 @@ class MongoSession: except Exception as exc: print('[ERROR reading log]:', exc) - def read_mongo_errors(self, this_session_id): + def read_and_filter_mongo(self, this_session_id, query_key, query_value): try: data = [] request_result = self.mongo_data_collection.find({"session_id": this_session_id, - "loglevel": "ERROR"}) + query_key: query_value}) for res in request_result: data.append(res) return data diff --git a/lib/read_errors_logs.py b/lib/read_and_filter_logs.py similarity index 74% rename from lib/read_errors_logs.py rename to lib/read_and_filter_logs.py index 7e9d977da33bf7364f87b1624853c95eddab6820..5687bcd32c0e9c21e16d3b50f59b30d309081cb0 100644 --- a/lib/read_errors_logs.py +++ b/lib/read_and_filter_logs.py @@ -24,14 +24,18 @@ newPath = os.path.join(parentDir) sys.path.append(newPath) -class ReadErrorLogger: - def __init__(self, session_id, cfg): +class ReadFilterLogger: + def __init__(self, session_id, cfg, query_key, query_value): self.session_id = session_id + self.query_key = query_key + self.query_value = query_value self.cfg = cfg def main(self): - response_body = MongoSession(self.cfg).read_mongo_errors(self.session_id) + response_body = MongoSession(self.cfg).read_and_filter_mongo(self.session_id, + query_key=self.query_key, + query_value=self.query_value) return response_body @@ -44,7 +48,7 @@ if __name__ == '__main__': cfg = load(yamlfile, Loader=Loader) try: session_id = '17-06-14:57-indexation' - my_process_logger = ReadErrorLogger(session_id=session_id, cfg=cfg) + my_process_logger = ReadFilterLogger(session_id=session_id, cfg=cfg) data = my_process_logger.main() print('data : ', data) except pika.exceptions.ChannelClosed: