add plot step graph end-point

api.py

@@ -3,6 +3,9 @@ import atexit
 import argparse
 import uuid
 import json
+import matplotlib.pyplot as plt
+import io
+import base64
 
 from flask import Flask, jsonify, Response
 
@@ -24,6 +27,7 @@ except ImportError:
 from lib.read_status_logs import ReadLogger
 from lib.get_most_recent_status_logs import GetLatestLog
 from lib.read_and_filter_logs import ReadFilterLogger
+from lib.mongo_session import MongoSession
 
 # read 'n' parse the configuration
 with open("config.yaml", 'r') as yamlfile:
@@ -41,6 +45,7 @@ cfg['rabbitmq']['exchange'] = os.environ['RMQ_EXCHANGE']
 logging.getLogger().setLevel(os.environ['LOGLEVEL'])
 
 
+
 # def say_hi(n):
 #     import time
 #     time.sleep(n)
@@ -125,5 +130,40 @@ def taskfilter(session_id, query_key, query_value):
     return Response(json.dumps(body, indent=4, sort_keys=True, default=str), mimetype='application/json')
 
 
+@api.route('/plot/<session_id>/<step>')
+def build_plot(session_id, step):
+
+    img = io.BytesIO()
+    x, y = MongoSession(cfg=cfg).get_array_from_step(this_session_id=session_id, step_name=step)
+    # y = [1,2,3,4,5]
+    # x = [0,2,1,3,4]
+    plt.clf()
+    plt.plot(x, y, label=step)
+    plt.savefig(img, format='png')
+    img.seek(0)
+
+    plot_url = base64.b64encode(img.getvalue()).decode()
+
+    return '<img src="data:image/png;base64,{}">'.format(plot_url)
+
+
+@api.route('/plot/<session_id>')
+def build_all_plot(session_id):
+    img = io.BytesIO()
+    plt.clf()
+    for step in ['main', 'doc-enricher', 'doc-processor', 'doc-indexer']:
+        x, y = MongoSession(cfg=cfg).get_array_from_step(this_session_id=session_id, step_name=step)
+        # y = [1,2,3,4,5]
+        # x = [0,2,1,3,4]
+        plt.plot(x, y, label=step)
+    plt.legend()
+    plt.savefig(img, format='png')
+    img.seek(0)
+
+    plot_url = base64.b64encode(img.getvalue()).decode()
+
+    return '<img src="data:image/png;base64,{}">'.format(plot_url)
+
+
 if __name__ == '__main__':
     api.run(host='0.0.0.0', port=8000, debug=True)

lib/mongo_session.py

@@ -95,3 +95,39 @@ class MongoSession:
             return data
         except Exception as exc:
             print('[ERROR reading log]:', exc)
+
+    def get_array_from_step(self, this_session_id, step_name):
+        try:
+            data_time = []
+            data_value = []
+
+            request_result = self.mongo_data_collection.find({"session_id": this_session_id, "step": step_name},
+                                                             {'timestamp': 1, 'progress_ratio': 1})
+
+            for res in request_result:
+                data_time.append(res['timestamp'])
+                data_value.append(res['progress_ratio'])
+
+            return data_time, data_value
+        except Exception as exc:
+            print('[ERROR reading log]:', exc)
+
+
+if __name__ == '__main__':
+    # TODO the config below should be extracted from the config.yaml file
+    cfg = dict()
+    cfg['rabbitmq'] = dict()
+    cfg['mongo'] = dict()
+
+    cfg['mongo']['host'] = 'localhost'
+    cfg['mongo']['database'] = 'indexerdb'
+    cfg['mongo']['port'] = 27017
+    cfg['mongo']['user'] = 'root'
+    cfg['mongo']['password'] = 'example'
+    cfg['mongo']['collection'] = 'indexer_logs'
+    this_session_id = 'bb0764fe-324e-4719-9214-dc4abc59fe50'
+    data_time, data_value = MongoSession(cfg=cfg).get_array_from_step(this_session_id, "doc-enricher")
+    print('data_time: ', data_time)
+    print('data_value: ', data_value)
+
+

requirements.txt

@@ -18,3 +18,4 @@ flask
 gunicorn
 apscheduler
 flask-executor
+matplotlib>=3.1.0