Preventing the doc-indexer from entering kind of an infinite loop in case...

Preventing the doc-indexer from entering kind of an infinite loop in case documents cannot be pushed to Elasticsearch.

workers/doc-indexer.py

@@ -113,10 +113,8 @@ def index_docs(channel, method, properties, body):
         # header['index']['_id'] = hashlib.md5( json.dumps(tmp, sort_keys=True).encode("utf-8") ).hexdigest()
         es_body += '{0}\n{1}\n'.format(json.dumps(header), json.dumps(doc))
 
-
     rep = es.bulk(body=es_body)
 
-    #print(rep)
     t2 = time.time()
 
     if rep['errors'] == False:
@@ -157,23 +155,17 @@ def index_docs(channel, method, properties, body):
                                         properties=pika.BasicProperties(delivery_mode = 2))
 
 
-
     else:
         channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1)
         #print("")
-        logging.error(json.dumps(rep, indent=4))
-        logging.error("Failed")
-
-
-    #time.sleep(5)
+        #logging.error(json.dumps(rep, indent=4))
+        raise Exception('Failed to push documents to Elasticsearch.')
 
     return
 
 
 def main(cfg):
 
-    #from lib.close_connection import on_timeout
-
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'], port=cfg['rabbitmq_port']))
     # timeout = 5
     # connection.add_timeout(timeout, on_timeout(connection))
@@ -232,3 +224,8 @@ if __name__ == '__main__':
             logging.error(e)
             time.sleep(5)
             exit(1)
+        except Exception as e:
+            logging.error(e)
+            logging.info('Sleeping for 5 seconds before shutting down.')
+            time.sleep(5)
+            exit(1)