Waiting progressively longer for delete_by_query to be done - II

workers/reindexer.py

@@ -140,13 +140,15 @@ def on_msg_callback(channel, method, properties, body):
         logging.debug(res)
         task_id = res['task']
         # wait until ES is done
+        seconds_to_sleep_for = 1
         while True:
             res = es.tasks.get(task_id=task_id)
             logging.debug(res)
             completed = res['completed']
             if not completed:
-                logging.info('Waiting for delete_by_query to complete...')
-                time.sleep(5)
+                logging.info('Waiting for delete_by_query to complete: sleeping for %i seconds...' % seconds_to_sleep_for)
+                time.sleep(seconds_to_sleep_for)
+                seconds_to_sleep_for += 1
             else:
                 break
     except NotFoundError:

workers/sample-generator.py

@@ -177,13 +177,15 @@ def callback(channel, method, properties, body):
         logging.debug(res)
         task_id = res['task']
         # wait until ES is done
+        seconds_to_sleep_for = 1
         while True:
-            res = destin_es.tasks.get(task_id=task_id)
+            res = es.tasks.get(task_id=task_id)
             logging.debug(res)
             completed = res['completed']
             if not completed:
-                logging.info('Waiting for delete_by_query to complete...')
-                time.sleep(5)
+                logging.info('Waiting for delete_by_query to complete: sleeping for %i seconds...' % seconds_to_sleep_for)
+                time.sleep(seconds_to_sleep_for)
+                seconds_to_sleep_for += 1
             else:
                 break
     except Exception as e: