Renamed lib folder

.gitignore

@@ -18,7 +18,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+#lib/
 lib64/
 parts/
 sdist/

2-metadata-processor.py

@@ -10,11 +10,11 @@ import datetime
 from copy import deepcopy
 from dateutil.parser import parse
 from collections import OrderedDict
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
-from utils.fix_links import fix_links
-from utils.enrich_links import enrich_links
-from utils.generate_slug import generate_slug
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.fix_links import fix_links
+from lib.enrich_links import enrich_links
+from lib.generate_slug import generate_slug
 
 
 def list_to_dictlist( the_input, the_context=None ):
@@ -417,7 +417,7 @@ def create_reindex_task( the_cfg, the_channel, the_uuid ):
 
 def main(cfg):
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     #logging.debug(cfg)
     #global connection
@@ -465,8 +465,6 @@ if __name__ == '__main__':
     logging.getLogger().setLevel(args.loglevel)
     logging.info('Starting...')
 
-    main(cfg)
-    exit(0)
     while True:
         try:
             main(cfg)

3-doc-enricher.py

@@ -4,10 +4,10 @@ import requests
 import json
 import datetime
 #from decimal import Decimal
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
-from utils.postgis_helper import Remote
-from utils.serializers import encode_datetime
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.postgis_helper import Remote
+from lib.serializers import encode_datetime
 from sqlalchemy.exc import NoSuchTableError
 
 
@@ -256,7 +256,7 @@ def enrich_docs( channel, method, properties, body ):
 
 def main(cfg):
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'], port=cfg['rabbitmq_port']))
     #timeout = 5
@@ -316,7 +316,7 @@ if __name__ == '__main__':
             time.sleep(5)
         except pika.exceptions.AMQPConnectionError:
             logging.info('Waiting for RabbitMQ to be reachable...')
-            time.sleep(5)    
+            time.sleep(5)
         except Exception as e:
             logging.error(e)
             time.sleep(5)

5-pg-field-type-detector.py

@@ -3,11 +3,11 @@ import json
 import re
 import time
 
-from utils.flatten_utils import flatten_json, unflatten_json
-from utils.type_utils import detect_type, detect_type_new
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
-from utils.postgis_helper import Remote
+from lib.flatten_utils import flatten_json, unflatten_json
+from lib.type_utils import detect_type, detect_type_new
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.postgis_helper import Remote
 
 
 

6-doc-processor.py

@@ -7,13 +7,13 @@ import re
 import json
 import sys
 import os
-from utils.flatten_utils import flatten_json, unflatten_json
+from lib.flatten_utils import flatten_json, unflatten_json
 #import pytz
 #import sys
-from utils.type_utils import convert_to_datetime, convert_to_float, convert_to_int, convert_to_str, convert_to_boolean
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
-from utils.serializers import decode_datetime
+from lib.type_utils import convert_to_datetime, convert_to_float, convert_to_int, convert_to_str, convert_to_boolean
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.serializers import decode_datetime
 
 
 def fix_field_types( in_docs, out_types ):
@@ -128,7 +128,7 @@ def process_docs( channel, method, properties, body ):
 
 def main(cfg):
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'],port=cfg['rabbitmq_port']))
     #timeout = 5

7-doc-indexer.py

@@ -5,8 +5,8 @@ import json
 from elasticsearch import Elasticsearch
 from elasticsearch.exceptions import AuthorizationException
 import hashlib
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
 
 from pprint import pprint
 
@@ -132,7 +132,7 @@ def index_docs(channel, method, properties, body):
 
 def main(cfg):
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'], port=cfg['rabbitmq_port']))
     # timeout = 5

8-incremental-reindexer.py

@@ -3,8 +3,8 @@ import json
 import msgpack
 import pika
 from elasticsearch import Elasticsearch, NotFoundError
-from utils.my_logging import logging
-from utils.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.exit_gracefully import exit_gracefully
 
 class NotEmptyQueueException(Exception):
     pass
@@ -187,7 +187,7 @@ def on_msg_callback(channel, method, properties, body):
 
 def main(cfg):
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'], port=cfg['rabbitmq_port']))
     #timeout = 5

8-reindexer.py

@@ -3,8 +3,8 @@ from pymongo import MongoClient
 import time
 import msgpack
 import pika
-from utils.my_logging import logging
-from utils.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
+from lib.exit_gracefully import exit_gracefully
 
 class NotEmptyQueueException(Exception):
     pass

9-sample-generator.py

@@ -4,8 +4,8 @@ import requests
 import json
 import time
 from elasticsearch import Elasticsearch
-from utils.exit_gracefully import exit_gracefully
-from utils.my_logging import logging
+from lib.exit_gracefully import exit_gracefully
+from lib.my_logging import logging
 
 
 def old_callback(channel, method, properties, body):
@@ -200,7 +200,7 @@ def main(cfg):
     # es_logger = logging.getLogger('elasticsearch')
     # es_logger.setLevel(logging.INFO)
 
-    #from utils.close_connection import on_timeout
+    #from lib.close_connection import on_timeout
 
     connection = pika.BlockingConnection(pika.ConnectionParameters(host=cfg['rabbitmq_host'], port=cfg['rabbitmq_port']))
     #timeout = 5

Dockerfile

@@ -9,4 +9,4 @@ RUN pip install -r requirements.txt
 COPY *.py ./
 #COPY config.yaml .
 COPY es_template.py .
-COPY utils/*.py ./utils/
+COPY lib/*.py ./lib/