Updated Docker-related files

Dockerfile

@@ -2,12 +2,11 @@ FROM python:3-slim
 
 WORKDIR /app
 
-
 COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
 COPY *.py ./
 #COPY config.yaml .
 COPY es_template.py .
 COPY utils/*.py ./utils/
-
-
-RUN pip install -r requirements.txt

docker-compose-masters.yml

@@ -28,7 +28,7 @@ services:
   field-type-detector:
     build: .
     image: data-grandlyon-com-indexer
-    command: python 5-field-type-detector.py
+    command: python 5-pg-field-type-detector.py
     volumes:
       - ./config.yaml:/app/config.yaml:ro
       - working-directory:/app/output

docker-compose-workers.yml

 version: "3.0"
 
 services:
-  rabbitmq:
-    image: rabbitmq:3-management
-    volumes:
-      - rabbitmq:/var/lib/rabbitmq
-    ports:
-      - 15672:15672
-      - 5672:5672
-    restart: unless-stopped
-
-  mongo:
-    image: mongo
-    #restart: unless-stopped
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: root
-      MONGO_INITDB_ROOT_PASSWORD: example
-    ports:
-      - 27017:27017
-    volumes:
-      - mongo:/data/db
+  # rabbitmq:
+  #   image: rabbitmq:3-management
+  #   volumes:
+  #     - rabbitmq:/var/lib/rabbitmq
+  #   ports:
+  #     - 15672:15672
+  #     - 5672:5672
+  #   restart: unless-stopped
 
-  mongo-express:
-    image: mongo-express
-    #restart: unless-stopped
-    ports:
-      - 8081:8081
-    environment:
-      ME_CONFIG_MONGODB_ADMINUSERNAME: root
-      ME_CONFIG_MONGODB_ADMINPASSWORD: example
+  # mongo:
+  #   image: mongo
+  #   #restart: unless-stopped
+  #   environment:
+  #     MONGO_INITDB_ROOT_USERNAME: root
+  #     MONGO_INITDB_ROOT_PASSWORD: example
+  #   ports:
+  #     - 27017:27017
+  #   volumes:
+  #     - mongo:/data/db
+  #
+  # mongo-express:
+  #   image: mongo-express
+  #   #restart: unless-stopped
+  #   ports:
+  #     - 8081:8081
+  #   environment:
+  #     ME_CONFIG_MONGODB_ADMINUSERNAME: root
+  #     ME_CONFIG_MONGODB_ADMINPASSWORD: example
 
-  reset-session:
-    build: .
-    image: data-grandlyon-com-indexer
-    command: python 0-reset-session.py
-    volumes:
-      - ./config.yaml:/app/config.yaml:ro
-      - working-directory:/app/output
+  # reset-session:
+  #   build: .
+  #   image: data-grandlyon-com-indexer
+  #   command: python 0-reset-session.py
+  #   volumes:
+  #     - ./config.yaml:/app/config.yaml:ro
+  #     - working-directory:/app/output
 
-  metadata-getter:
-    build: .
-    image: data-grandlyon-com-indexer
-    command: python 1-metadata-getter.py
-    volumes:
-      - ./config.yaml:/app/config.yaml:ro
+  # metadata-getter:
+  #   build: .
+  #   image: data-grandlyon-com-indexer
+  #   command: python 1-metadata-getter.py
+  #   volumes:
+  #     - ./config.yaml:/app/config.yaml:ro
 
   metadata-processor:
     build: .
     image: data-grandlyon-com-indexer
-    command: python 2-metadata-processor.py
+    command: python 2-metadata-processor.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_metadata_records_to_process
     volumes:
-      - ./config.yaml:/app/config.yaml:ro
+        - ./config.yaml:/app/config.yaml:ro
+        - working-directory:/app/output
+    restart: unless-stopped
 
   doc-enricher:
-    build: .
+    #build: .
     image: data-grandlyon-com-indexer
-    command: python 3-doc-enricher.py
+    command: python 3-doc-enricher.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_doc_pages_to_enrich
     volumes:
       - ./config.yaml:/app/config.yaml:ro
+    restart: unless-stopped
 
-  docs-to-mongodb:
-    build: .
-    image: data-grandlyon-com-indexer
-    command: python 4-docs-to-mongodb.py
-    volumes:
-      - ./config.yaml:/app/config.yaml:ro
+  # docs-to-mongodb:
+  #   build: .
+  #   image: data-grandlyon-com-indexer
+  #   command: python 4-docs-to-mongodb.py
+  #   volumes:
+  #     - ./config.yaml:/app/config.yaml:ro
 
-  field-type-detector:
-    build: .
-    image: data-grandlyon-com-indexer
-    command: python 5-field-type-detector.py
-    volumes:
-      - ./config.yaml:/app/config.yaml:ro
-      - working-directory:/app/output
+  # field-type-detector:
+  #   build: .
+  #   image: data-grandlyon-com-indexer
+  #   command: python 5-field-type-detector.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_doc_pages_to_enrich
+  #   volumes:
+  #     - ./config.yaml:/app/config.yaml:ro
+  #     - working-directory:/app/output
 
   doc-processor:
-    build: .
+    #build: .
     image: data-grandlyon-com-indexer
-    command: python 6-doc-processor.py
+    command: python 6-doc-processor.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_doc_pages_to_process
     volumes:
       - ./config.yaml:/app/config.yaml:ro
       - working-directory:/app/output
+    restart: unless-stopped
 
   doc-indexer:
-    build: .
+    #build: .
     image: data-grandlyon-com-indexer
-    command: python 7-doc-indexer.py
+    command: python 7-doc-indexer.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_doc_pages_to_index
     volumes:
       - ./config.yaml:/app/config.yaml:ro
+    restart: unless-stopped
 
   reindexer:
-    build: .
+    #build: .
+    image: data-grandlyon-com-indexer
+    command: python 8-incremental-reindexer.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_reindex_tasks
+    volumes:
+      - ./config.yaml:/app/config.yaml:ro
+    restart: unless-stopped
+
+  sampler:
+    #build: .
     image: data-grandlyon-com-indexer
-    command: python 8-reindexer.py
+    command: python 9-sample-generator.py --host rabbitmq --exchange download_data_grandlyon_com_index --queue abcd1234_sampling_tasks
     volumes:
       - ./config.yaml:/app/config.yaml:ro
+    restart: unless-stopped
+
+
 
 volumes:
   rabbitmq:
-  mongo:
+  #mongo:
   working-directory: