Improvement.

The previous configuration files (config.yaml + config-docker.yaml) were replaced by a unique template configuration file.

config-docker.yaml → config.template.yaml

 session:
-  id: abcd1234
+  id: <the session id, ex.: abcd1234>
   working_directory: output
-  log_level: DEBUG
+  log_level: <the log level: DEBUG, INFO, WARNING, ERROR, INFO, CRITICAL>
 
 geonetwork:
-  url: https://download.data.grandlyon.com/catalogue/srv/fre/q
+  url: <the URL of GeoNetwork's q API, ex.: https://download.data.grandlyon.com/catalogue/srv/fre/q >
   records_per_page: 100
 
 wfs:
   features_per_page: 1000
 
 rabbitmq:
-  host:          rabbitmq
-  exchange:      download_data_grandlyon_com_index
+  host: <the RabbitMQ host. N.B.: in case docker-compose is used, this should correspond to the service name>
+  exchange: download_data_grandlyon_com_indexing
   routing_key_1_suffix: metadata_pages_to_process
   queue_name_1_suffix:  metadata_pages_to_process
   routing_key_2_suffix: doc_pages_to_index
@@ -29,33 +29,35 @@ rabbitmq:
 mongo:
   username: root
   password: example
-  host: mongo
+  host: <the MongoDB host. N.B.: in case docker-compose is used, this should correspond to the service name>
   port: 27017
-  data-db: download_data_grandlyon_com
-  report-db: download_data_grandlyon_com
+  data-db: <the name of the database where data are stored, ex.: download_data_grandlyon_com>
+  report-db: <the name of the database where reporting data are stored, ex.: download_data_grandlyon_com>
 
 metadata_getter:
   uuids_to_get:
-    #- all
-    - 46eac9e4-33cb-45b7-a104-7bcc8e654c98
+    #- all # 'all' means that all the uuids will be retrieved
+    - <1st uuid to get>
+    - <2nd uuid to get>
+    - <...>
   uuids_to_filter_out:
-    - ef5745f1-060d-496d-8aea-2461538964a8
-    - eff0fb32-d259-45b8-b662-9eb00c64ade5
-
+    - <1st uuid to filter out>
+    - <2nd uuid to filter out>
+    - <...>
 
 field_type_detector:
-  wait: true
-  rewrite: false
-  debug: false
+  wait: true # whether the field-type-detector should wait for the no. of docs in MongoDB to be stable
+  rewrite: false # whether it should overwrite already existing output files
+  debug: false # whether it should run in debug mode
 
 indexer:
-  url: https://elastic2.alpha.grandlyon.com
-  index: download.data.grandlyon.com.ingest.v1
+  url: <the root URL of Elasticsearch, ex.: https://elastic2.alpha.grandlyon.com>
+  index: <the destination index, ex.: download.data.grandlyon.com.ingest.v2>
 
 reindexer:
-  source_url: https://elastic2.alpha.grandlyon.com:443 # N.B.: the port number MUST be explicit
-  destination_url: https://elastic2.alpha.grandlyon.com
-  source_index: download.data.grandlyon.com.ingest.v1
-  destination_index: download.data.grandlyon.com.v1
-  template_name: download.data.grandlyon.com.v1
-  template_index_pattern: download.data.grandlyon.com.v1
+  #source_url: https://elastic2.alpha.grandlyon.com:443 # N.B.: the port number MUST be explicit; this parameter is optional
+  destination_url: <ex.: https://elastic2.alpha.grandlyon.com>
+  source_index: <ex.: download.data.grandlyon.com.ingest.v2>
+  destination_index: <ex.: download.data.grandlyon.com.v2>
+  template_name: <ex.: download.data.grandlyon.com.v2>
+  template_index_pattern: <ex.: download.data.grandlyon.com.v2>

config.yaml

-session:
-  id: abcd1234
-  working_directory: output
-  log_level: DEBUG
-
-geonetwork:
-  url: https://download.data.grandlyon.com/catalogue/srv/fre/q
-  records_per_page: 100
-
-wfs:
-  features_per_page: 1000
-
-rabbitmq:
-  host:          localhost
-  exchange:      download_data_grandlyon_com_index
-  routing_key_1_suffix: metadata_pages_to_process
-  queue_name_1_suffix:  metadata_pages_to_process
-  routing_key_2_suffix: doc_pages_to_index
-  queue_name_2_suffix:  doc_pages_to_index
-  routing_key_3_suffix: doc_pages_to_enrich
-  queue_name_3_suffix:  doc_pages_to_enrich
-  routing_key_4_suffix: doc_pages_to_store_in_mongo
-  queue_name_4_suffix:  doc_pages_to_store_in_mongo
-  routing_key_5_suffix: doc_pages_to_process
-  queue_name_5_suffix:  doc_pages_to_process
-  #routing_key_6_suffix: reindex_tasks
-  #queue_name_6_suffix:  reindex_tasks
-
-mongo:
-  username: root
-  password: example
-  host: localhost
-  port: 27017
-  data-db: download_data_grandlyon_com
-  report-db: download_data_grandlyon_com
-
-metadata_getter:
-  uuids_to_get:
-    #- all
-    - 46eac9e4-33cb-45b7-a104-7bcc8e654c98
-  uuids_to_filter_out:
-    - ef5745f1-060d-496d-8aea-2461538964a8
-    - eff0fb32-d259-45b8-b662-9eb00c64ade5
-
-
-field_type_detector:
-  wait: true
-  rewrite: false
-  debug: false
-
-indexer:
-  url: https://elastic2.alpha.grandlyon.com
-  index: download.data.grandlyon.com.ingest.v1
-
-reindexer:
-  source_url: https://elastic2.alpha.grandlyon.com:443 # N.B.: the port number MUST be explicit
-  destination_url: https://elastic2.alpha.grandlyon.com
-  source_index: download.data.grandlyon.com.ingest.v1
-  destination_index: download.data.grandlyon.com.v1
-  template_name: download.data.grandlyon.com.v1
-  template_index_pattern: download.data.grandlyon.com.v1

docker-compose.yml

@@ -35,7 +35,7 @@ services:
     image: data-grandlyon-com-indexer
     command: python 0-reset-session.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
       - working-directory:/app/output
 
   metadata-getter:
@@ -43,35 +43,35 @@ services:
     image: data-grandlyon-com-indexer
     command: python 1-metadata-getter.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
   metadata-processor:
     build: .
     image: data-grandlyon-com-indexer
     command: python 2-metadata-processor.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
   doc-enricher:
     build: .
     image: data-grandlyon-com-indexer
     command: python 3-doc-enricher.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
   docs-to-mongodb:
     build: .
     image: data-grandlyon-com-indexer
     command: python 4-docs-to-mongodb.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
   field-type-detector:
     build: .
     image: data-grandlyon-com-indexer
     command: python 5-field-type-detector.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
       - working-directory:/app/output
 
   doc-processor:
@@ -79,7 +79,7 @@ services:
     image: data-grandlyon-com-indexer
     command: python 6-doc-processor.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
       - working-directory:/app/output
 
   doc-indexer:
@@ -87,14 +87,14 @@ services:
     image: data-grandlyon-com-indexer
     command: python 7-doc-indexer.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
   reindexer:
     build: .
     image: data-grandlyon-com-indexer
     command: python 8-reindexer.py
     volumes:
-      - ./config-docker.yaml:/app/config.yaml:ro
+      - ./config.yaml:/app/config.yaml:ro
 
 volumes:
   rabbitmq: