Merge branch 'development' into 'master'

Development

See merge request !3

README.md

@@ -18,6 +18,26 @@ A simplified overview of the entire workflow is provided by the attached [draw.i
 5. Wait !
 6. `$ curl -X GET http://<the_hostname_where_the_API_is_running>:<the_API_listening_port/uuid/<the_uuid_of_a_given_dataset|all>[?force=true]`
 
+## Aliases migration
+
+This project also include a script that allow one to migrate aliases from one instance of elasticsearch to another.
+
+Exemple d'usage :
+
+```python
+python tools/alias_copier.py --src-es https://<source-host>:443 --dst-es https://<destination-host>:443 --src-idx <src-index> --dst-idx <dst-index> --skip <ex: preprod>
+```
+
+Prefixes or suffixes to the alias with `--prepend` and `--append`.
+
+It is possible to skip the copy of aliases including a particular string. The argument takes a list of strings: `--skip bar foo`.
+
+La liste complète des arguments est visible en executant la commande suivante: 
+
+```python
+python tools/alias_copier.py --help
+```
+
 # TODO
 
 * producing indexation reports out of log messages (cf. the branches `Denis_clean_full_datalogger_31Oct` and `Denis_full_datalogs_Stack_October_31`)

lib/elasticsearch_template.py

@@ -111,7 +111,7 @@ template = {
                 {
                     "keyword-template" : {
                         "match_pattern": "regex",
-                        "path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid",
+                        "path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.name|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid",
                         "mapping": {
                             "type": "text",
                             "index": False,

tools/alias_copier.py

+import argparse
+import json
+from elasticsearch import Elasticsearch
+
+
+def copy_aliases(src_es, dst_es, src_idx, dst_idx, skip_list=None, prepend=None, append=None):
+
+    source_es = Elasticsearch([src_es], timeout=60)
+    destin_es = Elasticsearch([dst_es], timeout=60)
+
+    print(f'Getting aliases from {src_es}/{src_idx} ...')
+    try:
+        source_aliases = source_es.indices.get_alias(index=args.src_idx, name="*")
+    except Exception as e:
+        raise(e)
+
+    total = len(source_aliases[src_idx]['aliases'])
+    print('...done! %i aliases found' % total)
+
+    aliases_to_copy = {k: v for k, v in source_aliases[src_idx]['aliases'].items() if all(y not in k for y in skip_list)}
+    to_copy = len(aliases_to_copy)
+
+    print("")
+    print(f"{to_copy}/{total} aliases to copy; {total-to_copy}/{total} aliases to skip")
+    print("")
+
+    print("Copying aliases...")
+    print(f"from {src_es}/{src_idx}")
+    print(f"to   {dst_es}/{dst_idx}")
+
+    cnt = 0
+
+    for src_alias_name, alias_body in aliases_to_copy.items():
+
+        cnt += 1
+
+        dst_alias_name = src_alias_name
+
+        if prepend != None:
+            dst_alias_name = prepend + dst_alias_name
+
+        if append != None:
+            dst_alias_name = dst_alias_name + append
+
+        print(f"{cnt}/{len(aliases_to_copy)} {src_es}/{src_alias_name} -> {dst_es}/{dst_alias_name}")
+
+        try:
+            destin_es.indices.delete_alias(index='_all', name=dst_alias_name, ignore=404)
+        except Exception as e:
+            raise(e)
+
+        try:
+            destin_es.indices.put_alias(index=dst_idx, name=dst_alias_name, body=alias_body)
+        except Exception as e:
+            raise(e)
+
+    print('...done!')
+
+    return
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='This tool copies aliases across Elasticsearch instances.')
+    parser.add_argument('--src-es', dest='src_es', help='the source Elasticsearch URL (including the port)', type=str, required=True)
+    parser.add_argument('--dst-es', dest='dst_es', help='the destination Elasticsearch URL (including the port)', type=str, required=True)
+    parser.add_argument('--src-idx', dest='src_idx', help='the source index', type=str, required=True)
+    parser.add_argument('--dst-idx', dest='dst_idx', help='the destination index', type=str, required=True)
+    parser.add_argument('--skip', dest="skip", nargs='+', help='do not copy aliases including the provided substrings!', type=str, required=False)
+    parser.add_argument('--preprend', dest="preprend", help='the string to preprend to alias names', type=str, required=False)
+    parser.add_argument('--append', dest="append", help='the string to append to alias names', type=str, required=False)
+    args = parser.parse_args()
+
+    try:
+        copy_aliases(args.src_es, args.dst_es, args.src_idx, args.dst_idx, args.skip, args.preprend, args.append)
+    except Exception as e:
+        print(e)
+        exit(1)