diff --git a/README.md b/README.md index 6c9f0c3888af929a2fab831a8cdd8b2510e45627..0edc1d7f2c2009be37a8e8526c5af09be1be0998 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,26 @@ A simplified overview of the entire workflow is provided by the attached [draw.i 5. Wait ! 6. `$ curl -X GET http://<the_hostname_where_the_API_is_running>:<the_API_listening_port/uuid/<the_uuid_of_a_given_dataset|all>[?force=true]` +## Aliases migration + +This project also include a script that allow one to migrate aliases from one instance of elasticsearch to another. + +Exemple d'usage : + +```python +python tools/alias_copier.py --src-es https://<source-host>:443 --dst-es https://<destination-host>:443 --src-idx <src-index> --dst-idx <dst-index> --skip <ex: preprod> +``` + +Prefixes or suffixes to the alias with `--prepend` and `--append`. + +It is possible to skip the copy of aliases including a particular string. The argument takes a list of strings: `--skip bar foo`. + +La liste complète des arguments est visible en executant la commande suivante: + +```python +python tools/alias_copier.py --help +``` + # TODO * producing indexation reports out of log messages (cf. the branches `Denis_clean_full_datalogger_31Oct` and `Denis_full_datalogs_Stack_October_31`) diff --git a/lib/elasticsearch_template.py b/lib/elasticsearch_template.py index b953b953ce2656aeda6a26bdc648852c55ae29a0..5d98482c120cf8c8e5014ccb764bbccba76c5e38 100644 --- a/lib/elasticsearch_template.py +++ b/lib/elasticsearch_template.py @@ -111,7 +111,7 @@ template = { { "keyword-template" : { "match_pattern": "regex", - "path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid", + "path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.name|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid", "mapping": { "type": "text", "index": False, diff --git a/tools/alias_copier.py b/tools/alias_copier.py new file mode 100644 index 0000000000000000000000000000000000000000..5233a7c6653735609f7bd9152544c028c0081173 --- /dev/null +++ b/tools/alias_copier.py @@ -0,0 +1,78 @@ +import argparse +import json +from elasticsearch import Elasticsearch + + +def copy_aliases(src_es, dst_es, src_idx, dst_idx, skip_list=None, prepend=None, append=None): + + source_es = Elasticsearch([src_es], timeout=60) + destin_es = Elasticsearch([dst_es], timeout=60) + + print(f'Getting aliases from {src_es}/{src_idx} ...') + try: + source_aliases = source_es.indices.get_alias(index=args.src_idx, name="*") + except Exception as e: + raise(e) + + total = len(source_aliases[src_idx]['aliases']) + print('...done! %i aliases found' % total) + + aliases_to_copy = {k: v for k, v in source_aliases[src_idx]['aliases'].items() if all(y not in k for y in skip_list)} + to_copy = len(aliases_to_copy) + + print("") + print(f"{to_copy}/{total} aliases to copy; {total-to_copy}/{total} aliases to skip") + print("") + + print("Copying aliases...") + print(f"from {src_es}/{src_idx}") + print(f"to {dst_es}/{dst_idx}") + + cnt = 0 + + for src_alias_name, alias_body in aliases_to_copy.items(): + + cnt += 1 + + dst_alias_name = src_alias_name + + if prepend != None: + dst_alias_name = prepend + dst_alias_name + + if append != None: + dst_alias_name = dst_alias_name + append + + print(f"{cnt}/{len(aliases_to_copy)} {src_es}/{src_alias_name} -> {dst_es}/{dst_alias_name}") + + try: + destin_es.indices.delete_alias(index='_all', name=dst_alias_name, ignore=404) + except Exception as e: + raise(e) + + try: + destin_es.indices.put_alias(index=dst_idx, name=dst_alias_name, body=alias_body) + except Exception as e: + raise(e) + + print('...done!') + + return + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='This tool copies aliases across Elasticsearch instances.') + parser.add_argument('--src-es', dest='src_es', help='the source Elasticsearch URL (including the port)', type=str, required=True) + parser.add_argument('--dst-es', dest='dst_es', help='the destination Elasticsearch URL (including the port)', type=str, required=True) + parser.add_argument('--src-idx', dest='src_idx', help='the source index', type=str, required=True) + parser.add_argument('--dst-idx', dest='dst_idx', help='the destination index', type=str, required=True) + parser.add_argument('--skip', dest="skip", nargs='+', help='do not copy aliases including the provided substrings!', type=str, required=False) + parser.add_argument('--preprend', dest="preprend", help='the string to preprend to alias names', type=str, required=False) + parser.add_argument('--append', dest="append", help='the string to append to alias names', type=str, required=False) + args = parser.parse_args() + + try: + copy_aliases(args.src_es, args.dst_es, args.src_idx, args.dst_idx, args.skip, args.preprend, args.append) + except Exception as e: + print(e) + exit(1)