Commit a198c75c authored by Fabien FORESTIER's avatar Fabien FORESTIER
Browse files

Merge branch 'development' into 'master'

Development

See merge request !3
parents e88a7095 50e2061d
Pipeline #5473 passed with stages
in 22 seconds
...@@ -18,6 +18,26 @@ A simplified overview of the entire workflow is provided by the attached [draw.i ...@@ -18,6 +18,26 @@ A simplified overview of the entire workflow is provided by the attached [draw.i
5. Wait ! 5. Wait !
6. `$ curl -X GET http://<the_hostname_where_the_API_is_running>:<the_API_listening_port/uuid/<the_uuid_of_a_given_dataset|all>[?force=true]` 6. `$ curl -X GET http://<the_hostname_where_the_API_is_running>:<the_API_listening_port/uuid/<the_uuid_of_a_given_dataset|all>[?force=true]`
## Aliases migration
This project also include a script that allow one to migrate aliases from one instance of elasticsearch to another.
Exemple d'usage :
```python
python tools/alias_copier.py --src-es https://<source-host>:443 --dst-es https://<destination-host>:443 --src-idx <src-index> --dst-idx <dst-index> --skip <ex: preprod>
```
Prefixes or suffixes to the alias with `--prepend` and `--append`.
It is possible to skip the copy of aliases including a particular string. The argument takes a list of strings: `--skip bar foo`.
La liste complète des arguments est visible en executant la commande suivante:
```python
python tools/alias_copier.py --help
```
# TODO # TODO
* producing indexation reports out of log messages (cf. the branches `Denis_clean_full_datalogger_31Oct` and `Denis_full_datalogs_Stack_October_31`) * producing indexation reports out of log messages (cf. the branches `Denis_clean_full_datalogger_31Oct` and `Denis_full_datalogs_Stack_October_31`)
......
...@@ -111,7 +111,7 @@ template = { ...@@ -111,7 +111,7 @@ template = {
{ {
"keyword-template" : { "keyword-template" : {
"match_pattern": "regex", "match_pattern": "regex",
"path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid", "path_match": ".*md5.*|metadata-fr\.link\.formats.*|metadata-fr\.link\.name|metadata-fr\.link\.service.*|metadata-fr\.parentId.*|metadata-fr\.geonet\:info\.uuid|slug|uuid",
"mapping": { "mapping": {
"type": "text", "type": "text",
"index": False, "index": False,
......
import argparse
import json
from elasticsearch import Elasticsearch
def copy_aliases(src_es, dst_es, src_idx, dst_idx, skip_list=None, prepend=None, append=None):
source_es = Elasticsearch([src_es], timeout=60)
destin_es = Elasticsearch([dst_es], timeout=60)
print(f'Getting aliases from {src_es}/{src_idx} ...')
try:
source_aliases = source_es.indices.get_alias(index=args.src_idx, name="*")
except Exception as e:
raise(e)
total = len(source_aliases[src_idx]['aliases'])
print('...done! %i aliases found' % total)
aliases_to_copy = {k: v for k, v in source_aliases[src_idx]['aliases'].items() if all(y not in k for y in skip_list)}
to_copy = len(aliases_to_copy)
print("")
print(f"{to_copy}/{total} aliases to copy; {total-to_copy}/{total} aliases to skip")
print("")
print("Copying aliases...")
print(f"from {src_es}/{src_idx}")
print(f"to {dst_es}/{dst_idx}")
cnt = 0
for src_alias_name, alias_body in aliases_to_copy.items():
cnt += 1
dst_alias_name = src_alias_name
if prepend != None:
dst_alias_name = prepend + dst_alias_name
if append != None:
dst_alias_name = dst_alias_name + append
print(f"{cnt}/{len(aliases_to_copy)} {src_es}/{src_alias_name} -> {dst_es}/{dst_alias_name}")
try:
destin_es.indices.delete_alias(index='_all', name=dst_alias_name, ignore=404)
except Exception as e:
raise(e)
try:
destin_es.indices.put_alias(index=dst_idx, name=dst_alias_name, body=alias_body)
except Exception as e:
raise(e)
print('...done!')
return
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='This tool copies aliases across Elasticsearch instances.')
parser.add_argument('--src-es', dest='src_es', help='the source Elasticsearch URL (including the port)', type=str, required=True)
parser.add_argument('--dst-es', dest='dst_es', help='the destination Elasticsearch URL (including the port)', type=str, required=True)
parser.add_argument('--src-idx', dest='src_idx', help='the source index', type=str, required=True)
parser.add_argument('--dst-idx', dest='dst_idx', help='the destination index', type=str, required=True)
parser.add_argument('--skip', dest="skip", nargs='+', help='do not copy aliases including the provided substrings!', type=str, required=False)
parser.add_argument('--preprend', dest="preprend", help='the string to preprend to alias names', type=str, required=False)
parser.add_argument('--append', dest="append", help='the string to append to alias names', type=str, required=False)
args = parser.parse_args()
try:
copy_aliases(args.src_es, args.dst_es, args.src_idx, args.dst_idx, args.skip, args.preprend, args.append)
except Exception as e:
print(e)
exit(1)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment