reindexer.py 2.01 KB
Newer Older
Alessandro CERIONI's avatar
Alessandro CERIONI committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
from elasticsearch import Elasticsearch
import time
from pprint import pprint

from .es_template import template

def reindexer(cfg):

  source_es = Elasticsearch( [cfg['reindexer']['source_url']] )
  destination_es = Elasticsearch( [cfg['reindexer']['destination_url']] )
  template['index_patterns'] = [ cfg['reindexer']['destination_index'] ]


  try:
    rep = destination_es.indices.delete_template(cfg['reindexer']['template'])
    #print(rep)
  except:
    pass

  rep = destination_es.indices.put_template(cfg['reindexer']['template'], template)
  # rep = es.indices.get_template("template_1")
  # print(rep)


  # t1 = time.time()
26
27
28
29
30
  try:
      rep = destination_es.indices.delete(cfg['reindexer']['destination_index'])
      #print(rep)
  except:
      pass
Alessandro CERIONI's avatar
Alessandro CERIONI committed
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

  try:
    rep = destination_es.indices.create(cfg['reindexer']['destination_index'])
    #print(rep)
  except:
    pass

  body = {
          # "conflicts": "proceed",
          # "size": 1000,
          "source": {
            "remote": {
              "host": cfg['reindexer']['source_url']
            },
            "index": cfg['reindexer']['source_index'],
            "type": "_doc",
            "size": 100
            # "slice" : {
            #     "id": i,
            #     "max": no_slices
            # }
          },
          "dest": {
            "index": cfg['reindexer']['destination_index'],
            "type": "_doc"
            }
          }


  # waiting for source index to be stable
  count1 = 0
  count2 = 1

  while count1 != count2:
    source_es.indices.refresh(index=cfg['reindexer']['source_index'])
    count1 = source_es.count(cfg['reindexer']['source_index']).get('count')
Alessandro CERIONI's avatar
Alessandro CERIONI committed
67
68
    time.sleep(1)
    source_es.indices.refresh(index=cfg['reindexer']['source_index'])
Alessandro CERIONI's avatar
Alessandro CERIONI committed
69
70
71
72
73
74
75
76
77
    count2 = source_es.count(cfg['reindexer']['source_index']).get('count')

  rep = destination_es.reindex(body, wait_for_completion=False)

  #if rep['failures'] == []:
  if 'task' in rep:
      return cfg['reindexer']['destination_url'] + '/_tasks/' + rep['task']
  else:
      return False