Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
web-et-numerique
web-et-numerique-internet
data.grandlyon.com
web-portal
components
indexers
editorial-content
Commits
1d9f8fea
Commit
1d9f8fea
authored
Jun 15, 2019
by
Alessandro CERIONI
Browse files
Cleaning indices before (re)indexing, in order to make it happen for real!
parent
da608322
Pipeline
#3233
passed with stage
in 1 minute and 31 seconds
Changes
3
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
lib/index_cleaner.py
View file @
1d9f8fea
from
elasticsearch
import
Elasticsearch
from
elasticsearch
import
Elasticsearch
,
NotFoundError
def
index_cleaner
(
cfg
,
content_type
,
content_id
=
None
):
...
...
@@ -17,13 +17,16 @@ def index_cleaner( cfg, content_type, content_id=None ):
try
:
res
=
source_es
.
indices
.
delete
(
source_index
)
except
NotFoundError
:
pass
except
Exception
as
e
:
print
(
e
)
successful
=
False
try
:
rep
=
destin_es
.
indices
.
delete
(
destin_index
)
#print(rep)
except
NotFoundError
:
pass
except
Exception
as
e
:
print
(
e
)
successful
=
False
...
...
@@ -36,16 +39,18 @@ def index_cleaner( cfg, content_type, content_id=None ):
try
:
res
=
source_es
.
delete
(
index
=
source_index
,
doc_type
=
'_doc'
,
id
=
content_id
)
except
NotFoundError
:
pass
except
Exception
as
e
:
print
(
e
)
successful
=
False
pass
try
:
res
=
source_es
.
delete
(
index
=
destin_index
,
doc_type
=
'_doc'
,
id
=
content_id
)
res
=
destin_es
.
delete
(
index
=
destin_index
,
doc_type
=
'_doc'
,
id
=
content_id
)
except
NotFoundError
:
pass
except
Exception
as
e
:
print
(
e
)
successful
=
False
pass
return
successful
lib/reindexer.py
View file @
1d9f8fea
...
...
@@ -5,7 +5,7 @@ from pprint import pprint
from
.elastic_mapping_template_pages
import
template
as
template_pages
from
.elastic_mapping_template_posts
import
template
as
template_posts
def
reindexer
(
cfg
,
content_type
):
def
reindexer
(
cfg
,
content_type
,
content_id
=
None
):
template
=
eval
(
'template_%s'
%
content_type
)
...
...
@@ -63,6 +63,11 @@ def reindexer(cfg, content_type):
}
}
if
content_id
!=
None
:
body
[
'source'
][
'query'
]
=
{
"term"
:
{
"_id"
:
content_id
}}
print
(
body
)
# waiting for source index to be stable
count1
=
0
...
...
@@ -79,6 +84,8 @@ def reindexer(cfg, content_type):
#if rep['failures'] == []:
if
'task'
in
rep
:
return
cfg
[
'reindexer'
][
'destination_url'
]
+
'/_tasks/'
+
rep
[
'task'
]
task_url
=
cfg
[
'reindexer'
][
'destination_url'
]
+
'/_tasks/'
+
rep
[
'task'
]
print
(
"Task URL: %s"
%
task_url
)
return
task_url
else
:
return
False
main.py
View file @
1d9f8fea
...
...
@@ -55,9 +55,8 @@ def add_content( cfg, content_type, data ):
# N.B. : pages -> page; posts -> post, that's why we remove the last letter from content_type
content_id
=
data
[
content_type
[
0
:
len
(
content_type
)
-
1
]
][
'current'
][
'id'
]
print
(
'Here'
,
content_id
)
print
(
'Getting content with id
=
%s...'
%
content_id
)
print
(
'Getting content with id
=
%s...'
%
content_id
)
try
:
content
=
get_content
(
cfg
,
content_type
,
content_id
)
...
...
@@ -66,14 +65,19 @@ def add_content( cfg, content_type, data ):
return
False
processed_content
=
eval
(
'process_%s'
%
content_type
)([
content
])
successful
=
index_docs
(
cfg
,
content_type
,
processed_content
)
successful
=
index_cleaner
(
cfg
,
content_type
,
content_id
)
if
not
successful
:
raise
Exception
(
'Something went wrong. Exiting...'
)
exit
(
1
)
successful
=
index_docs
(
cfg
,
content_type
,
processed_content
)
if
not
successful
:
raise
Exception
(
'Something went wrong. Exiting...'
)
exit
(
1
)
print
(
'Reindexing...'
)
task_url
=
reindexer
(
cfg
,
content_type
)
task_url
=
reindexer
(
cfg
,
content_type
,
content_id
)
#print(task_url)
if
not
successful
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment