diff --git a/lib/fix_links.py b/lib/fix_links.py index 68d1ee82fb70d765584f069eb55afa36b769a6b9..50db86faed6c34b0b40e086f5388f81e428fd602 100644 --- a/lib/fix_links.py +++ b/lib/fix_links.py @@ -151,7 +151,7 @@ def fix_links( links, credentials=None ): the_fixed_ws_url = None for k, link in enumerate(fixed_links): - if link['protocol'] == 'JSON' and 'download.data.grandlyon.com/ws' in link['url']: + if link['protocol'] == 'JSON' and ('download.data.grandlyon.com/ws' in link['url'] or 'download.recette.data.grandlyon.com/ws' in link['url']): fixed_links[k]['protocol'] = 'WS' tmp = links[k]['url'] diff --git a/main.py b/main.py index 28bfd9af2ddad68896e13ce7fb1b9152b3eb3db2..48f98bcd4e5711e09cb65cd53c5a23c2eed649c0 100644 --- a/main.py +++ b/main.py @@ -77,8 +77,10 @@ def setup_indices(cfg): logging.error(e) try: + logging.info('Create index %s'% cfg['reindexer']['template_name']) rep = destin_es.indices.create(cfg['reindexer']['template_name'], template) - logging.debug(rep) + # ES7 rep = destin_es.indices.put_template(cfg['reindexer']['template_name'], template) + logging.info(rep) except Exception as e: logging.warning(e) diff --git a/workers/doc_indexer.py b/workers/doc_indexer.py index 4e806671e406b537466ab45f2a55e0106d1281af..2a6494876869e9a635f236431c17918f028d094d 100644 --- a/workers/doc_indexer.py +++ b/workers/doc_indexer.py @@ -24,7 +24,7 @@ def tag_doc( the_doc ): # tag_dict[tag] = False # isOpen? - if 'license' in the_doc['metadata-fr'].keys() and not any( [x in the_doc['metadata-fr']['license'] for x in ["Licence de réutilisation des données d'intérêt général","Accord de réutilisation de données en accès privé et exclusif"] ] ): + if 'license' in the_doc['metadata-fr'].keys() and not any( [x in the_doc['metadata-fr']['license'] for x in ["Licence Mobilités","Licence de réutilisation des données d'intérêt général","Accord de réutilisation de données en accès privé et exclusif"] ] ): tag_dict['isOpenAccess'] = True else: tag_dict['isOpenAccess'] = False diff --git a/workers/metadata_processor.py b/workers/metadata_processor.py index 0b8b0cb1263e85ad93a2ba1d78430a84fc97b237..f5fd321fe322cb2cc7a41820ed15ab6b1086f002 100644 --- a/workers/metadata_processor.py +++ b/workers/metadata_processor.py @@ -321,7 +321,8 @@ def callback( channel, method, properties, body ): else: logging.warning("No information about the needed fields and types found (%s) in the field catalog: generating a new catalog...", lookup_key) field_type_detector(cfg) - channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1) + logging.error("ERREUR ELECTION DES TYPES POUR:%s",out_record['metadata-fr']['geonet:info']['uuid']) + channel.basic_nack(delivery_tag = method.delivery_tag, requeue=0) return # --------------------------------------------------------------------------------------------------------------------- diff --git a/workers/reindexer.py b/workers/reindexer.py index 0bc029909e2358cd696039ca0e43b6cc0da28aef..eb7aec5f899fb8495ff524b69c0533fd90ef19ec 100644 --- a/workers/reindexer.py +++ b/workers/reindexer.py @@ -79,7 +79,7 @@ def on_msg_callback(channel, method, properties, body): the_query['query']['term'] = {'uuid.keyword': '{0}'.format(uuid)} es_source.indices.refresh(index=cfg['reindexer']['source_index']) - count_es = es_source.count(cfg['reindexer']['source_index'], body=the_query).get('count') + count_es = es_source.count(index=cfg['reindexer']['source_index'], doc_type='_doc', body=the_query).get('count') # logging.debug("%i document(s) found in the source index with uuid = %s" % (count1, uuid)) # if uuid.endswith('.full'): diff --git a/workers/sample_generator.py b/workers/sample_generator.py index 82dd4e90691d64e5a761e3050f8e90bde084ff4c..88f774191f49b0b5cdcdc7bb9e99417c6dc85143 100644 --- a/workers/sample_generator.py +++ b/workers/sample_generator.py @@ -40,7 +40,7 @@ def callback(channel, method, properties, body): the_query['query'] = dict() the_query['query']['term'] = {'uuid.keyword': uuid} - res = source_es.search(cfg['reindexer']['source_index'], '_doc', the_query) + res = source_es.search(index=cfg['reindexer']['source_index'], doc_type='_doc', body=the_query) docs_to_index = [ doc['_source'] for doc in res['hits']['hits'] ] @@ -61,7 +61,7 @@ def callback(channel, method, properties, body): logging.info("Deleting already existing samples for dataset with slug = %s" % docs_to_index[0]['slug']) try: - res = destin_es.delete_by_query(cfg['reindexer']['destination_index'], doc_type='_doc', body=the_query, conflicts='proceed', refresh=True, wait_for_completion=False) + res = destin_es.delete_by_query(index=cfg['reindexer']['destination_index'], doc_type='_doc', body=the_query, conflicts='proceed', refresh=True, wait_for_completion=False) logging.debug(res) task_id = res['task'] # wait until ES is done