Merge branch 'development' into 'master'

merge dev into master

See merge request !23

lib/fix_links.py

@@ -151,7 +151,7 @@ def fix_links( links, credentials=None ):
     the_fixed_ws_url = None
     for k, link in enumerate(fixed_links):
 
-        if link['protocol'] == 'JSON' and 'download.data.grandlyon.com/ws' in link['url']:
+        if link['protocol'] == 'JSON' and ('download.data.grandlyon.com/ws' in link['url'] or 'download.recette.data.grandlyon.com/ws' in link['url']):
             fixed_links[k]['protocol'] = 'WS'
 
             tmp = links[k]['url']

main.py

@@ -77,8 +77,10 @@ def setup_indices(cfg):
         logging.error(e)
 
     try:
+        logging.info('Create index %s'% cfg['reindexer']['template_name'])
         rep = destin_es.indices.create(cfg['reindexer']['template_name'], template)
-        logging.debug(rep)
+        # ES7 rep = destin_es.indices.put_template(cfg['reindexer']['template_name'], template)
+        logging.info(rep)
     except Exception as e:
         logging.warning(e)
 

workers/doc_indexer.py

@@ -24,7 +24,7 @@ def tag_doc( the_doc ):
     #   tag_dict[tag] = False
 
     # isOpen?
-    if 'license' in the_doc['metadata-fr'].keys() and not any( [x in the_doc['metadata-fr']['license'] for x in ["Licence de réutilisation des données d'intérêt général","Accord de réutilisation de données en accès privé et exclusif"] ] ):
+    if 'license' in the_doc['metadata-fr'].keys() and not any( [x in the_doc['metadata-fr']['license'] for x in ["Licence Mobilités","Licence de réutilisation des données d'intérêt général","Accord de réutilisation de données en accès privé et exclusif"] ] ):
         tag_dict['isOpenAccess'] = True
     else:
         tag_dict['isOpenAccess'] = False

workers/metadata_processor.py

@@ -321,7 +321,8 @@ def callback( channel, method, properties, body ):
                 else:
                     logging.warning("No information about the needed fields and types found (%s) in the field catalog: generating a new catalog...", lookup_key)
                     field_type_detector(cfg)
-                    channel.basic_nack(delivery_tag = method.delivery_tag, requeue=1)
+                    logging.error("ERREUR ELECTION DES TYPES POUR:%s",out_record['metadata-fr']['geonet:info']['uuid'])
+                    channel.basic_nack(delivery_tag = method.delivery_tag, requeue=0)
                     return
                 # ---------------------------------------------------------------------------------------------------------------------
 

workers/reindexer.py

@@ -79,7 +79,7 @@ def on_msg_callback(channel, method, properties, body):
     the_query['query']['term'] = {'uuid.keyword': '{0}'.format(uuid)}
 
     es_source.indices.refresh(index=cfg['reindexer']['source_index'])
-    count_es = es_source.count(cfg['reindexer']['source_index'], body=the_query).get('count')
+    count_es = es_source.count(index=cfg['reindexer']['source_index'], doc_type='_doc', body=the_query).get('count')
     # logging.debug("%i document(s) found in the source index with uuid = %s" % (count1, uuid))
 
     # if uuid.endswith('.full'):

workers/sample_generator.py

@@ -40,7 +40,7 @@ def callback(channel, method, properties, body):
     the_query['query'] = dict()
     the_query['query']['term'] = {'uuid.keyword': uuid}
 
-    res = source_es.search(cfg['reindexer']['source_index'], '_doc', the_query)
+    res = source_es.search(index=cfg['reindexer']['source_index'], doc_type='_doc', body=the_query)
 
     docs_to_index = [ doc['_source'] for doc in res['hits']['hits'] ]
 
@@ -61,7 +61,7 @@ def callback(channel, method, properties, body):
 
     logging.info("Deleting already existing samples for dataset with slug = %s" % docs_to_index[0]['slug'])
     try:
-        res = destin_es.delete_by_query(cfg['reindexer']['destination_index'], doc_type='_doc', body=the_query, conflicts='proceed', refresh=True, wait_for_completion=False)
+        res = destin_es.delete_by_query(index=cfg['reindexer']['destination_index'], doc_type='_doc', body=the_query, conflicts='proceed', refresh=True, wait_for_completion=False)
         logging.debug(res)
         task_id = res['task']
         # wait until ES is done