Improved 'protocol' detection for HTML ressources.

utils/fix_links.py

@@ -4,7 +4,6 @@ import json
 import requests
 from .my_logging import logging
 
-
 def translate_content_type( content_type ):
 
     # TODO: complete the following list!
@@ -39,13 +38,15 @@ def fix_links( links ):
 
         # FIX links in which the declared protocol is as bizarre as "WWW:LINK-1.0-http--link"
         # The KML protocol needs also to be fixed.
-        if 'protocol' in link.keys() and any( [x in link['protocol'] for x in ['WWW', 'kml', 'html'] ] ):
+        if 'protocol' in link.keys() and any( [x in link['protocol'] for x in ['WWW', 'kml', 'html', 'null'] ] ):
             try:
                 # let's try getting the information from the Web Server...
                 resp = requests.head( link['url'], allow_redirects=True )
                 # the presence of the content-length assures that the Web Server knows what it is talking about,
-                # that is why we include the following line in this try-except block
-                fixed_links[k]['content-length'] = resp.headers['Content-Length']
+                # that is why we include the following line in this try-except block, except for HTML pages, in which case
+                # webservers do not send back the content-length
+                if 'text/html' not in resp.headers['Content-Type']:
+                    fixed_links[k]['content-length'] = resp.headers['Content-Length']
                 # N.B.: we use the attribute 'protocol' in the output dict 'cause that's an attribute that GeoNetwork lets the user fill,
                 # as opposed to the 'content-type' attribute which is not shown by GeoNetwork's GUI, despite being exposed by the 'q' API.
                 # Ideally, in the future the 'protocol' attribute will be filled the right way by the back-office team.
@@ -63,6 +64,13 @@ def fix_links( links ):
 
     #pprint(fixed_links)
 
+    # FIX TIF -> TIFF
+    for k, link in enumerate(fixed_links):
+
+        if link['protocol'] == 'TIF':
+            fixed_links[k]['protocol'] = 'TIFF'
+
+
     # FIX WS / JSON
     the_fixed_ws_url = None
     for k, link in enumerate(fixed_links):
@@ -151,6 +159,14 @@ if __name__ == '__main__':
                     'unknown': "1",
                     'protocol': "WWW-LINK"
                 },
+                {
+                    'url': "https://download.data.grandlyon.com/files/grandlyon/LicenceOuverte.pdf",
+                    'name': "Licence ouverte",
+                    'content-type': "text/html",
+                    'description': "Licence ouverte",
+                    'unknown': "1",
+                    'protocol': "null"
+                },
                 {
                     'name': "Historique des disponibilités des stations Vélo'V(OGC:SOS)",
                     'description': "Historique des disponibilités des stations Vélo'V(OGC:SOS)",
@@ -166,6 +182,14 @@ if __name__ == '__main__':
                     'protocol': "text/html",
                     'content-type': "text/html",
                     'unknown': "1"
+                },
+                {
+                    "name": "alerte-pollens",
+                    "description": "alerte-pollens",
+                    "url": "http://www.pollens.fr/alerte-pollens",
+                    "protocol": "WWW-LINK",
+                    "content-type": "WWW:LINK",
+                    "unknown": "0"
                 }
         ]