[FIX] ISIDORE/SPARQLE fix duplicates in scan and corpus building. Improving query with date desc.

gargantext/util/crawlers/sparql/bool2sparql.py

@@ -49,7 +49,7 @@ def isidore(query, count=False, offset=None, limit=None):
         for r in results:
             doc        = dict()
             doc_values = dict()
-            doc["url"], doc["id"], doc["title"], doc["date"], doc["abstract"], doc["source"] = r
+            doc["url"], doc["title"], doc["date"], doc["abstract"], doc["source"] = r
             
             for k in doc.keys():
                 doc_values[k] = doc[k].value

gargantext/util/parsers/ISIDORE.py

@@ -31,6 +31,8 @@ class IsidoreParser(Parser):
                          , "source"   : "source"
                          }
         
+        uniq_id = set()
+
         for doc in json_docs:
 
             hyperdata = {}
@@ -38,6 +40,10 @@ class IsidoreParser(Parser):
             for key, path in hyperdata_path.items():
                     hyperdata[key] = doc.get(path, "")
             
+            if hyperdata["url"] not in uniq_id:
+                # Removing the duplicates implicitly
+                uniq_id.add(hyperdata["url"])
+                
                 # Source is the Journal Name 
                 hyperdata["source"] = doc.get("journal", "ISIDORE Database")