Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into samuel

admin/utils.py

+import os, sys
+import linecache
+
+from gargantext_web.settings import MEDIA_ROOT
+
+def ensure_dir(user):
+    '''
+    If user is new, folder does not exist yet, create it then
+    '''
+    dirpath = '%s/corpora/%s' % (MEDIA_ROOT, user.username)
+    if not os.path.exists(dirpath):
+        print("Creating folder %s" % dirpath)
+        os.makedirs(dirpath)
+
+
+def PrintException():
+    '''
+    This function has to be used in except part to print error with:
+    - the file
+    - the line number
+    - an explicit error araising
+    '''
+    exc_type, exc_obj, tb = sys.exc_info()
+    f = tb.tb_frame
+    lineno = tb.tb_lineno
+    filename = f.f_code.co_filename
+    linecache.checkcache(filename)
+    line = linecache.getline(filename, lineno, f.f_globals)
+    print('EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj))
+

gargantext_web/settings.py

@@ -75,7 +75,9 @@ ALLOWED_HOSTS = ['localhost',
                 'stable.gargantext.org', 
                 'dev.gargantext.org', 
                 'iscpif.gargantext.org', 
+                'gargantext.iscpif.fr', 
                 'mines.gargantext.org', 
+                'pasteur.gargantext.org', 
                 'beta.gargantext.org', 
                 'garg-dev.iscpif.fr', 
                 'garg-stable.iscpif.fr', 

gargantext_web/views_optimized.py

@@ -26,6 +26,9 @@ from parsing.corpustools import add_resource, parse_resources, extract_ngrams, c
 
 from gargantext_web.celery import apply_workflow
 
+from admin.utils import ensure_dir
+
+
 def project(request, project_id):
 
     # do we have a valid project id?
@@ -140,11 +143,7 @@ def project(request, project_id):
             session.commit()
             
             # If user is new, folder does not exist yet, create it then
-            dirpath = '%s/corpora/%s' % (MEDIA_ROOT, request.user.username)
-
-            if not os.path.exists(dirpath):
-                print("Creating folder %s" % dirpath)
-                os.makedirs(dirpath)
+            ensure_dir(request.user)
             
             # Save the uploaded file
             filepath = '%s/corpora/%s/%s' % (MEDIA_ROOT, request.user.username, thefile._name)

parsing/FileParsers/IsiFileParser.py

@@ -4,14 +4,21 @@ from .RisFileParser import RisFileParser
 
 class IsiFileParser(RisFileParser):
     
-    _parameters = {
-        b"ER":  {"type": "delimiter"},
-        b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-        b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-        b"DI":  {"type": "metadata", "key": "doi"},
-        b"PY":  {"type": "metadata", "key": "publication_year"},
-        b"PD":  {"type": "metadata", "key": "publication_month"},
-        b"LA":  {"type": "metadata", "key": "language_fullname"},
-        b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-        b"WC":  {"type": "metadata", "key": "fields"},
-    }
+    def __init__(self):
+        
+        super(RisFileParser, self).__init__()
+        
+        self._begin = 3
+        
+        self._parameters = {
+            b"ER":  {"type": "delimiter"},
+            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
+            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
+            b"DI":  {"type": "metadata", "key": "doi"},
+            b"PY":  {"type": "metadata", "key": "publication_year"},
+            b"PD":  {"type": "metadata", "key": "publication_month"},
+            b"LA":  {"type": "metadata", "key": "language_fullname"},
+            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "metadata", "key": "fields"},
+        }
+

parsing/FileParsers/JstorFileParser.py

@@ -2,17 +2,22 @@ from .RisFileParser import RisFileParser
 
 
 class JstorFileParser(RisFileParser):
+ 
+    def __init__(self):
+        
+        super(RisFileParser, self).__init__()
+        
+        self._begin = 3
 
-    _parameters = {
-        b"ER":  {"type": "delimiter"},
-        b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-        b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-        b"UR":  {"type": "metadata", "key": "doi"},
-        b"Y1":  {"type": "metadata", "key": "publication_year"},
-        b"PD":  {"type": "metadata", "key": "publication_month"},
-        b"LA":  {"type": "metadata", "key": "language_iso2"},
-        b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-        b"WC":  {"type": "metadata", "key": "fields"},
-    }
-
+        self._parameters = {
+            b"ER":  {"type": "delimiter"},
+            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
+            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
+            b"UR":  {"type": "metadata", "key": "doi"},
+            b"Y1":  {"type": "metadata", "key": "publication_year"},
+            b"PD":  {"type": "metadata", "key": "publication_month"},
+            b"LA":  {"type": "metadata", "key": "language_iso2"},
+            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "metadata", "key": "fields"},
+        }
 

parsing/FileParsers/RisFileParser.py

 from django.db import transaction
 from .FileParser import FileParser
 
+from ..Caches import LanguagesCache
 
 class RisFileParser(FileParser):
 
-    _parameters = {
-        b"ER":  {"type": "delimiter"},
-        b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-        b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-        b"UR":  {"type": "metadata", "key": "doi"},
-        b"PY":  {"type": "metadata", "key": "publication_year"},
-        b"PD":  {"type": "metadata", "key": "publication_month"},
-        b"LA":  {"type": "metadata", "key": "language_iso2"},
-        b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-        b"WC":  {"type": "metadata", "key": "fields"},
-    }
+    def __init__(self, language_cache=None):
+        
+        super(FileParser, self).__init__()
+        self._languages_cache = LanguagesCache() if language_cache is None else language_cache
+        
+        self._begin = 6
+        
+        self._parameters = {
+            b"ER":  {"type": "delimiter"},
+            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
+            b"ST":  {"type": "metadata", "key": "subtitle", "separator": " "},
+            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
+            b"UR":  {"type": "metadata", "key": "doi"},
+            b"PY":  {"type": "metadata", "key": "publication_year"},
+            b"PD":  {"type": "metadata", "key": "publication_month"},
+            b"LA":  {"type": "metadata", "key": "language_iso2"},
+            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "metadata", "key": "fields"},
+        }
+        
 
     def _parse(self, file):
         metadata = {}
@@ -42,9 +52,10 @@ class RisFileParser(FileParser):
                     last_key = parameter_key
                     last_values = []
                 try:
-                    last_values.append(line[3:-1].decode())
+                    last_values.append(line[self._begin:-1].decode())
                 except Exception as error:
                     print(error)
         # if a metadata object is left in memory, yield it as well
         if metadata:
+            #print(metadata['title'])
             yield metadata

parsing/parsers_config.py

@@ -4,7 +4,7 @@ parsers = {
         'Pubmed (xml format)'               : PubmedFileParser,
         'Web of Science (ISI format)'       : IsiFileParser,
         'Scopus (RIS format)'               : RisFileParser,
-        'Zotero (RIS format)'               : RisFileParser,
+        'Zotero (RIS format)'               : JstorFileParser,
         'Jstor (RIS format)'                : JstorFileParser,
         #'Europress'                        : EuropressFileParser,
         'Europress (French)'                : EuropressFileParser,

scrappers/scrap_pubmed/views.py

@@ -37,6 +37,8 @@ from parsing.corpustools import add_resource, parse_resources, extract_ngrams, c
 from gargantext_web.celery import apply_workflow
 from time import sleep
 
+from admin.utils import ensure_dir
+
 def getGlobalStats(request ):
 	print(request.method)
 	alist = ["bar","foo"]
@@ -141,7 +143,9 @@ def doTheQuery(request , project_id):
 		# """
 
 
+		ensure_dir(request.user)
 		tasks = MedlineFetcher()
+
 		for i in range(8):
 			t = threading.Thread(target=tasks.worker2) #thing to do
 			t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.