[FIX] uber-kludge to make newCorpus use normal workflow

install/notebook/gargantext_notebook.py

@@ -14,11 +14,16 @@ import django
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
 django.setup()
 
-from gargantext.models import ProjectNode, DocumentNode
+from gargantext.constants import QUERY_SIZE_N_MAX, get_resource, get_resource_by_name
+from gargantext.models import ProjectNode, DocumentNode, UserNode, User
 from gargantext.util.db import session, get_engine
 from collections import Counter
+import importlib
+from django.http import Http404
 
 
+class NotebookError(Exception):
+    pass
 
 
 def documents(corpus_id):
@@ -64,43 +69,68 @@ def myProject_fromUrl(url):
     return project
 
 
-def newCorpus(project, resourceName=11, name="Machine learning", query="LSTM"):
-    print("Corpus \"%s\" in project \"%s\" created" % (name, project.name))
+def newCorpus(project, resource=11, name="Machine learning", query="LSTM"):
+    source = get_resource(resource) if isinstance(resource, int) else \
+             get_resource_by_name(resource)
 
-    corpus = project.add_child(name="Corpus name", typename='CORPUS')
-    corpus.hyperdata["resources"] = [{"extracted" : "true", "type" : 11}]
-    corpus.hyperdata["statuses"]  = [{"action" : "notebook", "complete" : "true"}]
-    # [TODO] Add informations needed to get buttons on the Project view.
-    session.add(corpus)
-    session.commit()
+    moissonneur_name = get_moissonneur_name(source) if source else \
+                       resource.lower()
 
-    hal = HalCrawler()
-    max_result = hal.scan_results(query)
-    paging = 100
-    for page in range(0, max_result, paging):
-        print("%s documents downloaded / %s." % (str( paging * (page +1)), str(max_result) ))
-        docs = (hal._get(query, fromPage=page, count=paging)
-                     .get("response", {})
-                      .get("docs", [])
-               )
-
-        from gargantext.util.parsers.HAL import HalParser
-        # [TODO] fix boilerplate for docs here
-        new_docs = HalParser(docs)._parse(docs)
-
-        for doc in new_docs:
-            new_doc = (corpus.add_child( name      = doc["title"][:255]
-                                       , typename  = 'DOCUMENT')
-                      )
-            new_doc["hyperdata"] = doc
-            session.add(new_doc)
-            session.commit()
-
-    print("Extracting the ngrams")
-    parse_extract_indexhyperdata(corpus)
-
-    print("Corpus is ready to explore:")
-    print("http://imt.gargantext.org/projects/%s/corpora/%s/" % (project.id, corpus.id))
+    try:
+        moissonneur = get_moissonneur(moissonneur_name)
+    except ImportError:
+        raise NotebookError("Invalid resource identifier: %r" % resource)
 
-    return corpus
+    return run_moissonneur(moissonneur, project, name, query)
+
+
+def get_moissonneur_name(ident):
+    """ Return moissonneur module name from RESOURCETYPE or crawler name """
+
+    # Does it quacks like a RESOURCETYPE ?
+    if hasattr(ident, 'get'):
+        ident = ident.get('crawler')
+
+    # Extract name from crawler class name, otherwise assume ident is already
+    # a moissonneur name.
+    if isinstance(ident, str) and ident.endswith('Crawler'):
+        return ident[:-len('Crawler')].lower()
+
+
+def get_moissonneur(name):
+    """ Return moissonneur module from its name """
+    if not isinstance(name, str) or not name.islower():
+        raise NotebookError("Invalid moissonneur name: %r" % name)
+    return importlib.import_module('moissonneurs.%s' % name)
 
+
+def run_moissonneur(moissonneur, project, name, query):
+    """ Run moissonneur and return resulting corpus """
+
+    # XXX Uber-kludge with gory details. Spaghetti rulezzzzz!
+    class Dummy(object):
+        pass
+
+    request = Dummy()
+    request.method = 'POST'
+    request.path = 'nowhere'
+    request.META = {}
+    request.POST = {'string': name,
+                    'query': query,
+                    'N': QUERY_SIZE_N_MAX}
+    request.user = (session.query(UserNode).join(User)
+                           .filter(User.username=='gargantua').first())
+
+    try:
+        moissonneur.query(request)
+        corpus = moissonneur.save(request, project.id, return_corpus=True)
+
+    except (ValueError, Http404) as e:
+        raise e
+
+    # Sometimes strange things happens...
+    if corpus.name != name:
+        corpus.name = name
+        session.commit()
+
+    return corpus

moissonneurs/cern.py

@@ -30,7 +30,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -101,6 +101,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/hal.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -103,6 +103,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/isidore.py

@@ -29,7 +29,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -100,6 +100,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/istex.py

@@ -52,7 +52,7 @@ def query( request ):
 
 
 
-def save(request , project_id):
+def save(request , project_id, return_corpus=False):
     print("testISTEX:")
     print(request.method)
     alist = ["bar","foo"]
@@ -171,6 +171,9 @@ def save(request , project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/multivac.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -104,6 +104,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/pubmed.py

@@ -69,7 +69,7 @@ def query( request ):
     return JsonHttpResponse(data)
 
 
-def save( request , project_id ) :
+def save( request , project_id, return_corpus=False ) :
     # implicit global session
     # do we have a valid project id?
     try:
@@ -164,6 +164,10 @@ def save( request , project_id ) :
             session.rollback()
             # --------------------------------------------
         sleep(1)
+
+        if return_corpus:
+            return corpus
+
         return HttpResponseRedirect('/projects/' + str(project_id))
 
     data = alist