[FEAT] need to fix the crawler.

gargantext/util/crawlers/HAL.py

@@ -113,7 +113,7 @@ class HalCrawler(Crawler):
             msg = "Invalid sample size N = %i (max = %i)" % ( self.query_max
                                                             , QUERY_SIZE_N_MAX
                                                             )
-            print("ERROR (scrap: Multivac d/l ): " , msg)
+            print("ERROR (scrap: HAL d/l ): " , msg)
             self.query_max = QUERY_SIZE_N_MAX
         
         #for page in range(1, trunc(self.query_max / 100) + 2):

gargantext/util/parsers/HAL.py

@@ -11,17 +11,8 @@ from datetime import datetime
 import json
 
 class HalParser(Parser):
-
-    def parse(self, filebuf):
-        '''
-        parse :: FileBuff -> [Hyperdata]
-        '''
-        contents = filebuf.read().decode("UTF-8")
-        data = json.loads(contents)
+    def _parse(self, json_docs):
         
-        filebuf.close()
-        
-        json_docs = data
         hyperdata_list = []
         
         hyperdata_path = { "id"       : "isbn_s"
@@ -73,3 +64,13 @@ class HalParser(Parser):
                 hyperdata_list.append(hyperdata)
         
         return hyperdata_list
+
+    def parse(self, filebuf):
+        '''
+        parse :: FileBuff -> [Hyperdata]
+        '''
+        contents = filebuf.read().decode("UTF-8")
+        data = json.loads(contents)
+
+        return self._parse(data)
+

install/notebook.run

@@ -16,7 +16,7 @@ sudo docker run \
         --env POSTGRES_HOST=localhost \
         -v /srv/gargantext:/srv/gargantext \
         -it garg-notebook:latest \
-        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
+        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /home/notebooks && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
 #        #&& jupyter nbextension enable --py widgetsnbextension --sys-prefix 
         #/bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser --notebook-dir=/home/notebooks/'"
 

install/notebook/gargantext_notebook.py

@@ -22,7 +22,7 @@ application = get_wsgi_application()
 from gargantext.util.toolchain.main import parse_extract_indexhyperdata
 from gargantext.util.db import *
 from gargantext.models import Node
-
+from gargantext.util.toolchain.main import parse_extract_indexhyperdata
 from nltk.tokenize import wordpunct_tokenize
 
 from gargantext.models import *
@@ -56,9 +56,7 @@ def chart(docs, field):
     frame1 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'], index=frame0.Date)
     return frame1
 
-
 from gargantext.util.crawlers.HAL import HalCrawler
-
 def scan_hal(request):
     hal = HalCrawler()
     return hal.scan_results(request)
@@ -73,3 +71,53 @@ def scan_gargantext(corpus_id, lang, request):
     return [i for i in connection.execute(query)][0][0]
     connection.close()
 
+
+def myProject_fromUrl(url):
+    """
+    myProject :: String -> Project
+    """
+    project_id = url.split("/")[4]
+    project = session.query(Node).filter(Node.id == project_id).first()
+    return project
+
+
+def newCorpus(project, resourceName=11, name="Machine learning", query="LSTM"):
+    print("Corpus \"%s\" in project \"%s\" created" % (name, project.name))
+    
+    corpus = project.add_child(name="Corpus name", typename='CORPUS')
+    corpus.hyperdata["resources"] = [{"extracted" : "true", "type" : 11}]
+    corpus.hyperdata["statuses"]  = [{"action" : "notebook", "complete" : "true"}]
+    # [TODO] Add informations needed to get buttons on the Project view.
+    session.add(corpus)
+    session.commit()
+    
+    hal = HalCrawler()
+    max_result = hal.scan_results(query)
+    paging = 100 
+    for page in range(0, max_result, paging):
+        print("%s documents downloaded / %s." % (str( paging * (page +1)), str(max_result) ))
+        docs = (hal._get(query, fromPage=page, count=paging)
+                     .get("response", {})
+                      .get("docs", [])
+               )
+            
+        from gargantext.util.parsers.HAL import HalParser
+        # [TODO] fix boilerplate for docs here
+        new_docs = HalParser(docs)._parse(docs)
+        
+        for doc in new_docs:
+            new_doc = (corpus.add_child( name      = doc["title"][:255]
+                                       , typename  = 'DOCUMENT')
+                      )
+            new_doc["hyperdata"] = doc
+            session.add(new_doc)
+            session.commit()
+    
+    print("Extracting the ngrams")
+    parse_extract_indexhyperdata(corpus)
+    
+    print("Corpus is ready to explore:")
+    print("http://imt.gargantext.org/projects/%s/corpora/%s/" % (project.id, corpus.id))
+    
+    return corpus
+