Merge branch 'unstable-notebook' into testing-notebook

docs/contribution.md

@@ -2,7 +2,7 @@
 
 ## Community
 * [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
+* IRC Chat: (OFTC/FreeNode) #gargantext
 
 ##Tools
 * gogs

gargantext/models/base.py

 from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint, Index
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, validates
 from sqlalchemy.types import TypeDecorator, \
                              Integer, Float, Boolean, DateTime, String, Text
 from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION as Double
@@ -7,6 +7,7 @@ from sqlalchemy.ext.mutable import MutableDict, MutableList
 from sqlalchemy.ext.declarative import declarative_base
 
 __all__ = ["Column", "ForeignKey", "UniqueConstraint", "relationship",
+           "validates", "ValidatorMixin",
            "Integer", "Float", "Boolean", "DateTime", "String", "Text",
            "TypeDecorator",
            "JSONB", "Double",
@@ -18,6 +19,25 @@ __all__ = ["Column", "ForeignKey", "UniqueConstraint", "relationship",
 # all tables handled by Alembic migration scripts.
 Base = declarative_base()
 
+
 # To be used by tables already handled by Django ORM, such as User model. We
 # separate them in order to keep those out of Alembic sight.
 DjangoBase = declarative_base()
+
+
+class ValidatorMixin(object):
+    def enforce_length(self, key, value):
+        """Truncate a string according to its column length
+
+        Usage example:
+
+        .. code-block:: python
+
+            @validates('some_column')
+            def validate_some_column(self, key, value):
+                self.enforce_length(key, value)
+        """
+        max_len = getattr(self.__class__, key).prop.columns[0].type.length
+        if value and len(value) > max_len:
+            return value[:max_len]
+        return value

gargantext/models/nodes.py

@@ -9,7 +9,7 @@ from datetime import datetime
 
 from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, \
                   Integer, Float, String, DateTime, JSONB, \
-                  MutableList, MutableDict
+                  MutableList, MutableDict, validates, ValidatorMixin
 from .users import User
 
 __all__ = ['Node', 'NodeNode', 'CorpusNode']
@@ -26,7 +26,7 @@ class NodeType(TypeDecorator):
         return NODETYPES[typeindex]
 
 
-class Node(Base):
+class Node(ValidatorMixin, Base):
     """This model can fit many purposes:
 
     myFirstCorpus = session.query(CorpusNode).first()
@@ -112,6 +112,10 @@ class Node(Base):
                'user_id={0.user_id}, parent_id={0.parent_id}, ' \
                'name={0.name!r}, date={0.date})>'.format(self)
 
+    @validates('name')
+    def validate_name(self, key, value):
+        return self.enforce_length(key, value)
+
     @property
     def ngrams(self):
         """Pseudo-attribute allowing to retrieve a node's ngrams.

gargantext/util/crawlers/HAL.py

@@ -113,7 +113,7 @@ class HalCrawler(Crawler):
             msg = "Invalid sample size N = %i (max = %i)" % ( self.query_max
                                                             , QUERY_SIZE_N_MAX
                                                             )
-            print("ERROR (scrap: Multivac d/l ): " , msg)
+            print("ERROR (scrap: HAL d/l ): " , msg)
             self.query_max = QUERY_SIZE_N_MAX
         
         #for page in range(1, trunc(self.query_max / 100) + 2):

gargantext/util/http.py

@@ -73,7 +73,8 @@ from rest_framework.views import APIView
 from gargantext.util.json import json_encoder
 def JsonHttpResponse(data, status=200):
     return HttpResponse(
-        content      = json_encoder.encode(data),
+        content      = data.encode('utf-8') if isinstance(data, str) else \
+                       json_encoder.encode(data),
         content_type = 'application/json; charset=utf-8',
         status       = status
     )

gargantext/util/parsers/HAL.py

@@ -11,17 +11,8 @@ from datetime import datetime
 import json
 
 class HalParser(Parser):
+    def _parse(self, json_docs):
         
-    def parse(self, filebuf):
-        '''
-        parse :: FileBuff -> [Hyperdata]
-        '''
-        contents = filebuf.read().decode("UTF-8")
-        data = json.loads(contents)
-        
-        filebuf.close()
-        
-        json_docs = data
         hyperdata_list = []
         
         hyperdata_path = { "id"       : "isbn_s"
@@ -73,3 +64,13 @@ class HalParser(Parser):
                 hyperdata_list.append(hyperdata)
         
         return hyperdata_list
+
+    def parse(self, filebuf):
+        '''
+        parse :: FileBuff -> [Hyperdata]
+        '''
+        contents = filebuf.read().decode("UTF-8")
+        data = json.loads(contents)
+
+        return self._parse(data)
+

install/notebook.run

@@ -16,7 +16,7 @@ sudo docker run \
         --env POSTGRES_HOST=localhost \
         -v /srv/gargantext:/srv/gargantext \
         -it garg-notebook:latest \
-        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
+        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /home/notebooks && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
 #        #&& jupyter nbextension enable --py widgetsnbextension --sys-prefix 
         #/bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser --notebook-dir=/home/notebooks/'"
 

install/notebook/Dockerfile

@@ -78,32 +78,8 @@ RUN . /env_3-5/bin/activate && pip3 install -r requirements.txt
 
 #RUN ./psql_configure.sh
 #RUN ./django_configure.sh
-
 RUN chown notebooks:notebooks -R /env_3-5
 
-########################################################################
-### Notebook IHaskell and IPYTHON ENVIRONNEMENT
-########################################################################
-
-#RUN apt-get update && apt-get install -y    \
-#        libtinfo-dev                        \
-#        libzmq3-dev                         \
-#        libcairo2-dev                       \
-#        libpango1.0-dev                     \
-#        libmagic-dev                        \
-#        libblas-dev                         \
-#        liblapack-dev
-#RUN curl -sSL https://get.haskellstack.org/ | sh 
-#RUN stack setup
-#RUN git clone https://github.com/gibiansky/IHaskell
-#RUN . /env_3-5/bin/activate     \
-#    && cd IHaskell                    \
-#    && stack install gtk2hs-buildtools \
-#    && stack install --fast             \
-#    && /root/.local/bin/ihaskell install --stack
-#
-
-#
 ########################################################################
 ### POSTGRESQL DATA (as ROOT)
 ########################################################################
@@ -115,3 +91,32 @@ RUN chown notebooks:notebooks -R /env_3-5
 EXPOSE 5432 8899
 
 VOLUME ["/srv/","/home/notebooks/"]
+
+
+########################################################################
+### Notebook IHaskell and IPYTHON ENVIRONNEMENT
+########################################################################
+
+RUN apt-get update && apt-get install -y \
+        libtinfo-dev                      \
+        libzmq3-dev                        \
+        libcairo2-dev                       \
+        libpango1.0-dev                      \
+        libmagic-dev                          \
+        libblas-dev                            \
+        liblapack-dev
+
+USER notebooks
+
+RUN cd  /home/notebooks                              \
+    &&  curl -sSL https://get.haskellstack.org/ | sh  \
+    &&  stack setup                                    \
+    &&  git clone https://github.com/gibiansky/IHaskell \
+    &&  . /env_3-5/bin/activate                          \
+    &&  cd IHaskell                                       \
+    &&  stack install gtk2hs-buildtools                    \
+    &&  stack install --fast                                \
+    &&  /root/.local/bin/ihaskell install --stack
+
+
+

install/notebook/gargantext_notebook.py

+#!/usr/bin/env python
 """
    Gargantext Software Copyright (c) 2016-2017 CNRS ISC-PIF -
 http://iscpif.fr
@@ -6,45 +7,29 @@ http://gitlab.iscpif.fr/humanities/gargantext/blob/stable/LICENSE )
     - In France : a CECILL variant affero compliant
     - GNU aGPLV3 for all other countries
 """
-#!/usr/bin/env python
-import sys
-import os
 
+import os
+import django
 
-# Django settings
-dirname = os.path.dirname(os.path.realpath(__file__))
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
-
-# initialize Django application
-from django.core.wsgi import get_wsgi_application
-application = get_wsgi_application()
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
 
-from gargantext.util.toolchain.main import parse_extract_indexhyperdata
-from gargantext.util.db import *
-from gargantext.models import Node
+from gargantext.constants import QUERY_SIZE_N_MAX, get_resource, get_resource_by_name
+from gargantext.models import ProjectNode, DocumentNode, UserNode, User
+from gargantext.util.db import session, get_engine
+from collections import Counter
+import importlib
+from django.http import Http404
 
-from nltk.tokenize import wordpunct_tokenize
 
-from gargantext.models import *
-from nltk.tokenize import word_tokenize
-import nltk as nltk
-from statistics import mean
-from math import log
-from collections import defaultdict
-import matplotlib.pyplot as plt
-import numpy as np
-import datetime
+class NotebookError(Exception):
+    pass
 
-from collections import Counter
-from langdetect import detect as detect_lang
 
 def documents(corpus_id):
-    return (session.query(Node).filter( Node.parent_id==corpus_id
-                                  , Node.typename=="DOCUMENT"
-                                  )
-        # .order_by(Node.hyperdata['publication_date'])
-        .all()
-        )
+    return (session.query(DocumentNode).filter_by(parent_id=corpus_id)
+                  #.order_by(Node.hyperdata['publication_date'])
+                   .all())
 
 
 #import seaborn as sns
@@ -63,6 +48,7 @@ def scan_hal(request):
     hal = HalCrawler()
     return hal.scan_results(request)
 
+
 def scan_gargantext(corpus_id, lang, request):
     connection = get_engine().connect()
     # TODO add some sugar the request (ideally request should be the same for hal and garg)
@@ -73,3 +59,123 @@ def scan_gargantext(corpus_id, lang, request):
     return [i for i in connection.execute(query)][0][0]
     connection.close()
 
+
+def myProject_fromUrl(url):
+    """
+    myProject :: String -> Project
+    """
+    project_id = url.split("/")[4]
+    project = session.query(ProjectNode).get(project_id)
+    return project
+
+
+def newCorpus(project, source, name=None, query=None):
+    error = False
+
+    if name is None:
+        name = query
+
+    if not isinstance(project, ProjectNode):
+        error = "a valid project"
+    if not isinstance(source, int) and not isinstance(source, str):
+        error = "a valid source identifier: id or name"
+    elif not isinstance(query, str):
+        error = "a valid query"
+    elif not isinstance(name, str):
+        error = "a valid name"
+
+    if error:
+        raise NotebookError("Please provide %s." % error)
+
+    resource = get_resource(source) if isinstance(source, int) else \
+               get_resource_by_name(source)
+
+    moissonneur_name = get_moissonneur_name(resource) if resource else \
+                       source.lower()
+
+    try:
+        moissonneur = get_moissonneur(moissonneur_name)
+    except ImportError:
+        raise NotebookError("Invalid source identifier: %r" % source)
+
+    return run_moissonneur(moissonneur, project, name, query)
+
+
+def get_moissonneur_name(ident):
+    """ Return moissonneur module name from RESOURCETYPE or crawler name """
+
+    # Does it quacks like a RESOURCETYPE ?
+    if hasattr(ident, 'get'):
+        ident = ident.get('crawler')
+
+    # Extract name from crawler class name, otherwise assume ident is already
+    # a moissonneur name.
+    if isinstance(ident, str) and ident.endswith('Crawler'):
+        return ident[:-len('Crawler')].lower()
+
+
+def get_moissonneur(name):
+    """ Return moissonneur module from its name """
+    if not isinstance(name, str) or not name.islower():
+        raise NotebookError("Invalid moissonneur name: %r" % name)
+
+    module = importlib.import_module('moissonneurs.%s' % name)
+    module.name = name
+
+    return module
+
+
+def run_moissonneur(moissonneur, project, name, query):
+    """ Run moissonneur and return resulting corpus """
+
+    # XXX Uber-kludge with gory details. Spaghetti rulezzzzz!
+    class Dummy(object):
+        pass
+
+    request = Dummy()
+    request.method = 'POST'
+    request.path = 'nowhere'
+    request.META = {}
+    # XXX 'string' only have effect on moissonneurs.pubmed; its value is added
+    #     when processing request client-side, take a deep breath and see
+    #     templates/projects/project.html for more details.
+    request.POST = {'string': name,
+                    'query': query,
+                    'N': QUERY_SIZE_N_MAX}
+    request.user = Dummy()
+    request.user.id = project.user_id
+    request.user.is_authenticated = lambda: True
+
+    if moissonneur.name == 'istex':
+        # Replace ALL spaces by plus signs
+        request.POST['query'] = '+'.join(filter(None, query.split(' ')))
+
+    try:
+        import json
+
+        r = moissonneur.query(request)
+        raw_json = r.content.decode('utf-8')
+        data = json.loads(raw_json)
+
+        if moissonneur.name == 'pubmed':
+            count = sum(x['count'] for x in data)
+            request.POST['query'] = raw_json
+        elif moissonneur.name == 'istex':
+            count = data.get('total', 0)
+        else:
+            count = data.get('results_nb', 0)
+
+        if count > 0:
+            corpus = moissonneur.save(request, project.id, return_corpus=True)
+        else:
+            return None
+
+    except (ValueError, Http404) as e:
+        raise e
+
+    # Sometimes strange things happens...
+    if corpus.name != name:
+        corpus.name = name
+        session.commit()
+
+    return corpus

moissonneurs/cern.py

@@ -30,7 +30,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -101,6 +101,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/hal.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -103,6 +103,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/isidore.py

@@ -29,7 +29,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -100,6 +100,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/istex.py

@@ -52,7 +52,7 @@ def query( request ):
 
 
 
-def save(request , project_id):
+def save(request , project_id, return_corpus=False):
     print("testISTEX:")
     print(request.method)
     alist = ["bar","foo"]
@@ -171,6 +171,9 @@ def save(request , project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/multivac.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -104,6 +104,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/pubmed.py

@@ -69,7 +69,7 @@ def query( request ):
     return JsonHttpResponse(data)
 
 
-def save( request , project_id ) :
+def save( request , project_id, return_corpus=False ) :
     # implicit global session
     # do we have a valid project id?
     try:
@@ -164,6 +164,10 @@ def save( request , project_id ) :
             session.rollback()
             # --------------------------------------------
         sleep(1)
+
+        if return_corpus:
+            return corpus
+
         return HttpResponseRedirect('/projects/' + str(project_id))
 
     data = alist

AdvancedTutorial.ipynb → notebooks/.ipynb_checkpoints/AdvancedTutorial-checkpoint.ipynb

@@ -2,11 +2,38 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Advanced Gargantext Tutorial (Python)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "'list' object is not callable",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m               Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-3-a8e3501c9a54>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'/srv/gargantext'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m: 'list' object is not callable"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "sys.pa"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -28,7 +55,9 @@
    "cell_type": "code",
    "execution_count": 8,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {

templates/pages/projects/project.html

@@ -532,7 +532,7 @@
                                         $("#submit_thing").html("Process a {{ query_size }} sample!")
 
                                         thequeries = data
-                                        var N=0,k=0;
+                                        var N=0;
 
                                         for(var i in thequeries) N += thequeries[i].count
                                         if( N>0) {
@@ -571,12 +571,11 @@
                                         $("#submit_thing").html("Process a {{ query_size }} sample!")
 
                                         thequeries = data
-                                        var N=data.length,k=0;
-                                        // for(var i in thequeries) N += thequeries[i].count
-                                        if( N>1) {
-                                            var total = JSON.parse(data).total
-                                            console.log("N: "+total)
-                                            $("#theresults").html("<i> <b>"+pubmedquery+"</b>: "+total+" publications.</i><br>")
+                                        var N = data.total;
+
+                                        if (N > 0) {
+                                            console.log("N: "+N)
+                                            $("#theresults").html("<i> <b>"+pubmedquery+"</b>: "+N+" publications.</i><br>")
                                             $('#submit_thing').prop('disabled', false);
                                         } else {
                                             $("#theresults").html("<i>  <b>"+data[0]+"</b></i><br>")