[FEAT] Adding notebook backend. TODO: create user notebooks on server.

gargantext/models/nodes.py

@@ -58,28 +58,30 @@ class Node(Base):
     __table_args__ = (
             Index('nodes_user_id_typename_parent_id_idx', 'user_id', 'typename', 'parent_id'),
             Index('nodes_hyperdata_idx', 'hyperdata'))
+    
+    # TODO 
+    # create INDEX full_text_idx on nodes using gin(to_tsvector('english', hyperdata ->> 'abstract' || 'title'));
 
     id = Column(Integer, primary_key=True)
+    
     typename = Column(NodeType, index=True)
+    __mapper_args__ = { 'polymorphic_on': typename }
+    
     # foreign keys
     user_id       = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
+    user          = relationship(User)
+    
     parent_id     = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
-    # main data
+    parent        = relationship('Node', remote_side=[id])
+    
     name = Column(String(255))
     date  = Column(DateTime(timezone=True), default=datetime.now)
-    # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
-    hyperdata     = Column(JSONB, default=dict)
     
+    hyperdata     = Column(JSONB, default=dict)
+    # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
     # To make search possible uncomment the line below
     #search_vector = Column(TSVectorType('hyperdata'))
 
-    user = relationship(User)
-    parent = relationship('Node', remote_side=[id])
-
-    __mapper_args__ = {
-        'polymorphic_on': typename
-    }
-
     def __new__(cls, *args, **kwargs):
         if cls is Node and kwargs.get('typename'):
             typename = kwargs.pop('typename')

install/gargamelle/requirements.txt

@@ -34,4 +34,4 @@ requests-futures==0.9.7
 bs4==0.0.1
 requests==2.10.0
 alembic>=0.9.2
-SQLAlchemy-Searchable==0.10.4
+# SQLAlchemy-Searchable==0.10.4

install/notebook.install

+#!/bin/bash
+
+sudo docker rm $(sudo docker ps -a | grep sh | awk '{print $1}')
+sudo docker build -t garg-notebook:latest ./notebook

install/notebook.run

+#!/bin/bash
+
+        #-v /srv/gargandata:/srv/gargandata \
+        #-v /srv/gargantext_lib:/srv/gargantext_lib \
+
+sudo docker rm $(sudo docker ps -a | grep notebook | grep sh | awk '{print $1}')
+sudo docker run \
+        --name=garg-notebook \
+        -v /srv/gargantext:/srv/gargantext \
+        -p 8899:8899 \
+        -it garg-notebook:latest \
+        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser --notebook-dir=/home/notebooks/'"
+        #/bin/bash -c "/bin/su gargantua -c 'source /env_3-5/bin/activate && jupyter notebook --port=8899 --ip=127.0.0.1 --no-browser'"
+
+

install/notebook/Dockerfile

+###########################################################
+# Gargamelle WEB
+###########################################################
+#Build an image starting with debian:stretch image
+# wich contains all the source code of the app
+FROM debian:stretch
+MAINTAINER ISCPIF <gargantext@iscpif.fr>
+
+USER root
+
+
+### Update and install base dependencies
+RUN echo "############ DEBIAN LIBS ###############"
+RUN apt-get update &&                                     \
+    apt-get install -y                                     \
+    apt-utils ca-certificates locales                       \
+    sudo aptitude gcc g++ wget git vim                       \
+    build-essential make                                      \
+    curl                                                       \
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
+    postgresql-server-dev-9.6 libpq-dev libxml2                  \
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
+
+# Install Stack
+
+### Configure timezone and locale
+RUN echo "###########  LOCALES & TZ #################"
+RUN echo "Europe/Paris" > /etc/timezone
+ENV TZ "Europe/Paris"
+
+RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \
+    sed -i -e 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen &&  \
+    dpkg-reconfigure --frontend=noninteractive locales                   &&   \
+    echo 'LANG="fr_FR.UTF-8"' > /etc/default/locale
+ENV LANG fr_FR.UTF-8
+ENV LANGUAGE fr_FR.UTF-8
+ENV LC_ALL fr_FR.UTF-8
+
+
+### Install main dependencies and python packages based on Debian distrib
+RUN echo "############# PYTHON DEPENDENCIES ###############"
+RUN apt-get update && apt-get install -y       \
+        libxml2-dev xml-core libgfortran-6-dev  \
+        libpq-dev                                \
+        python3.5                                 \
+        python3-dev                                \
+        # for numpy, pandas and numpyperf           \
+        python3-six python3-numpy python3-setuptools \
+        python3-numexpr                               \
+        # python dependencies                          \
+        python3-pip                                     \
+        # for lxml
+        libxml2-dev libxslt-dev libxslt1-dev zlib1g-dev
+
+
+# UPDATE AND CLEAN
+RUN apt-get update && apt-get autoclean \ 
+    && rm -rf /var/lib/apt/lists/*
+#NB: removing /var/lib will avoid to significantly fill up your /var/ folder on your native system
+
+########################################################################
+### PYTHON ENVIRONNEMENT (as ROOT)
+########################################################################
+
+RUN adduser --disabled-password --gecos "" notebooks
+
+RUN pip3 install virtualenv
+RUN virtualenv /env_3-5
+RUN echo 'alias venv="source /env_3-5/bin/activate"' >> ~/.bashrc
+# CONFIG FILES
+ADD requirements.txt /
+ADD psql_configure.sh /
+ADD django_configure.sh /
+
+RUN . /env_3-5/bin/activate && pip3 install -r requirements.txt                         && \
+    pip3  install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1                  && \
+    python3 -m nltk.downloader averaged_perceptron_tagger -d /usr/local/share/nltk_data
+
+#RUN ./psql_configure.sh
+#RUN ./django_configure.sh
+
+RUN chown notebooks:notebooks -R /env_3-5
+
+########################################################################
+### Notebook IHaskell and IPYTHON ENVIRONNEMENT
+########################################################################
+
+RUN apt-get update && apt-get install -y    \
+        libtinfo-dev                        \
+        libzmq3-dev                         \
+        libcairo2-dev                       \
+        libpango1.0-dev                     \
+        libmagic-dev                        \
+        libblas-dev                         \
+        liblapack-dev
+RUN curl -sSL https://get.haskellstack.org/ | sh 
+RUN stack setup
+RUN git clone https://github.com/gibiansky/IHaskell
+RUN . /env_3-5/bin/activate     \
+    && cd IHaskell                    \
+    && stack install gtk2hs-buildtools \
+    && stack install --fast             \
+    && /root/.local/bin/ihaskell install --stack
+
+
+#
+########################################################################
+### POSTGRESQL DATA (as ROOT)
+########################################################################
+
+#RUN sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargandata\'%" /etc/postgresql/9.5/main/postgresql.conf
+#RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/9.5/main/pg_hba.conf
+#RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+
+EXPOSE 8899
+
+VOLUME ["/srv/","/home/notebooks/"]

install/notebook/django_configure.sh

+#!/bin/bash
+##################################################
+#             __| |(_) __ _ _ __   __ _  ___
+#            / _` || |/ _` | '_ \ / _` |/ _ \
+#           | (_| || | (_| | | | | (_| | (_) |
+#            \__,_|/ |\__,_|_| |_|\__, |\___/
+#                 |__/             |___/
+##################################################
+#configure django migrations
+##################################################
+echo "::::: DJANGO :::::"
+#echo "Starting Postgres"
+#/usr/sbin/service postgresql start
+
+
+
+su gargantua -c 'source /srv/env_3-5/bin/activate &&\
+    echo "Activated env" &&\
+    /srv/gargantext/manage.py makemigrations &&\
+    /srv/gargantext/manage.py migrate && \
+    echo "migrations ok" &&\
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/manage.py createsuperuser'
+
+service postgresql stop

install/notebook/gargantext_notebook.py

+"""
+   Gargantext Software Copyright (c) 2016 CNRS ISC-PIF -
+http://iscpif.fr
+    Licence (see :
+http://gitlab.iscpif.fr/humanities/gargantext/blob/stable/LICENSE )
+    - In France : a CECILL variant affero compliant
+    - GNU aGPLV3 for all other countries
+"""
+#!/usr/bin/env python
+import sys
+import os
+
+
+# Django settings
+dirname = os.path.dirname(os.path.realpath(__file__))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
+
+# initialize Django application
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from gargantext.util.toolchain.main import parse_extract_indexhyperdata
+from gargantext.util.db import *
+from gargantext.models import Node
+
+from nltk.tokenize import wordpunct_tokenize
+
+from gargantext.models import *
+from nltk.tokenize import word_tokenize
+import nltk as nltk
+from statistics import mean
+from math import log
+from collections import defaultdict
+import matplotlib.pyplot as plt
+import numpy as np
+import datetime
+
+from collections import Counter
+from langdetect import detect as detect_lang
+
+def documents(corpus_id):
+    return (session.query(Node).filter( Node.parent_id==corpus_id
+                                  , Node.typename=="DOCUMENT"
+                                  )
+        # .order_by(Node.hyperdata['publication_date'])
+        .all()
+        )
+
+
+import seaborn as sns
+import pandas as pd
+
+def chart(docs, field):
+    year_publis = list(Counter([doc.hyperdata[field] for doc in docs]).items())
+    frame0 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'])
+    frame1 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'], index=frame0.Date)
+    return frame1
+
+
+from gargantext.util.crawlers.HAL import HalCrawler
+
+def scan_hal(request):
+    hal = HalCrawler()
+    return hal.scan_results(request)
+
+def scan_gargantext(corpus_id, lang, request):
+    connection = get_engine().connect()
+    # TODO add some sugar the request (ideally request should be the same for hal and garg)
+    query = """select count(n.id) from nodes n
+                  where to_tsvector('%s', hyperdata ->> 'abstract' || 'title') 
+                  @@ to_tsquery('%s')
+                  AND n.parent_id = %s;""" % (lang, request, corpus_id) 
+    return [i for i in connection.execute(query)][0][0]
+    connection.close()
+

install/notebook/psql_configure.sh

+#!/bin/bash
+
+#######################################################################
+##    ____           _
+##   |  _ \ ___  ___| |_ __ _ _ __ ___  ___
+##   | |_) / _ \/ __| __/ _` | '__/ _ \/ __|
+##   |  __/ (_) \__ \ || (_| | | |  __/\__ \
+##   |_|   \___/|___/\__\__, |_|  \___||___/
+##                      |___/
+#######################################################################
+echo "::::: POSTGRESQL :::::"
+su postgres -c 'pg_dropcluster 9.4 main --stop'
+#done in docker but redoing it
+rm -rf /srv/gargandata && mkdir /srv/gargandata && chown postgres:postgres /srv/gargandata
+su postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /srv/gargandata/'
+su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /srv/gargandata/ -l /srv/gargandata/journal_applicatif start'
+
+su postgres -c 'pg_createcluster -D /srv/gargandata 9.6 main '
+su postgres -c 'pg_ctlcluster -D /srv/gargandata 9.6 main start '
+su postgres -c 'pg_ctlcluster 9.6 main start'
+
+service postgresql start
+
+su postgres -c "psql -c \"CREATE user gargantua WITH PASSWORD 'C8kdcUrAQy66U'\""
+su postgres -c "createdb -O gargantua gargandb"
+
+echo "Postgres configured"
+#service postgresql stop

install/notebook/requirements.txt

+# try bottleneck
+eventlet==0.20.1
+amqp==1.4.9
+anyjson==0.3.3
+billiard==3.3.0.23
+celery==3.1.25
+chardet==2.3.0
+dateparser==0.3.5
+Django==1.10.5
+django-celery==3.2.1
+django-pgfields==1.4.4
+django-pgjsonb==0.0.23
+djangorestframework==3.5.3
+html5lib==0.9999999
+#python-igraph>=0.7.1
+jdatetime==1.7.2
+kombu==3.0.37                  # messaging
+langdetect==1.0.6              #detectinglanguage
+nltk==3.1
+numpy==1.10.4
+psycopg2==2.6.2
+pycountry==1.20
+python-dateutil==2.4.2
+pytz==2016.10                   # timezones
+PyYAML==3.11
+RandomWords==0.1.12
+ujson==1.35
+umalqurra==0.2                 # arabic calendars (?? why use ??)
+networkx==1.11
+pandas==0.18.0
+six==1.10.0
+lxml==3.5.0
+requests-futures==0.9.7
+bs4==0.0.1
+requests==2.10.0
+djangorestframework-jwt==1.9.0
+jupyter==1.0.0
+jupyter-client==5.0.0
+jupyter-console==5.1.0
+jupyter-core==4.3.0
+ipython==5.2.0
+ipython-genutils==0.1.0