Merge branch 'unstable' into simon-unstable

CHANGELOG.md

@@ -2,6 +2,8 @@
 * Guided Tour
 * Sources form highlighting crawlers
 
+## Version 3.0.7
+* Alembic implemented to manage database migrations
 
 ## Version 3.0.6.8
 * REPEC Crawler (connection with https://multivac.iscpif.fr)

docs/manual_install.md

-tools/manual_install.md
\ No newline at end of file

docs/manual_install.md

+* Create user gargantua
+Main user of Gargantext is Gargantua (role of Pantagruel soon)!
+``` bash
+sudo adduser --disabled-password --gecos "" gargantua
+```
+
+* Create the directories you need
+
+here for the example gargantext package will be installed in /srv/
+``` bash
+for dir in "/srv/gargantext"
+           "/srv/gargantext_lib"
+           "/srv/gargantext_static"
+           "/srv/gargantext_media"
+           "/srv/env_3-5"; do
+    sudo mkdir -p $dir ;
+    sudo chown gargantua:gargantua $dir ;
+done
+```
+
+You should see:
+
+```bash
+$tree /srv
+/srv
+├── gargantext
+├── gargantext_lib
+├── gargantext_media
+│   └── srv
+│       └── env_3-5
+└── gargantext_static
+```
+* Get the main libraries
+
+Download uncompress and make main user access to it.
+PLease, Be patient due to the size of the packages libraries (27GO)
+this step can be long....
+
+``` bash
+wget http://dl.gargantext.org/gargantext_lib.tar.bz2 \
+&& tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
+&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
+&& echo "Libs installed"
+```
+
+* Get the source code of Gargantext
+
+by cloning the repository of gargantext
+``` bash
+git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
+        && cd /srv/gargantext \
+        && git fetch origin stable \
+        && git checkout stable \
+```
+
+    TODO(soon): git clone https://gogs.iscpif.fr/gargantext.git
+
+
+* Install and configure the virtual environment
+``` bash
+cd /srv/
+pip3 install virtualenv
+virtualenv /srv/env_3-5 -p /usr/bin/python3.5
+pip install -r /srv/gargantext/install
+echo '/srv/gargantext' > /srv/env_3-5/lib/python3.5/site-packages/gargantext.pth
+echo 'alias venv="source /srv/env_3-5/bin/activate"' >> ~/.bashrc
+```
+
+
+See the [next steps of installation procedure](install.md#Install)
+See the [next manual steps of installation procedure](Debian.sh)

gargantext/constants.py

@@ -59,25 +59,25 @@ LISTTYPES = {
 NODETYPES = [
     # TODO separate id not array index, read by models.node
     None,                    # 0
-    # documents hierarchy
+    # node/file hierarchy
     'USER',                  # 1
     'PROJECT',               # 2
     #RESOURCE should be here but last
     'CORPUS',                # 3
     'DOCUMENT',              # 4
-    # lists
+    # lists of ngrams
     'STOPLIST',              # 5
     'GROUPLIST',             # 6
     'MAINLIST',              # 7
     'MAPLIST',               # 8
     'COOCCURRENCES',         # 9
-    # scores
+    # scores for ngrams
     'OCCURRENCES',           # 10
     'SPECCLUSION',           # 11
     'CVALUE',                # 12
     'TFIDF-CORPUS',          # 13
     'TFIDF-GLOBAL',          # 14
-    # docs subset
+    # node subset
     'FAVORITES',             # 15
     # more scores (sorry!)
 

gargantext/models/nodes.py

@@ -2,6 +2,9 @@ from gargantext.util.db import session
 from gargantext.util.files import upload
 from gargantext.constants import *
 
+# Uncomment to make column full text searchable
+#from sqlalchemy_utils.types import TSVectorType
+
 from datetime import datetime
 
 from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, \
@@ -57,23 +60,28 @@ class Node(Base):
             Index('nodes_user_id_typename_parent_id_idx', 'user_id', 'typename', 'parent_id'),
             Index('nodes_hyperdata_idx', 'hyperdata', postgresql_using='gin'))
 
+    # TODO
+    # create INDEX full_text_idx on nodes using gin(to_tsvector('english', hyperdata ->> 'abstract' || 'title'));
+
     id = Column(Integer, primary_key=True)
+
     typename = Column(NodeType, index=True)
+    __mapper_args__ = { 'polymorphic_on': typename }
+
     # foreign keys
-    user_id = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
-    parent_id = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
-    # main data
+    user_id       = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
+    user          = relationship(User)
+
+    parent_id     = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
+    parent        = relationship('Node', remote_side=[id])
+
     name = Column(String(255))
     date  = Column(DateTime(timezone=True), default=datetime.now)
-    # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
-    hyperdata = Column(JSONB, default=dict)
-
-    user = relationship(User)
-    parent = relationship('Node', remote_side=[id])
 
-    __mapper_args__ = {
-        'polymorphic_on': typename
-    }
+    hyperdata     = Column(JSONB, default=dict)
+    # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
+    # To make search possible uncomment the line below
+    #search_vector = Column(TSVectorType('hyperdata'))
 
     def __new__(cls, *args, **kwargs):
         if cls is Node and kwargs.get('typename'):

gargantext/util/crawlers/HAL.py

@@ -45,6 +45,7 @@ class HalCrawler(Crawler):
                , uri_s
                , isbn_s
                , issue_s
+               , docType_s
                , journalPublisher_s
              """
                #, authUrl_s

gargantext/util/db.py

@@ -5,9 +5,15 @@ from gargantext.util.json import json_dumps
 ########################################################################
 # get engine, session, etc.
 ########################################################################
+import sqlalchemy as sa
 from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import delete
 
+# To make Full Text search possible, uncomment lines below
+# (and install it with pip before)
+#from sqlalchemy_searchable import make_searchable
+
 def get_engine():
     from sqlalchemy import create_engine
     return create_engine( settings.DATABASES['default']['URL']
@@ -18,6 +24,13 @@ def get_engine():
 
 engine = get_engine()
 
+# To make Full Text search possible, uncomment lines below
+# https://sqlalchemy-searchable.readthedocs.io/
+#sa.orm.configure_mappers()
+Base = declarative_base()
+#Base.metadata.create_all(engine)
+#make_searchable()
+
 session = scoped_session(sessionmaker(bind=engine))
 
 

gargantext_notebook.py

+install/notebook/gargantext_notebook.py
\ No newline at end of file

install/docker.install

+#!/bin/bash
+
+sudo apt-get update
+sudo apt-get install \
+     apt-transport-https \
+     ca-certificates \
+     curl \
+     gnupg2 \
+     software-properties-common
+
+curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -
+sudo apt-key fingerprint 0EBFCD88
+echo "Should be: Key fingerprint = 9DC8 5822 9FC7 DD38 854A  E2D8 8D81 803C 0EBF CD88"
+
+sudo add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable"
+
+sudo apt-get update
+sudo apt-get install docker-ce
+sudo docker run hello-world

install/gargamelle/Dockerfile

@@ -15,9 +15,9 @@ RUN apt-get update && \
     apt-utils ca-certificates locales \
     sudo aptitude gcc g++ wget git vim \
     build-essential make \
-    postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5 \
-    postgresql-server-dev-9.5 libpq-dev libxml2 \
-    postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
+    postgresql-server-dev-9.6 libpq-dev libxml2 \
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
 
 
 ### Configure timezone and locale
@@ -37,7 +37,7 @@ ENV LC_ALL fr_FR.UTF-8
 ### Install main dependencies and python packages based on Debian distrib
 RUN echo "############# PYTHON DEPENDENCIES ###############"
 RUN apt-get update && apt-get install -y \
-        libxml2-dev xml-core libgfortran-5-dev \
+        libxml2-dev xml-core libgfortran-6-dev \
         libpq-dev \
         python3.5 \
         python3-dev \
@@ -47,8 +47,8 @@ RUN apt-get update && apt-get install -y \
         # python dependencies
         python3-pip \
         # for lxml
-        libxml2-dev libxslt-dev
-        #libxslt1-dev zlib1g-dev
+        libxml2-dev libxslt-dev \
+        libxslt1-dev zlib1g-dev
 
 # UPDATE AND CLEAN
 RUN apt-get update && apt-get autoclean &&\

install/gargamelle/requirements.txt

@@ -17,7 +17,7 @@ jdatetime==1.7.2
 kombu==3.0.37                  # messaging
 langdetect==1.0.6              #detectinglanguage
 nltk==3.1
-numpy==1.10.4
+numpy==1.13.1
 psycopg2==2.6.2
 pycountry==1.20
 python-dateutil==2.4.2
@@ -34,3 +34,4 @@ requests-futures==0.9.7
 bs4==0.0.1
 requests==2.10.0
 alembic>=0.9.2
+# SQLAlchemy-Searchable==0.10.4

install/notebook.install

+#!/bin/bash
+
+sudo adduser --disabled-password --gecos "" notebooks
+sudo docker rm $(sudo docker ps -a | grep sh | awk '{print $1}')
+sudo docker build -t garg-notebook:latest ./notebook

install/notebook.run

+#!/bin/bash
+
+        #-v /srv/gargandata:/srv/gargandata \
+        #-v /srv/gargantext_lib:/srv/gargantext_lib \
+
+sudo docker rm $(sudo docker ps -a | grep notebook | grep sh | awk '{print $1}')
+
+#HOSTIP=$(ip route show 0.0.0.0/0 | awk '{print $3}')
+#--add-host=localhost:${HOSTIP} \
+
+
+sudo docker run \
+        --name=garg-notebook \
+        --net=host \
+        -p 8899:8899 \
+        --env POSTGRES_HOST=localhost \
+        -v /srv/gargantext:/srv/gargantext \
+        -it garg-notebook:latest \
+        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
+#        #&& jupyter nbextension enable --py widgetsnbextension --sys-prefix 
+        #/bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser --notebook-dir=/home/notebooks/'"
+
+

install/notebook/Dockerfile

+###########################################################
+# Gargamelle WEB
+###########################################################
+#Build an image starting with debian:stretch image
+# wich contains all the source code of the app
+FROM debian:stretch
+MAINTAINER ISCPIF <gargantext@iscpif.fr>
+
+USER root
+
+
+### Update and install base dependencies
+RUN echo "############ DEBIAN LIBS ###############"
+RUN apt-get update &&                                     \
+    apt-get install -y                                     \
+    apt-utils ca-certificates locales                       \
+    sudo aptitude gcc g++ wget git vim                       \
+    build-essential make                                      \
+    curl                                                       
+#    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
+#    postgresql-server-dev-9.6 libpq-dev libxml2                  \
+#    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
+
+# Install Stack
+
+### Configure timezone and locale
+RUN echo "###########  LOCALES & TZ #################"
+RUN echo "Europe/Paris" > /etc/timezone
+ENV TZ "Europe/Paris"
+
+RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \
+    sed -i -e 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen &&  \
+    dpkg-reconfigure --frontend=noninteractive locales                   &&   \
+    echo 'LANG="fr_FR.UTF-8"' > /etc/default/locale
+ENV LANG fr_FR.UTF-8
+ENV LANGUAGE fr_FR.UTF-8
+ENV LC_ALL fr_FR.UTF-8
+
+
+### Install main dependencies and python packages based on Debian distrib
+RUN echo "############# PYTHON DEPENDENCIES ###############"
+RUN apt-get update && apt-get install -y       \
+        libxml2-dev xml-core libgfortran-6-dev  \
+        libpq-dev                                \
+        python3.5                                 \
+        python3-dev                                \
+        # for numpy, pandas and numpyperf           \
+        python3-six python3-numpy python3-setuptools \
+        python3-numexpr                               \
+        # python dependencies                          \
+        python3-pip                                     \
+        # for lxml
+        libxml2-dev libxslt-dev libxslt1-dev zlib1g-dev
+
+
+# UPDATE AND CLEAN
+RUN apt-get update && apt-get autoclean \ 
+    && rm -rf /var/lib/apt/lists/*
+#NB: removing /var/lib will avoid to significantly fill up your /var/ folder on your native system
+
+########################################################################
+### PYTHON ENVIRONNEMENT (as ROOT)
+########################################################################
+
+RUN adduser --disabled-password --gecos "" notebooks
+
+RUN pip3 install virtualenv
+RUN virtualenv /env_3-5
+RUN echo 'alias venv="source /env_3-5/bin/activate"' >> ~/.bashrc
+# CONFIG FILES
+ADD requirements.txt /
+#ADD psql_configure.sh /
+ADD django_configure.sh /
+
+RUN . /env_3-5/bin/activate && pip3 install -r requirements.txt                         && \
+    pip3  install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1                  && \
+    python3 -m nltk.downloader averaged_perceptron_tagger -d /usr/local/share/nltk_data
+
+#RUN ./psql_configure.sh
+#RUN ./django_configure.sh
+
+RUN chown notebooks:notebooks -R /env_3-5
+
+########################################################################
+### Notebook IHaskell and IPYTHON ENVIRONNEMENT
+########################################################################
+
+#RUN apt-get update && apt-get install -y    \
+#        libtinfo-dev                        \
+#        libzmq3-dev                         \
+#        libcairo2-dev                       \
+#        libpango1.0-dev                     \
+#        libmagic-dev                        \
+#        libblas-dev                         \
+#        liblapack-dev
+#RUN curl -sSL https://get.haskellstack.org/ | sh 
+#RUN stack setup
+#RUN git clone https://github.com/gibiansky/IHaskell
+#RUN . /env_3-5/bin/activate     \
+#    && cd IHaskell                    \
+#    && stack install gtk2hs-buildtools \
+#    && stack install --fast             \
+#    && /root/.local/bin/ihaskell install --stack
+#
+
+#
+########################################################################
+### POSTGRESQL DATA (as ROOT)
+########################################################################
+
+#RUN sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargandata\'%" /etc/postgresql/9.5/main/postgresql.conf
+#RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/9.5/main/pg_hba.conf
+#RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+
+EXPOSE 5432 8899
+
+VOLUME ["/srv/","/home/notebooks/"]

install/notebook/django_configure.sh

+#!/bin/bash
+##################################################
+#             __| |(_) __ _ _ __   __ _  ___
+#            / _` || |/ _` | '_ \ / _` |/ _ \
+#           | (_| || | (_| | | | | (_| | (_) |
+#            \__,_|/ |\__,_|_| |_|\__, |\___/
+#                 |__/             |___/
+##################################################
+#configure django migrations
+##################################################
+echo "::::: DJANGO :::::"
+#echo "Starting Postgres"
+#/usr/sbin/service postgresql start
+
+
+
+su gargantua -c 'source /srv/env_3-5/bin/activate &&\
+    echo "Activated env" &&\
+    /srv/gargantext/manage.py makemigrations &&\
+    /srv/gargantext/manage.py migrate && \
+    echo "migrations ok" &&\
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/manage.py createsuperuser'
+
+service postgresql stop

install/notebook/gargantext_notebook.py

+"""
+   Gargantext Software Copyright (c) 2016-2017 CNRS ISC-PIF -
+http://iscpif.fr
+    Licence (see :
+http://gitlab.iscpif.fr/humanities/gargantext/blob/stable/LICENSE )
+    - In France : a CECILL variant affero compliant
+    - GNU aGPLV3 for all other countries
+"""
+#!/usr/bin/env python
+import sys
+import os
+
+
+# Django settings
+dirname = os.path.dirname(os.path.realpath(__file__))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
+
+# initialize Django application
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from gargantext.util.toolchain.main import parse_extract_indexhyperdata
+from gargantext.util.db import *
+from gargantext.models import Node
+
+from nltk.tokenize import wordpunct_tokenize
+
+from gargantext.models import *
+from nltk.tokenize import word_tokenize
+import nltk as nltk
+from statistics import mean
+from math import log
+from collections import defaultdict
+import matplotlib.pyplot as plt
+import numpy as np
+import datetime
+
+from collections import Counter
+from langdetect import detect as detect_lang
+
+def documents(corpus_id):
+    return (session.query(Node).filter( Node.parent_id==corpus_id
+                                  , Node.typename=="DOCUMENT"
+                                  )
+        # .order_by(Node.hyperdata['publication_date'])
+        .all()
+        )
+
+
+#import seaborn as sns
+import pandas as pd
+
+def chart(docs, field):
+    year_publis = list(Counter([doc.hyperdata[field] for doc in docs]).items())
+    frame0 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'])
+    frame1 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'], index=frame0.Date)
+    return frame1
+
+
+from gargantext.util.crawlers.HAL import HalCrawler
+
+def scan_hal(request):
+    hal = HalCrawler()
+    return hal.scan_results(request)
+
+def scan_gargantext(corpus_id, lang, request):
+    connection = get_engine().connect()
+    # TODO add some sugar the request (ideally request should be the same for hal and garg)
+    query = """select count(n.id) from nodes n
+                  where to_tsvector('%s', hyperdata ->> 'abstract' || 'title') 
+                  @@ to_tsquery('%s')
+                  AND n.parent_id = %s;""" % (lang, request, corpus_id) 
+    return [i for i in connection.execute(query)][0][0]
+    connection.close()
+

install/notebook/psql_configure.sh

+#!/bin/bash
+
+#######################################################################
+##    ____           _
+##   |  _ \ ___  ___| |_ __ _ _ __ ___  ___
+##   | |_) / _ \/ __| __/ _` | '__/ _ \/ __|
+##   |  __/ (_) \__ \ || (_| | | |  __/\__ \
+##   |_|   \___/|___/\__\__, |_|  \___||___/
+##                      |___/
+#######################################################################
+echo "::::: POSTGRESQL :::::"
+su postgres -c 'pg_dropcluster 9.4 main --stop'
+#done in docker but redoing it
+rm -rf /srv/gargandata && mkdir /srv/gargandata && chown postgres:postgres /srv/gargandata
+su postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /srv/gargandata/'
+su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /srv/gargandata/ -l /srv/gargandata/journal_applicatif start'
+
+su postgres -c 'pg_createcluster -D /srv/gargandata 9.6 main '
+su postgres -c 'pg_ctlcluster -D /srv/gargandata 9.6 main start '
+su postgres -c 'pg_ctlcluster 9.6 main start'
+
+service postgresql start
+
+su postgres -c "psql -c \"CREATE user gargantua WITH PASSWORD 'C8kdcUrAQy66U'\""
+su postgres -c "createdb -O gargantua gargandb"
+
+echo "Postgres configured"
+#service postgresql stop

install/notebook/requirements.txt

+# try bottleneck
+eventlet==0.20.1
+amqp==1.4.9
+anyjson==0.3.3
+billiard==3.3.0.23
+celery==3.1.25
+chardet==2.3.0
+dateparser==0.3.5
+Django==1.10.5
+django-celery==3.2.1
+django-pgfields==1.4.4
+django-pgjsonb==0.0.23
+djangorestframework==3.5.3
+html5lib==0.9999999
+#python-igraph>=0.7.1
+jdatetime==1.7.2
+kombu==3.0.37                  # messaging
+langdetect==1.0.6              #detectinglanguage
+nltk==3.1
+numpy==1.13.1
+psycopg2==2.6.2
+pycountry==1.20
+python-dateutil==2.4.2
+pytz==2016.10                   # timezones
+PyYAML==3.11
+RandomWords==0.1.12
+ujson==1.35
+umalqurra==0.2                 # arabic calendars (?? why use ??)
+networkx==1.11
+pandas==0.18.0
+six==1.10.0
+lxml==3.5.0
+requests-futures==0.9.7
+bs4==0.0.1
+requests==2.10.0
+djangorestframework-jwt==1.9.0
+jupyter==1.0.0
+jupyter-client==5.0.0
+jupyter-console==5.1.0
+jupyter-core==4.3.0
+ipython==5.2.0
+ipython-genutils==0.1.0
+ipywidgets
+matplotlib==2.0.2

templates/pages/menu.html

@@ -367,7 +367,7 @@
             <p>
                 Gargantext
                 <span class="glyphicon glyphicon-registration-mark" aria-hidden="true"></span>
-                , version 3.0.6.9.4,
+                , version 3.0.7,
                 <a href="http://www.cnrs.fr" target="blank" title="Institution that enables this project.">
                     Copyrights
                     <span class="glyphicon glyphicon-copyright-mark" aria-hidden="true"></span>