INstall.sh

gargantext/constants.py

@@ -106,8 +106,8 @@ from gargantext.util.taggers import EnglishMeltTagger, FrenchMeltTagger, TurboTa
 
 LANGUAGES = {
     'en': {
-        #'tagger': EnglishMeltTagger,
-        'tagger': TurboTagger,
+        'tagger': EnglishMeltTagger,
+        #'tagger': TurboTagger,
         #'tagger': NltkTagger,
     },
     'fr': {

gargantext/urls.py

@@ -13,7 +13,6 @@ from django.conf.urls    import include, url
 from django.contrib      import admin
 
 import gargantext.views.api.urls
-import gargantext.views.generated.urls
 import gargantext.views.pages.urls
 
 # Module Annotation

gargantext/views/api/ngramlists.py

@@ -297,6 +297,85 @@ class ListChange(APIView):
         return(base_list, change_list)
 
 
+class MapListGlance(APIView):
+    """
+    Fast infos about the maplist only
+
+    HOST/api/ngramlists/glance?corpus=2
+    HOST/api/ngramlists/glance?maplist=92
+
+    REST Parameters:
+    "maplist=92"
+        the maplist to retrieve
+    "corpus=ID"
+        alternatively, the corpus to which the maplist belongs
+    """
+
+    def get(self, request):
+        parameters = get_parameters(request)
+
+        maplist_id = None
+        scores_id = None
+
+        if "corpus" in parameters:
+            corpus_id = parameters['corpus']
+            corpus = cache.Node[corpus_id]
+            maplist_id = corpus.children('MAPLIST').first().id
+            # with a corpus_id, the explicit scoring pointer is optional
+            if "scoring" in parameters:
+                scores_id = parameters['scoring']
+            else:
+                scores_id = corpus.children('OCCURRENCES').first().id
+
+        elif "maplist" in parameters and "scoring" in parameters:
+            maplist_id = int(parameters['mainlist'])
+            scores_id = int(parameters['scoring'])
+        else:
+            raise ValidationException("A 'corpus' id or 'maplist' id is required, and a 'scoring' for occurences counts")
+
+        ngraminfo = {}           # ngram details sorted per ngram id
+        listmembers = {'maplist':[]}         # ngram ids sorted per list name
+
+        # infos for all ngrams from maplist
+        map_ngrams = _query_list(maplist_id, details=True,
+                                      scoring_metric_id= scores_id).all()
+
+        # ex:  [(8805, 'mean age', 4.0),
+        #        (1632, 'activity', 4.0),
+        #        (8423, 'present', 2.0),
+        #        (2928, 'objective', 2.0)]
+
+
+        # shortcut to useful function during loop
+        add_to_members = listmembers['maplist'].append
+
+        for ng in map_ngrams:
+            ng_id   = ng[0]
+            ngraminfo[ng_id] = ng[1:]
+
+            # maplist ngrams will already be <=> ngraminfos
+            # but the client side expects a membership lookup
+            # as when there are multiple lists or some groupings
+            add_to_members(ng_id)
+
+
+        return JsonHttpResponse({
+            'ngraminfos' : ngraminfo,
+            'listmembers' : listmembers,
+            'links' : {},   # no grouping links sent during glance (for speed)
+            'nodeids' : {
+                'mainlist':  None,
+                'maplist' :  maplist_id,
+                'stoplist':  None,
+                'groups':  None,
+                'scores':  None,
+            }
+        })
+
+
+
+
+
 class ListFamily(APIView):
     """
     Compact combination of *multiple* list info
@@ -320,7 +399,7 @@ class ListFamily(APIView):
     REST Parameters:
     "head=20"
         use pagination to only load the k top ngrams of the mainlist
-        (useful for fast loading of terms view)
+        (useful for fast loading of terms view) [CURRENTLY NOT USED]
     "corpus=ID"
         the corpus id to retrieve all 4 lists
     "scoring=ID"

gargantext/views/api/urls.py

@@ -21,10 +21,13 @@ urlpatterns = [ url(r'^nodes$'                , nodes.NodeListResource.as_view()
                 # post data looks like : {"767":[209,640],"779":[436,265,385]}"
 
               , url(r'^ngramlists/family$'     , ngramlists.ListFamily.as_view())
-                # entire combination of lists from a corpus
+                # entire combination of lists from a corpus, dedicated to termtable
                 # (or any combination of lists that go together :
                 #   - a mainlist
                 #   - an optional stoplist
                 #   - an optional maplist
                 #   - an optional grouplist
+
+              , url(r'^ngramlists/maplist$'     , ngramlists.MapListGlance.as_view())
+                # fast access to maplist, similarly formatted for termtable
               ]

init_accounts.py

+#!/usr/bin/env python
+import sys
+import os
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
+
+# initialize Django application
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+
+from django.contrib.auth import password_validation
+from django.contrib.auth.hashers import ( check_password
+                                        , is_password_usable
+                                        , make_password
+                                        )
+from django.db import models
+from django.utils.crypto import get_random_string, salted_hmac
+
+
+# retrieve Django models
+import django.apps
+django_models = django.apps.apps.get_models()
+django_models_names = set(model._meta.db_table for model in django_models)
+
+
+from gargantext.util.db      import session
+from gargantext.models.users import User
+from django.core.mail        import send_mail
+
+def make_random_password(length=10
+                         , allowed_chars='abcdefghjkmnpqrstuvwxyz'
+                                         'ABCDEFGHJKLMNPQRSTUVWXYZ'
+                                         '0123456789'):
+    """
+    Generate a random password with the given length and given
+    allowed_chars. The default value of allowed_chars does not have "I" or
+    "O" or letters and digits that look similar -- just to avoid confusion.
+    (source: django/contrib/auth)
+    """
+    return get_random_string(length, allowed_chars)
+
+
+def notify_user(username, email, password):
+    message = '''
+    Bonjour,
+    votre compte vient d'être créé.
+
+    Vous pouvez désormais vous connecter ici:
+    http://gargantext.org
+
+    Votre login est: %s
+    Votre mot de passe est : %s
+
+    Bientôt, il y aura une formation Gargantext (gratuite).
+    Inscription obligatoire pour les dernière places:
+
+                    http://iscpif.fr/
+
+    Nous restons à votre disposition pour tout complément d'information.
+    Cordialement
+    --
+        L'équipe de Gargantext (CNRS)
+
+    ''' % (username, password)
+
+    send_mail('[Gargantext] Votre accès à la plateforme', message, 'alexandre.delanoe@iscpif.fr', [email], fail_silently=False )
+
+    # add option for mass sending email
+
+def create_user(username, email, user=None, password=None, active=False, notify=True):
+    if user is None:
+        user = User()
+    user.username  = username
+    user.email     = email
+    user.is_active = True
+    if password is None or password == "":
+        password = make_random_password()
+    user.password = make_password(password)
+    session.add(user)
+    session.commit()
+
+    if notify == True:
+        notify_user(username, email, password)
+
+    return user
+
+def delete_user(username):
+    session.query(User).filter(User.username == username).delete()
+
+def active_user(username, active=True):
+    '''
+    To get inactive, active=False
+    '''
+    user = session.query(User).filter(User.username == username).first()
+    user.is_active = True
+    user.save()
+
+def mass_account_creation(fichier=None,init=False):
+    if fichier is None:
+        fichier = "/tmp/comptes.csv"
+    accounts = open(fichier, "r")
+    for line in accounts.readlines():
+        username, email, password, fin = line.split(',')
+        try:
+            user = session.query(User).filter(User.username == username).first()
+            print("User %s does exist already" % (username))
+            if init == True:
+                create_user(username, email, user=user, password=password, active=True, notify=True)
+                print("User %s updated" % (username))
+        except:
+            print("User %s does not exist already" % (username))
+            create_user(username, email, password=password, active=True, notify=True)
+        #delete_user(username)
+    accounts.close()
+
+if __name__ == "__main__":
+    mass_account_creation(fichier=sys.argv[1], init=True)
+
+

install/README.md

 #Gargantext
 ==========
 
+<<<<<<< HEAD
 Install Instructions for Gargantext (CNRS):
 
 1. [SETUP](##SETUP)
@@ -165,7 +166,120 @@ Password : autnagrag
 Enjoy :)
 
 
+=======
+Install Instructions for Gargantext (CNRS).
 
+## Help needed ?
+See http://gargantext.org/about and tools for the community
+
+## Create user Gargantua
+
+Main user of Gargantext is Gargantua (role of Pantagruel soon)!
+>>>>>>> constance-docker
+
+``` bash
+sudo adduser --disabled-password --gecos "" gargantua
+```
+
+<<<<<<< HEAD
+
+=======
+## Create the directories you need
+
+``` bash
+for dir in "/srv/gargantext"
+           "/srv/gargantext_lib"
+           "/srv/gargantext_static"
+           "/srv/gargantext_media"
+           "/srv/env_3-5"; do 
+    sudo mkdir -p $dir ;
+    sudo chown gargantua:gargantua $dir ; 
+done
+```
+
+## Get the source code of Gargantext
+
+``` bash
+git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
+        && cd /srv/gargantext \
+        && git fetch origin refactoring \
+        && git checkout refactoring \
+```
+
+### TODO (soon) : git clone https://gogs.iscpif.fr/gargantext.git
+
+## Build your OS dependencies
+
+2 ways, for each you need to install Debian GNU/Linux dependencies.
+
+1) [EASY] Docker way (directory install/docker)
+2) [EXPERTS] Debian way (directory install/debian)
+
+## Build your docker image
+``` bash
+cd /srv/gargantext/install/docker/dev
+./build
+```
+
+## Install Python environment
+Inside the docker image, execute as root:
+``` bash
+/srv/gargantext/install/python/configure
+```
+
+## Configure PostgreSql
+Inside the docker image, execute as root:
+``` bash
+/srv/gargantext/install/postgres/configure
+```
+
+## Get main librairies
+``` bash
+wget http://dl.gargantext.org/gargantext_lib.tar.bz2 \
+&& tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
+&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
+&& echo "Libs installed"
+```
 
+## Configure && Launch Gargantext
+
+### Enter docker container
+``` bash
+/srv/gargantext/install/docker/enterGargantextImage
+```
 
+### Inside docker container configure the database
+``` bash
+service postgresql start
+su gargantua
+source /srv/env_3-5/bin/activate
+python /srv/gargantext/dbmigrate.py
+/srv/gargantext/manage.py migrate
+python /srv/gargantext/dbmigrate.py
+python /srv/gargantext/dbmigrate.py
+python /srv/gargantext/init_accounts.py /srv/gargantext/install/init/account.csv
+```
+
+FIXME: dbmigrate need to launched several times since tables are
+ordered with alphabetical order (and not dependencies order)
+
+### Inside docker container launch Gargantext
+``` bash
+service postgresql start
+su gargantua
+source /srv/env_3-5/bin/activate
+/srv/gargantext/manage.py runserver 0.0.0.0:8000
+```
+
+### Outside docker container launch browser
+``` bash
+chromium http://127.0.0.1:8000/
+```
+
+Click on Test Gargantext
+Login : gargantua
+Password : autnagrag
+
+Enjoy :)
+>>>>>>> constance-docker
 

install/docker/dev/Dockerfile

@@ -6,18 +6,18 @@
 #    \____|\__,_|_|   \____|\__,_|_| |_|_|\___/_/\_\\__|  #
 #                                                         #
 ###########################################################
-
+# https://docs.docker.com/compose/django/
 ######################################################################
 FROM debian:stretch
-#jFROM gargantext
+#FROM gargantext
 MAINTAINER ISCPIF <alexandre.delanoe@iscpif.fr>
 ######################################################################
-#
+
 USER root
 
 RUN apt-get update && \
     apt-get install -y \
-    apt-utils ca-certificates locales \
+    apt-utils ca-certificates locales man dialog\
     sudo aptitude gcc g++ wget git postgresql-9.5 vim
 
 ### Configure timezone and locale
@@ -55,10 +55,10 @@ RUN apt-get install -y sudo && adduser gargantua sudo \
      && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 
-#######################################################################
-### CONFIGURE POSTGRESQL
-#######################################################################
+#####################################################################
+# CONFIGURE POSTGRESQL
+#####################################################################
 
-RUN sed -iP 's%^data_directory.*%data_directory = '\/srv\/gargantext_data'%' /etc/postgresql/9.5/main/postgresql.conf
+RUN sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargantext_data\'%" /etc/postgresql/9.5/main/postgresql.conf
 
-######################################################################
+#####################################################################

install/docker/dev/build

+#!/bin/bash
+
+#######################################################################
+#    ____             _             
+#   |  _ \  ___   ___| | _____ _ __ 
+#   | | | |/ _ \ / __| |/ / _ \ '__|
+#   | |_| | (_) | (__|   <  __/ |   
+#   |____/ \___/ \___|_|\_\___|_|   
+#                                   
+######################################################################
+
+sudo docker build -t gargantext .
+# OR
+# cd /tmp
+# wget http://dl.gargantext.org/gargantext_docker_image.tar \
+# && sudo docker import - gargantext:latest < gargantext_docker_image.tar
+
+
+
+
+
+
+

install/docker/dev/launch.sh → install/docker/enterGargantextImage

@@ -3,8 +3,6 @@
 
 sudo docker run -i -p 8000:8000 \
                    -v /srv:/srv \
-                   -v /home/alexandre:/home/alexandre \
                    -t gargantext:latest \
                    /bin/bash
 
-#sudo docker run -i --name srv -v /srv:/srv -t gargantext:latest /bin/bash

install/init/account.csv

+gargantua,contact@gargantext,autnagrag,

install/postgres/configure

+#!/bin/bash
+
+#######################################################################
+##    ____           _                      
+##   |  _ \ ___  ___| |_ __ _ _ __ ___  ___ 
+##   | |_) / _ \/ __| __/ _` | '__/ _ \/ __|
+##   |  __/ (_) \__ \ || (_| | | |  __/\__ \
+##   |_|   \___/|___/\__\__, |_|  \___||___/
+##                      |___/               
+#######################################################################
+
+su postgres -c 'pg_dropcluster 9.4 main --stop'
+rm -rf /srv/gargantext_data && mkdir /srv/gargantext_data && chown postgres:postgres /srv/gargantext_data
+su postgres -c '/usr/lib/postgresql/9.5/bin/initdb -D /srv/gargantext_data/'
+su postgres -c '/usr/lib/postgresql/9.5/bin/pg_ctl -D /srv/gargantext_data/ -l journal_applicatif start'
+
+#su postgres -c 'pg_createcluster -D /srv/gargantext_data 9.5 main '
+#su postgres -c 'pg_ctlcluster -D /srv/gargantext_data 9.5 main start '
+#su postgres -c 'pg_ctlcluster 9.5 main start'
+
+service postgresql start
+
+su postgres -c "psql -c \"CREATE user gargantua WITH PASSWORD 'C8kdcUrAQy66U'\""
+su postgres -c "createdb -O gargantua gargandb"
+
+echo "Postgres configured"
+
+

install/python/configure

+#!/bin/bash
+
+ENV="/srv/env_3-5"
+
+/usr/bin/virtualenv --py=/usr/bin/python3.5 $ENV \
+&& /bin/bash -c "source ${ENV}/bin/activate" \
+&& /bin/bash -c "${ENV}/bin/pip install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1" \
+&& /bin/bash -c "${ENV}/bin/pip install -r /srv/gargantext/install/python/requirements.txt"

install/run_gargantext.sh

-#!/bin/bash
-
-echo "Need to finish the dependencies. So soon... :)"

static/js/gargantext/NGrams_dyna_chart_and_table.js

@@ -1756,6 +1756,10 @@ function MainTableAndCharts( data , initial , search_filter) {
             }
         }
 
+    // and set this filter's initial status to 'maplist' (aka state == 1)
+    MyTable.data('dynatable').settings.dataset.queries['my_state_filter'] = 1 ;
+    MyTable.data('dynatable').process();
+
     // moves pagination over table
     if ( $(".imadiv").length>0 ) return 1;
     $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
@@ -1897,9 +1901,18 @@ $("#corpusdisplayer").hide()
 
 
 
-// NEW AJAX
+// NEW AJAX x 2
+
+var prefetch_url = window.location.origin+"/api/ngramlists/maplist?corpus="+corpus_id ;
 var new_url = window.location.origin+"/api/ngramlists/family?corpus="+corpus_id ;
-GET_(new_url, function(res) {
+
+// faster call: just the maplist, will return first
+GET_(prefetch_url, HandleAjax);
+
+// longer call (full list of terms) to return when ready and refresh all data
+GET_(new_url, HandleAjax)
+
+function HandleAjax(res) {
     if (res && res.ngraminfos) {
 
         main_ngrams_objects = {}
@@ -1955,8 +1968,7 @@ GET_(new_url, function(res) {
     $("input#groups_id").val(res.nodeids['groups'])
     $("input#scores_id").val(res.nodeids['scores'])
     AfterAjax() ;
-});
-
+}
 
 function AfterAjax() {
   // -------------------------------------------------------------------

templates/pages/corpora/terms.html

@@ -67,10 +67,11 @@
                     <div class="pull-left" style="margin-top:1.85em;">
                         Filter:
                         <select id="picklistmenu" name="my_state_filter">
-                          <option value='reset' selected="selected" >All terms</option>
-                          <option value='0'>Mainlist only</option>
-                          <option value='1'>Maplist only</option>
-                          <option value='2'>Stoplist only</option>
+                          <option value='reset'>All terms</option>
+                          <option value='0'>Candidates only</option>
+                          <!-- <option value='1' selected="selected">Map terms only</option> -->
+                          <option value='1'>Map terms only</option>
+                          <option value='2'>New stopwords only</option>
                         </select>
                     </div>
                   </h4>

templates/pages/main/about.html

@@ -39,7 +39,7 @@
                             <li>
                                 Version 3.0.0
                                 <ul>
-                                    <li>[NAME] Blue Mustard</li>
+                                    <li>[NAME] Blue Jasmine</li>
                                     <li>[CODE] Refactored</li>
                                     <li>[DATABASE] New schema</li>
                                 </ul>