Merge branch 'refactoring' of ssh://delanoe.org:1979/gargantext into refactoring

README.md

@@ -39,7 +39,7 @@ pip3.5 install -U -r requirements.txt
 ./manage.py migrate --fake-initial
 ```
 
-...or...
+...or if it fails, try the commandes below:
 
 ```bash
 ./manage.py makemigrations

gargantext/constants.py

@@ -4,25 +4,34 @@
 from gargantext.util.lists import *
 
 LISTTYPES = {
-    'DOCUMENT': WeightedList,
-    'SYNONYMS': Translations,
-    'MIAMLIST': UnweightedList,
-    'STOPLIST': UnweightedList,
+    'DOCUMENT'     : WeightedList,
+    'GROUPLIST'    : Translations,
+    'STOPLIST'     : UnweightedList,
+    'MAINLIST'     : UnweightedList,
+    'MAPLIST'      : UnweightedList,
+    'OCCURRENCES'  : WeightedList,
     'COOCCURRENCES': WeightedMatrix,
 }
 
 NODETYPES = [
     None,
-    # documents hierachy
+    # documents hierarchy
     'USER',
     'PROJECT',
     'CORPUS',
     'DOCUMENT',
     # lists
-    'SYNONYMS',
-    'MIAMLIST',
     'STOPLIST',
+    'GROUPLIST',
+    'MAINLIST',
+    'MAPLIST',
     'COOCCURRENCES',
+    # scores
+    'OCCURRENCES',
+    'SPECIFICITY',
+    'CVALUE',
+    'TFIDF-CORPUS',
+    'TFIDF-GLOBAL',
 ]
 
 import datetime
@@ -70,23 +79,23 @@ RESOURCETYPES = [
         'default_language': 'fr',
     },
     {   'name': 'Jstor (RIS format)',
-        # 'parser': RISParser,
+        'parser': RISParser,
         'default_language': 'en',
     },
     {   'name': 'Pubmed (XML format)',
         'parser': PubmedParser,
         'default_language': 'en',
     },
-        {   'name': 'Scopus (RIS format)',
-        # 'parser': RISParser,
+    {   'name': 'Scopus (RIS format)',
+        'parser': RISParser,
         'default_language': 'en',
     },
     {   'name': 'Web of Science (ISI format)',
-        # 'parser': ISIParser,
+        'parser': ISIParser,
         'default_language': 'fr',
     },
     {   'name': 'Zotero (RIS format)',
-        # 'parser': RISParser,
+        'parser': RISParser,
         'default_language': 'en',
     },
     # {   'name': 'CSV',

gargantext/models/ngrams.py

@@ -2,7 +2,7 @@ from gargantext.util.db import *
 
 from .nodes import Node
 
-__all__ = ['Ngram', 'NodeNgram', 'NodeNgramNgram']
+__all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
 
 
 class Ngram(Base):
@@ -18,6 +18,23 @@ class NodeNgram(Base):
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float)
 
+class NodeNodeNgram(Base):
+    """ for instance for tfidf:
+    (
+        doc                              ::Node ,
+        corpus                           ::Node ,
+        word                             ::Ngram ,
+        tfidf of ngram in doc in corpus  ::Float (real)
+    )
+    """
+    __tablename__ = 'nodes_nodes_ngrams'
+    node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
+    node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
+    ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
+    score = Column(Float(precision=24))
+    # précision max 24 bit pour un type sql "real" (soit 7 chiffres après virgule)
+    # sinon par défaut on aurait un type sql "double_precision" (soit 15 chiffres)
+    # (cf. www.postgresql.org/docs/9.4/static/datatype-numeric.html#DATATYPE-FLOAT)
 
 class NodeNgramNgram(Base):
     __tablename__ = 'nodes_ngrams_ngrams'

gargantext/models/users.py

@@ -55,7 +55,8 @@ class User(Base):
 
     def owns(self, node):
         """check if a given node is owned by the user"""
-        return (node.user_id == self.id) or node.id in (contact.id for contact in self.contacts())
+        return (node.user_id == self.id) or \
+                node.id in (contact.id for contact in self.contacts())
 
 
 class Contact(Base):

gargantext/util/languages.py

@@ -47,3 +47,5 @@ for pycountry_language in pycountry.languages:
 # because PubMed has weird language codes:
 languages['fre'] = languages['fr']
 languages['ger'] = languages['de']
+languages['Français'] = languages['fr']
+languages['en_US'] = languages['en']

gargantext/util/parsers/CSVParser.py

+from ._Parser import Parser
+# from ..NgramsExtractors import *
+import sys
+import csv
+csv.field_size_limit(sys.maxsize)
+import numpy as np
+import os
+
+class CSVParser(Parser):
+
+    def CSVsample( self, filename , delim) :
+        ifile  = open( filename, "r" )
+        reader = csv.reader(ifile, delimiter=delim)
+
+        Freqs = []
+        for row in reader:
+            Freqs.append(len(row))
+
+
+        ifile.close()
+        return Freqs
+
+    
+    def parse(self, filename):
+
+        sample_size = 10
+        sample_file = filename.replace(".csv","_sample.csv")
+
+        hyperdata_list = []
+
+        command_for_sample = "cat '"+filename+"' | head -n "+str(sample_size)+" > '"+sample_file+"'"
+        os.system(command_for_sample) # you just created a  *_sample.csv
+
+        # # = = = = [ Getting delimiters frequency ] = = = = #
+        PossibleDelimiters = [ ',',' ','\t', ';', '|', ':' ]
+        AllDelimiters = {}
+        for delim in PossibleDelimiters:
+            AllDelimiters[delim] = self.CSVsample( sample_file , delim ) 
+        # # = = = = [ / Getting delimiters frequency ] = = = = #
+        # # OUTPUT example:
+        # #  AllDelimiters = {
+        # #   '\t': [1, 1, 1, 1, 1],
+        # #   ' ': [1, 13, 261, 348, 330],
+        # #   ',': [15, 15, 15, 15, 15],
+        # #   ';': [1, 1, 1, 1, 1],
+        # #   '|': [1, 1, 1, 1, 1]
+        # #  }
+
+        # # = = = = [ Stand.Dev=0 & Sum of delimiters ] = = = = #
+        Delimiters = []
+        for d in AllDelimiters:
+            freqs = AllDelimiters[d]
+            suma = np.sum( freqs )
+            if suma >0:
+                std = np.std( freqs )
+                # print [ d , suma , len(freqs) , std]
+                if std == 0:
+                    Delimiters.append ( [ d , suma , len(freqs) , std] )
+        # # = = = = [ / Stand.Dev=0 & Sum of delimiters ] = = = = #
+        # # OUTPUT example:
+        # #  Delimiters = [
+        # #     ['\t', 5, 5, 0.0], 
+        # #     [',', 75, 5, 0.0], 
+        # #     ['|', 5, 5, 0.0]
+        # #  ]
+
+
+        # # = = = = [ Delimiter selection ] = = = = #
+        Sorted_Delims = sorted(Delimiters, key=lambda x: x[1], reverse=True)
+        HighestDelim = Sorted_Delims[0][0]
+        # print("selected delimiter:",[HighestDelim]
+        # print
+        # # = = = = [ / Delimiter selection ] = = = = #
+
+
+
+
+        # # = = = = [ First data coordinate ] = = = = #
+        Coords = {
+            "row": -1,
+            "column": -1
+        }
+
+        ifile  = open( sample_file, "r" )
+        reader = csv.reader(ifile, delimiter=HighestDelim)
+
+        for rownum, tokens in enumerate(reader):
+            joined_tokens = "".join (tokens)
+            if Coords["row"]<0 and len( joined_tokens )>0 :
+                Coords["row"] = rownum
+                for columnum in range(len(tokens)):
+                    t = tokens[columnum]
+                    if len(t)>0:
+                        Coords["column"] = columnum
+                        break
+        ifile.close()
+        # # = = = = [ / First data coordinate ] = = = = #
+
+
+
+        # # = = = = [ Setting Headers ] = = = = #
+        Headers_Int2Str = {}
+        ifile  = open( sample_file, "r" )
+        reader = csv.reader(ifile, delimiter=HighestDelim)
+        for rownum, tokens in enumerate(reader):
+            if rownum>=Coords["row"]:
+                for columnum in range( Coords["column"],len(tokens) ):
+                    t = tokens[columnum]
+                    Headers_Int2Str[columnum] = t
+                break
+        ifile.close()
+        # # = = = = [ / Setting Headers ] = = = = #
+        # # OUTPUT example:
+        # #  Headers_Int2Str = {
+        # #     0: 'publication_date',
+        # #      1: 'publication_month',
+        # #      2: 'publication_second',
+        # #      3: 'abstract'
+        # #  }
+
+
+
+        # # = = = = [ Reading the whole CSV and saving ] = = = = #
+        hyperdata_list = []
+        ifile  = open( filename, "r" )
+        reader = csv.reader(ifile, delimiter=HighestDelim)
+        for rownum, tokens in enumerate(reader):
+            if rownum>Coords["row"]:
+                RecordDict = {}
+                for columnum in range( Coords["column"],len(tokens) ):
+                    data = tokens[columnum]
+                    RecordDict[ Headers_Int2Str[columnum] ] = data
+                hyperdata_list.append( RecordDict )
+        ifile.close()
+        # # = = = = [ / Reading the whole CSV and saving ] = = = = #
+
+        return hyperdata_list

gargantext/util/parsers/ISTex.py

+from lxml import etree
+from ._Parser import Parser
+from datetime import datetime
+from io import BytesIO
+import json
+
+class ISTex(Parser):
+
+    def parse(self, thefile):
+        json_data=open(thefile,"r")
+        data = json.load(json_data)
+        json_data.close()
+        json_docs = data["hits"]
+        hyperdata_list = []
+        hyperdata_path = {
+            "id"                : "id",
+            "source"           : 'corpusName',
+            "title"             : 'title',
+            "genre"             : "genre",
+            "language_iso3"     : 'language',
+            "doi"               : 'doi',
+            "host"              : 'host',
+            "publication_date"  : 'publicationDate',
+            "abstract"  : 'abstract',
+            # "authors"           : 'author',
+            "authorsRAW"        : 'author',
+            "keywords"          : "keywords"
+        }
+
+        suma = 0
+
+        for json_doc in json_docs:
+
+            hyperdata = {}
+            for key, path in hyperdata_path.items():
+                try:
+                    # print(path," ==> ",len(json_doc[path]))
+                    hyperdata[key] = json_doc[path]
+                except:
+                    pass
+
+            # print("|",hyperdata["language_iso3"])
+
+            if "doi" in hyperdata:
+                hyperdata["doi"] = hyperdata["doi"][0]
+
+            keywords = []
+            if "keywords" in hyperdata:
+                for keyw in hyperdata["keywords"]:
+                    keywords.append(keyw["value"] )
+                hyperdata["keywords"] = ", ".join( keywords )
+
+            moredate=False
+            moresource=False
+            if "host" in hyperdata:
+
+                if "genre" in hyperdata["host"] and len(hyperdata["host"]["genre"])>0:
+                    if "genre" in hyperdata and len(hyperdata["genre"])==0:
+                        hyperdata["genre"] = hyperdata["host"]["genre"]
+
+                # print(hyperdata["host"])
+                if "pubdate" in hyperdata["host"]:
+                    onebuffer = hyperdata["publication_date"]
+                    hyperdata["publication_date"] = []
+                    hyperdata["publication_date"].append(onebuffer)
+                    hyperdata["publication_date"].append( hyperdata["host"]["pubdate"] )
+
+                if "title" in hyperdata["host"]:
+                    hyperdata["journal"] = hyperdata["host"]["title"]
+
+            authors=False
+            if "authorsRAW" in hyperdata:
+                names = []
+                for author in hyperdata["authorsRAW"]:
+                    names.append(author["name"])
+                hyperdata["authors"] = ", ".join(names)
+
+            if "host" in hyperdata: hyperdata.pop("host")
+            if "genre" in hyperdata:
+                if len(hyperdata["genre"])==0:
+                    hyperdata.pop("genre")
+            if "language_iso3" in hyperdata:
+                # retrieve lang if lang != [] and lang != ["unknown"]
+                # ---------------------------------------------------
+                if len(hyperdata["language_iso3"])>0 and hyperdata["language_iso3"][0] != "unknown" :
+                    hyperdata["language_iso3"] = hyperdata["language_iso3"][0]
+                
+                # default value = eng
+                # possible even better: langid.classify(abstract)
+                else:
+                    # NB 97% des docs istex sont eng donc par défaut
+                    # ----------------------------------------------
+                    hyperdata["language_iso3"] = "eng"
+                    # (cf. api.istex.fr/document/?q=*&facet=language 
+                    #  et  tests langid sur les language=["unknown"])
+                    
+
+            if "publication_date" in hyperdata:
+                RealDate = hyperdata["publication_date"]
+                if "publication_date" in hyperdata:
+                    hyperdata.pop("publication_date")
+
+                if isinstance(RealDate, list):
+                    RealDate = RealDate[0]
+
+                # print( RealDate ," | length:",len(RealDate))
+                Decision=""
+                if len(RealDate)>4:
+                    if len(RealDate)>8:
+                        try: Decision = datetime.strptime(RealDate, '%Y-%b-%d').date()
+                        except:
+                            try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
+                            except: Decision=False
+                    else:
+                        try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
+                        except:
+                            try: Decision = datetime.strptime(RealDate, '%Y-%m').date()
+                            except: Decision=False
+                else:
+                    try: Decision = datetime.strptime(RealDate, '%Y').date()
+                    except: Decision=False
+
+                if Decision!=False:
+                    hyperdata["publication_year"] = str(Decision.year)
+                    hyperdata["publication_month"] = str(Decision.month)
+                    hyperdata["publication_day"] = str(Decision.day)
+                    hyperdata_list.append(hyperdata)
+                    # print("\t||",hyperdata["title"])
+                    # print("\t\t",Decision)
+                    # print("=============================")
+                # else:
+                #     suma+=1
+                #     if "pubdate" in json_doc:
+                #         print ("\tfail pubdate:",json_doc["pubdate"])
+
+
+        # print ("nb_hits:",len(json_docs))
+        # print("\t - nb_fails:",suma)
+        # print("  -- - - - - - -- - -")
+
+        return hyperdata_list

gargantext/util/parsers/Isi.py

+from .Ris import RISParser
+
+
+class ISIParser(RISParser):
+    
+        _begin = 3
+        
+        _parameters = {
+            b"ER":  {"type": "delimiter"},
+            b"TI":  {"type": "hyperdata", "key": "title", "separator": " "},
+            b"AU":  {"type": "hyperdata", "key": "authors", "separator": ", "},
+            b"DI":  {"type": "hyperdata", "key": "doi"},
+            b"SO":  {"type": "hyperdata", "key": "journal"},
+            b"PY":  {"type": "hyperdata", "key": "publication_year"},
+            b"PD":  {"type": "hyperdata", "key": "publication_month"},
+            b"LA":  {"type": "hyperdata", "key": "language_fullname"},
+            b"AB":  {"type": "hyperdata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "hyperdata", "key": "fields"},
+        }
+

gargantext/util/parsers/Ris.py

+from ._Parser import Parser
+
+from gargantext.util.languages import languages
+
+#from admin.utils import PrintException
+
+class RISParser(Parser):
+
+#    def __init__(self, language_cache=None):
+#
+#        #super(Parser, self).__init__()
+#        #super(Parser, self).__init__()
+#        self._languages_cache = LanguagesCache() if language_cache is None else language_cache
+    
+
+    _begin = 6
+    _parameters = {
+        b"ER":  {"type": "delimiter"},
+        b"TI":  {"type": "hyperdata", "key": "title", "separator": " "},
+        b"ST":  {"type": "hyperdata", "key": "subtitle", "separator": " "},
+        b"AU":  {"type": "hyperdata", "key": "authors", "separator": ", "},
+        b"T2":  {"type": "hyperdata", "key": "journal"},
+        b"UR":  {"type": "hyperdata", "key": "doi"},
+        b"PY":  {"type": "hyperdata", "key": "publication_year"},
+        b"PD":  {"type": "hyperdata", "key": "publication_month"},
+        b"LA":  {"type": "hyperdata", "key": "language_iso2"},
+        b"AB":  {"type": "hyperdata", "key": "abstract", "separator": " "},
+        b"WC":  {"type": "hyperdata", "key": "fields"},
+    }
+
+    def parse(self, file):
+        
+        hyperdata = {}
+        last_key = None
+        last_values = []
+        # browse every line of the file
+        for line in file:
+            if len(line) > 2:
+                # extract the parameter key
+                parameter_key = line[:2]
+                if parameter_key != b'  ' and parameter_key != last_key:
+                    if last_key in self._parameters:
+                        # translate the parameter key
+                        parameter = self._parameters[last_key]
+                        if parameter["type"] == "hyperdata":
+                            separator = parameter["separator"] if "separator" in parameter else ""
+                            hyperdata[parameter["key"]] = separator.join(last_values)
+                        elif parameter["type"] == "delimiter":
+                            if 'language_fullname' not in hyperdata.keys():
+                                if 'language_iso3' not in hyperdata.keys():
+                                    if 'language_iso2' not in hyperdata.keys():
+                                        hyperdata['language_iso2'] = 'en'
+                            yield hyperdata
+                            hyperdata = {}
+                    last_key = parameter_key
+                    last_values = []
+                try:
+                    last_values.append(line[self._begin:-1].decode())
+                except Exception as error:
+                    print(error)
+        # if a hyperdata object is left in memory, yield it as well
+        if hyperdata:
+#            try:
+#                if hyperdata['date_to_parse']:
+#                    print(hyperdata['date_to_parse'])
+#            except:
+#                pass
+#
+            #print(hyperdata['title'])
+            yield hyperdata

gargantext/util/parsers/__init__.py

-# from .Ris import RisParser
-# from .Isi import IsiParser
+from .Ris import RISParser
+from .Isi import ISIParser
 # from .Jstor import JstorParser
 # from .Zotero import ZoteroParser
 from .Pubmed import PubmedParser
+
 # # 2015-12-08: parser 2 en 1
 from .Europress import EuropressParser
+
 # from .ISTex import ISTexParser
 # from .CSV import CSVParser

gargantext/views/api/nodes.py

@@ -6,6 +6,7 @@ from gargantext.constants import *
 
 from gargantext.util.validation import validate
 
+from collections import defaultdict
 
 _node_available_fields = ['id', 'parent_id', 'name', 'typename', 'hyperdata', 'ngrams']
 _node_default_fields = ['id', 'parent_id', 'name', 'typename']
@@ -13,7 +14,7 @@ _node_available_types = NODETYPES
 
 
 def _query_nodes(request, node_id=None):
-    user = cache.User[request.user.username]
+    user = cache.User[request.user.id]
     # parameters validation
     parameters = get_parameters(request)
     parameters = validate(parameters, {'type': dict, 'items': {
@@ -109,3 +110,70 @@ class NodeResource(APIView):
         )
         session.commit()
         return JsonHttpResponse({'deleted': result.rowcount})
+
+
+class CorpusFacet(APIView):
+    """Loop through a corpus node's docs => do counts by a hyperdata field
+        (url: /api/nodes/<node_id>/facets?hyperfield=<journal>)
+    """
+    # - old url: '^project/(\d+)/corpus/(\d+)/journals/journals.json$',
+    # - old view: tests.ngramstable.views.get_journals_json()
+    # - now generalized for various hyperdata field:
+    #    -> journal
+    #    -> publication_year
+    #    -> rubrique
+    #    -> language...
+
+    def get(self, request, node_id):
+        # check that the node is a corpus
+        #   ? faster from cache than: corpus = session.query(Node)...
+        corpus = cache.Node[node_id]
+        if corpus.typename != 'CORPUS':
+            raise ValidationException(
+                "Only nodes of type CORPUS can accept facet queries" +
+                " (but this node has type %s)..." % corpus.typename
+            )
+        else:
+            self.corpus = corpus
+
+        # check that the hyperfield parameter makes sense
+        _facet_available_subfields = [
+            'journal', 'publication_year', 'rubrique',
+            'language_iso2', 'language_iso3', 'language_name'
+        ]
+        parameters = get_parameters(request)
+
+        # validate() triggers an info message if subfield not in range
+        parameters = validate(parameters, {'type': dict, 'items': {
+            'hyperfield': {'type': str, 'range': _facet_available_subfields}
+            }})
+
+        subfield = parameters['hyperfield']
+
+        # do the aggregated sum
+        (xcounts, total) = self._ndocs_by_facet(subfield)
+
+        # response
+        return JsonHttpResponse({
+            'doc_count' : total,
+            'by': { subfield: xcounts }
+        })
+
+    def _ndocs_by_facet(self, subfield='journal'):
+        """for example on 'journal'
+         xcounts = {'j good sci' : 25, 'nature' : 32, 'j bla bla' : 1... }"""
+
+        xcounts = defaultdict(int)
+        total = 0
+        for doc in self.corpus.children(typename='DOCUMENT'):
+            if subfield in doc.hyperdata:
+                xcounts[doc.hyperdata[subfield]] += 1
+            else:
+                xcounts["_NA_"] += 1
+
+            total += 1
+
+        # the counts below could also be memoized
+        #  // if subfield not in corpus.aggs:
+        #  //     corpus.aggs[subfield] = xcounts
+        return (xcounts, total)

gargantext/views/api/urls.py

@@ -6,4 +6,6 @@ from . import nodes
 urlpatterns = [
     url(r'^nodes$', nodes.NodeListResource.as_view()),
     url(r'^nodes/(\d+)$', nodes.NodeResource.as_view()),
+
+    url(r'^nodes/(\d+)/facets$', nodes.CorpusFacet.as_view()),
 ]

gargantext/views/pages/corpora.py

@@ -12,19 +12,22 @@ def _get_user_project_corpus(request, project_id, corpus_id):
     """Helper method to get a corpus, knowing the project's and corpus' ID.
     Raises HTTP errors when parameters (user, IDs...) are invalid.
     """
-    user = cache.User[request.user.username]
+    user = cache.User[request.user.id]
     project = session.query(Node).filter(Node.id == project_id).first()
     corpus = session.query(Node).filter(Node.id == corpus_id).filter(Node.parent_id == project_id).first()
     if corpus is None:
         raise Http404()
     if not user.owns(corpus):
-        raise HttpResponseForbidden()
-    return user, project, corpus
+        print("CORPORA: invalid user %i (User doesn't own this corpus)" % user.id)
+        return (False, user, project, corpus)
+    return (True, user, project, corpus)
 
 
 @requires_auth
 def corpus(request, project_id, corpus_id):
-    user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)
+    authorized, user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)
+    if not authorized:
+        return HttpResponseForbidden()
     # response!
     return render(
         template_name = 'pages/corpora/corpus.html',
@@ -43,4 +46,31 @@ def corpus(request, project_id, corpus_id):
 
 @requires_auth
 def chart(request, project_id, corpus_id):
-    user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)
+    authorized, user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)
+
+
+@requires_auth
+def docs_by_journals(request, project_id, corpus_id):
+    '''
+    Browse journal titles for a given corpus
+    NB: javascript in page will GET counts from our api: facets?subfield=journal
+    # TODO refactor Journals_dyna_charts_and_table.js
+    '''
+    # we pass our corpus to mark it's a corpora page
+    corpus = cache.Node[corpus_id]
+
+    # and the project just for project.id in corpusBannerTop
+    project = cache.Node[project_id]
+
+    # rendered page : journals.html
+    return render(
+        template_name = 'pages/corpora/journals.html',
+        request = request,
+        context = {
+            'debug': settings.DEBUG,
+            'date': datetime.now(),
+            'project': project,
+            'corpus' : corpus,
+            'view': 'journals'
+        },
+    )

gargantext/views/pages/projects.py

@@ -20,7 +20,7 @@ def overview(request):
     To each project, we can link a resource that can be an image.
     '''
 
-    user = cache.User[request.user.username]
+    user = cache.User[request.user.id]
 
     # If POST method, creates a new project...
     if request.method == 'POST':
@@ -74,7 +74,7 @@ class NewCorpusForm(forms.Form):
 @requires_auth
 def project(request, project_id):
     # current user
-    user = cache.User[request.user.username]
+    user = cache.User[request.user.id]
     # viewed project
     project = session.query(Node).filter(Node.id == project_id).first()
     if project is None:

gargantext/views/pages/urls.py

@@ -20,7 +20,9 @@ urlpatterns = [
     url(r'^projects/(\d+)/?$', projects.project),
 
     # corpora
-    url(r'^projects/(\d+)/corpora/(\d+)?$', corpora.corpus),
-    url(r'^projects/(\d+)/corpora/(\d+)/chart?$', corpora.chart),
+    url(r'^projects/(\d+)/corpora/(\d+)/?$', corpora.corpus),
+    url(r'^projects/(\d+)/corpora/(\d+)/chart/?$', corpora.chart),
 
+    # corpus by journals
+    url(r'^projects/(\d+)/corpora/(\d+)/journals/?$', corpora.docs_by_journals),
 ]

requirements.txt

 amqp==1.4.9
 anyjson==0.3.3
-billiard==3.3.0.22
+billiard==3.3.0.22             # multiprocessing fork
 celery==3.1.20
+chardet==2.3.0
 dateparser==0.3.2
 Django==1.9.2
 django-celery==3.1.17
@@ -10,18 +11,18 @@ django-pgjsonb==0.0.16
 djangorestframework==3.3.2
 html5lib==0.9999999
 jdatetime==1.7.2
-kombu==3.0.33
+kombu==3.0.33                  # messaging
 lxml==3.5.0
 nltk==3.1
 numpy==1.10.4
 psycopg2==2.6.1
 pycountry==1.20
 python-dateutil==2.4.2
-pytz==2015.7
+pytz==2015.7                   # timezones
 PyYAML==3.11
 RandomWords==0.1.12
 six==1.10.0
 SQLAlchemy==1.1.0b1.dev0
 ujson==1.35
-umalqurra==0.2
+umalqurra==0.2                 # arabic calendars (?? why use ??)
 wheel==0.29.0

static/css/gargantext/tables.css

+// styles for dynatables
+
+.no-transition {
+  -webkit-transition: height 0.1s;
+  -moz-transition: height 0.1s;
+  -ms-transition: height 0.1s;
+  -o-transition: height 0.1s;
+  transition: height 0.1s;
+}
+th { color: #fff; }
+
+th a {
+  color: #fff;
+  font-weight: bold;
+  font-size: 0.9em;
+}
+
+.dynatable-record-count {
+  font-size: 0.7em;
+}
+.dynatable-pagination-links {
+  font-size: 0.7em;
+}
+
+#corpusdisplayer {
+    width:200px;
+    margin:0 auto;
+    display:block;
+}

static/docs/gargantua_book/Source.txt

+Project Gutenberg's Gargantua and Pantagruel, Complete., by Francois Rabelais
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever.  You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.net
+
+
+Title: Gargantua and Pantagruel, Complete.
+       Five Books Of The Lives, Heroic Deeds And Sayings Of Gargantua And
+       His Son Pantagruel
+       
+
+Author: Francois Rabelais
+
+Release Date: August 8, 2004 [EBook #1200]
+
+Language: English
+
+
+*** START OF THIS PROJECT GUTENBERG EBOOK GARGANTUA AND PANTAGRUEL, ***
+
+
+
+
+Produced by Sue Asscher and David Widger
+
+
+
+
+
+MASTER FRANCIS RABELAIS
+
+
+FIVE BOOKS OF THE LIVES, HEROIC DEEDS AND SAYINGS OF
+
+GARGANTUA AND HIS SON PANTAGRUEL
+
+
+
+Translated into English by
+
+Sir Thomas Urquhart of Cromarty
+
+and
+
+Peter Antony Motteux
+
+
+
+
+The text of the first Two Books of Rabelais has been reprinted from the
+first edition (1653) of Urquhart's translation.  Footnotes initialled 'M.'
+are drawn from the Maitland Club edition (1838); other footnotes are by the
+translator.  Urquhart's translation of Book III. appeared posthumously in
+1693, with a new edition of Books I. and II., under Motteux's editorship.
+Motteux's rendering of Books IV. and V. followed in 1708.  Occasionally (as
+the footnotes indicate) passages omitted by Motteux have been restored from
+the 1738 copy edited by Ozell.
+
+
+

static/docs/gargantua_book/gargantua_chapter_1.txt

+Chapter 1.I. Of the Genealogy and Antiquity of Gargantua.
+Chapter 1.II. The Antidoted Fanfreluches:  or, a Galimatia of extravagant Conceits found in an ancient Monument.
+Chapter 1.III. How Gargantua was carried eleven months in his mother's belly.
+Chapter 1.IV. How Gargamelle, being great with Gargantua, did eat a huge deal of tripes.
+Chapter 1.IX. The colours and liveries of Gargantua.
+Chapter 1.L. Gargantua's speech to the vanquished.
+Chapter 1.LI. How the victorious Gargantuists were recompensed after the battle.
+Chapter 1.LII. How Gargantua caused to be built for the Monk the Abbey of Theleme.
+Chapter 1.LIII. How the abbey of the Thelemites was built and endowed.
+Chapter 1.LIV. The inscription set upon the great gate of Theleme.
+Chapter 1.LV. What manner of dwelling the Thelemites had.
+Chapter 1.LVI. How the men and women of the religious order of Theleme were apparelled.
+Chapter 1.LVII. How the Thelemites were governed, and of their manner of living.
+Chapter 1.LVIII. A prophetical Riddle.
+Chapter 1.V. The Discourse of the Drinkers.
+Chapter 1.VI. How Gargantua was born in a strange manner.
+Chapter 1.VII. After what manner Gargantua had his name given him, and how he tippled, bibbed, and curried the can.
+Chapter 1.VIII. How they apparelled Gargantua.
+Chapter 1.X. Of that which is signified by the colours white and blue.
+Chapter 1.XI. Of the youthful age of Gargantua.
+Chapter 1.XII. Of Gargantua's wooden horses.
+Chapter 1.XIII. How Gargantua's wonderful understanding became known to his father Grangousier, by the invention of a torchecul or wipebreech.
+Chapter 1.XIV. How Gargantua was taught Latin by a Sophister.
+Chapter 1.XIX. The oration of Master Janotus de Bragmardo for recovery of the bells.
+Chapter 1.XL. Why monks are the outcasts of the world; and wherefore some have bigger noses than others.
+Chapter 1.XLI. How the Monk made Gargantua sleep, and of his hours and breviaries.
+Chapter 1.XLII. How the Monk encouraged his fellow-champions, and how he hanged upon a tree.
+Chapter 1.XLIII. How the scouts and fore-party of Picrochole were met with by Gargantua, and how the Monk slew Captain Drawforth, and then was taken prisoner by his enemies.
+Chapter 1.XLIV. How the Monk rid himself of his keepers, and how Picrochole's forlorn hope was defeated.
+Chapter 1.XLIX. How Picrochole in his flight fell into great misfortunes, and what Gargantua did after the battle.
+Chapter 1.XLV. How the Monk carried along with him the Pilgrims, and of the good words that Grangousier gave them.
+Chapter 1.XLVI. How Grangousier did very kindly entertain Touchfaucet his prisoner.
+Chapter 1.XLVII. How Grangousier sent for his legions, and how Touchfaucet slew Rashcalf, and was afterwards executed by the command of Picrochole.
+Chapter 1.XLVIII. How Gargantua set upon Picrochole within the rock Clermond, and utterly defeated the army of the said Picrochole.
+Chapter 1.XV. How Gargantua was put under other schoolmasters.
+Chapter 1.XVI. How Gargantua was sent to Paris, and of the huge great mare that he rode on; how she destroyed the oxflies of the Beauce.
+Chapter 1.XVII. How Gargantua paid his welcome to the Parisians, and how he took away the great bells of Our Lady's Church.
+Chapter 1.XVIII. How Janotus de Bragmardo was sent to Gargantua to recover the great bells.
+Chapter 1.XX. How the Sophister carried away his cloth, and how he had a suit in law against the other masters.
+Chapter 1.XXI. The study of Gargantua, according to the discipline of his schoolmasters the Sophisters.
+Chapter 1.XXII. The games of Gargantua.
+Chapter 1.XXIII. How Gargantua was instructed by Ponocrates, and in such sort disciplinated, that he lost not one hour of the day.
+Chapter 1.XXIV. How Gargantua spent his time in rainy weather.
+Chapter 1.XXIX. The tenour of the letter which Grangousier wrote to his son Gargantua.
+Chapter 1.XXV. How there was great strife and debate raised betwixt the cake-bakers of Lerne, and those of Gargantua's country, whereupon were waged great wars.
+Chapter 1.XXVI. How the inhabitants of Lerne, by the commandment of Picrochole their king, assaulted the shepherds of Gargantua unexpectedly and on a sudden.
+Chapter 1.XXVII. How a monk of Seville saved the close of the abbey from being ransacked by the enemy.
+Chapter 1.XXVIII. How Picrochole stormed and took by assault the rock Clermond, and of Grangousier's unwillingness and aversion from the undertaking of war.
+Chapter 1.XXX. How Ulric Gallet was sent unto Picrochole.
+Chapter 1.XXXI. The speech made by Gallet to Picrochole.
+Chapter 1.XXXII. How Grangousier, to buy peace, caused the cakes to be restored.
+Chapter 1.XXXIII. How some statesmen of Picrochole, by hairbrained counsel, put him in extreme danger.
+Chapter 1.XXXIV. How Gargantua left the city of Paris to succour his country, and how Gymnast encountered with the enemy.
+Chapter 1.XXXIX. How the Monk was feasted by Gargantua, and of the jovial discourse they had at supper.
+Chapter 1.XXXV. How Gymnast very souply and cunningly killed Captain Tripet and others of Picrochole's men.
+Chapter 1.XXXVI. How Gargantua demolished the castle at the ford of Vede, and how they passed the ford.
+Chapter 1.XXXVII. How Gargantua, in combing his head, made the great cannon-balls fall out of his hair.
+Chapter 1.XXXVIII. How Gargantua did eat up six pilgrims in a salad.

static/docs/gargantua_book/gargantua_chapter_2.txt

+Chapter 2.I. Of the original and antiquity of the great Pantagruel.
+Chapter 2.II. Of the nativity of the most dread and redoubted Pantagruel.
+Chapter 2.III. Of the grief wherewith Gargantua was moved at the decease of his wife Badebec.
+Chapter 2.IV. Of the infancy of Pantagruel.
+Chapter 2.IX. How Pantagruel found Panurge, whom he loved all his lifetime.
+Chapter 2.V. Of the acts of the noble Pantagruel in his youthful age.
+Chapter 2.VI. How Pantagruel met with a Limousin, who too affectedly did counterfeit the French language.
+Chapter 2.VII. How Pantagruel came to Paris, and of the choice books of the Library of St. Victor.
+Chapter 2.VIII. How Pantagruel, being at Paris, received letters from his father Gargantua, and the copy of them.
+Chapter 2.X. How Pantagruel judged so equitably of a controversy, which was wonderfully obscure and difficult, that, by reason of his just decree therein, he was reputed to have a most admirable judgment.
+Chapter 2.XI. How the Lords of Kissbreech and Suckfist did plead before Pantagruel without an attorney.
+Chapter 2.XII. How the Lord of Suckfist pleaded before Pantagruel.
+Chapter 2.XIII. How Pantagruel gave judgment upon the difference of the two lords.
+Chapter 2.XIV. How Panurge related the manner how he escaped out of the hands of the Turks.
+Chapter 2.XIX. How Panurge put to a nonplus the Englishman that argued by signs.
+Chapter 2.XV. How Panurge showed a very new way to build the walls of Paris.
+Chapter 2.XVI. Of the qualities and conditions of Panurge.
+Chapter 2.XVII. How Panurge gained the pardons, and married the old women, and of the suit in law which he had at Paris.
+Chapter 2.XVIII. How a great scholar of England would have argued against Pantagruel, and was overcome by Panurge.
+Chapter 2.XX. How Thaumast relateth the virtues and knowledge of Panurge.
+Chapter 2.XXI. How Panurge was in love with a lady of Paris.
+Chapter 2.XXII. How Panurge served a Parisian lady a trick that pleased her not very well.
+Chapter 2.XXIII. How Pantagruel departed from Paris, hearing news that the Dipsodes had invaded the land of the Amaurots; and the cause wherefore the leagues are so short in France.
+Chapter 2.XXIV. A letter which a messenger brought to Pantagruel from a lady of Paris, together with the exposition of a posy written in a gold ring.
+Chapter 2.XXIX. How Pantagruel discomfited the three hundred giants armed.
+Chapter 2.XXV. How Panurge, Carpalin, Eusthenes, and Epistemon, the gentlemen attendants of Pantagruel, vanquished and discomfited six hundred and threescore horsemen very cunningly.
+Chapter 2.XXVI. How Pantagruel and his company were weary in eating still salt meats; and how Carpalin went a-hunting to have some venison.
+Chapter 2.XXVII. How Pantagruel set up one trophy in memorial of their valour, and Panurge another in remembrance of the hares.  How Pantagruel likewise with his farts begat little men, and with his fisgs little women; and how Panurge broke a great staff over two glasses.
+Chapter 2.XXVIII. How Pantagruel got the victory very strangely over the Dipsodes and the Giants.
+Chapter 2.XXX. How Epistemon, who had his head cut off, was finely healed by Panurge, and of the news which he brought from the devils, and of the damned people in hell.
+Chapter 2.XXXI. How Pantagruel entered into the city of the Amaurots, and how Panurge married King Anarchus to an old lantern-carrying hag, and made him a crier of green sauce.
+Chapter 2.XXXII. How Pantagruel with his tongue covered a whole army, and what the author saw in his mouth.
+Chapter 2.XXXIII. How Pantagruel became sick, and the manner how he was recovered.
+Chapter 2.XXXIV. The conclusion of this present book, and the excuse of the author.

static/docs/gargantua_book/gargantua_chapter_3.txt

+Chapter 3.I. How Pantagruel transported a colony of Utopians into Dipsody.
+Chapter 3.II. How Panurge was made Laird of Salmigondin in Dipsody, and did waste his revenue before it came in.
+Chapter 3.III. How Panurge praiseth the debtors and borrowers.
+Chapter 3.IV. Panurge continueth his discourse in the praise of borrowers and lenders.
+Chapter 3.IX. How Panurge asketh counsel of Pantagruel whether he should marry, yea, or no.
+Chapter 3.L. How the famous Pantagruelion ought to be prepared and wrought.
+Chapter 3.LI. Why it is called Pantagruelion, and of the admirable virtues.
+Chapter 3.LII. How a certain kind of Pantagruelion is of that nature that the fire is not able to consume it.
+Chapter 3.V. How Pantagruel altogether abhorreth the debtors and borrowers.
+Chapter 3.VI. Why new married men were privileged from going to the wars.
+Chapter 3.VII. How Panurge had a flea in his ear, and forbore to wear any longer his magnificent codpiece.
+Chapter 3.VIII. Why the codpiece is held to be the chief piece of armour amongst warriors.
+Chapter 3.X. How Pantagruel representeth unto Panurge the difficulty of giving advice in the matter of marriage; and to that purpose mentioneth somewhat of the Homeric and Virgilian lotteries.
+Chapter 3.XI. How Pantagruel showeth the trial of one's fortune by the throwing of dice to be unlawful.
+Chapter 3.XII. How Pantagruel doth explore by the Virgilian lottery what fortune Panurge shall have in his marriage.
+Chapter 3.XIII. How Pantagruel adviseth Panurge to try the future good or bad luck of his marriage by dreams.
+Chapter 3.XIV. Panurge's dream, with the interpretation thereof.
+Chapter 3.XIX. How Pantagruel praiseth the counsel of dumb men.
+Chapter 3.XL. How Bridlegoose giveth reasons why he looked upon those law- actions which he decided by the chance of the dice.
+Chapter 3.XLI. How Bridlegoose relateth the history of the reconcilers of parties at variance in matters of law.
+Chapter 3.XLII. How suits at law are bred at first, and how they come afterwards to their perfect growth.
+Chapter 3.XLIII. How Pantagruel excuseth Bridlegoose in the matter of sentencing actions at law by the chance of the dice.
+Chapter 3.XLIV. How Pantagruel relateth a strange history of the perplexity of human judgment.
+Chapter 3.XLIX. How Pantagruel did put himself in a readiness to go to sea; and of the herb named Pantagruelion.
+Chapter 3.XLV. How Panurge taketh advice of Triboulet.
+Chapter 3.XLVI. How Pantagruel and Panurge diversely interpret the words of Triboulet.
+Chapter 3.XLVII. How Pantagruel and Panurge resolved to make a visit to the Oracle of the Holy Bottle.
+Chapter 3.XLVIII. How Gargantua showeth that the children ought not to marry without the special knowledge and advice of their fathers and mothers.
+Chapter 3.XV. Panurge's excuse and exposition of the monastic mystery concerning powdered beef.
+Chapter 3.XVI. How Pantagruel adviseth Panurge to consult with the Sibyl of Panzoust.
+Chapter 3.XVII. How Panurge spoke to the Sibyl of Panzoust.
+Chapter 3.XVIII. How Pantagruel and Panurge did diversely expound the verses of the Sibyl of Panzoust.
+Chapter 3.XX. How Goatsnose by signs maketh answer to Panurge.
+Chapter 3.XXI. How Panurge consulteth with an old French poet, named Raminagrobis.
+Chapter 3.XXII. How Panurge patrocinates and defendeth the Order of the Begging Friars.
+Chapter 3.XXIII. How Panurge maketh the motion of a return to Raminagrobis.
+Chapter 3.XXIV. How Panurge consulteth with Epistemon.
+Chapter 3.XXIX. How Pantagruel convocated together a theologian, physician, lawyer, and philosopher, for extricating Panurge out of the perplexity wherein he was.
+Chapter 3.XXV. How Panurge consulteth with Herr Trippa.
+Chapter 3.XXVI. How Panurge consulteth with Friar John of the Funnels.
+Chapter 3.XXVII. How Friar John merrily and sportingly counselleth Panurge.
+Chapter 3.XXVIII. How Friar John comforteth Panurge in the doubtful matter of cuckoldry.
+Chapter 3.XXX. How the theologue, Hippothadee, giveth counsel to Panurge in the matter and business of his nuptial enterprise.
+Chapter 3.XXXI. How the physician Rondibilis counselleth Panurge.
+Chapter 3.XXXII. How Rondibilis declareth cuckoldry to be naturally one of the appendances of marriage.
+Chapter 3.XXXIII. Rondibilis the physician's cure of cuckoldry.
+Chapter 3.XXXIV. How women ordinarily have the greatest longing after things prohibited.
+Chapter 3.XXXIX. How Pantagruel was present at the trial of Judge Bridlegoose, who decided causes and controversies in law by the chance and fortune of the dice.
+Chapter 3.XXXV. How the philosopher Trouillogan handleth the difficulty of marriage.
+Chapter 3.XXXVI. A continuation of the answer of the Ephectic and Pyrrhonian philosopher Trouillogan.
+Chapter 3.XXXVII. How Pantagruel persuaded Panurge to take counsel of a fool.
+Chapter 3.XXXVIII. How Triboulet is set forth and blazed by Pantagruel and Panurge.

static/docs/gargantua_book/gargantua_chapter_4.txt

+Chapter 4.I. How Pantagruel went to sea to visit the oracle of Bacbuc, alias the Holy Bottle.
+Chapter 4.II. How Pantagruel bought many rarities in the island of Medamothy.
+Chapter 4.III. How Pantagruel received a letter from his father Gargantua, and of the strange way to have speedy news from far distant places.
+Chapter 4.IV. How Pantagruel writ to his father Gargantua, and sent him several curiosities.
+Chapter 4.IX. How Pantagruel arrived at the island of Ennasin, and of the strange ways of being akin in that country.
+Chapter 4.L. How Homenas showed us the archetype, or representation of a pope.
+Chapter 4.LI. Table-talk in praise of the decretals.
+Chapter 4.LII. A continuation of the miracles caused by the decretals.
+Chapter 4.LIII. How, by the virtue of the decretals, gold is subtilely drawn out of France to Rome.
+Chapter 4.LIV. How Homenas gave Pantagruel some bon-Christian pears.
+Chapter 4.LIX. Of the ridiculous statue Manduce; and how and what the Gastrolaters sacrifice to their ventripotent god.
+Chapter 4.LV. How Pantagruel, being at sea, heard various unfrozen words.
+Chapter 4.LVI. How among the frozen words Pantagruel found some odd ones.
+Chapter 4.LVII. How Pantagruel went ashore at the dwelling of Gaster, the first master of arts in the world.
+Chapter 4.LVIII. How, at the court of the master of ingenuity, Pantagruel detested the Engastrimythes and the Gastrolaters.
+Chapter 4.LX. What the Gastrolaters sacrificed to their god on interlarded fish-days.
+Chapter 4.LXI. How Gaster invented means to get and preserve corn.
+Chapter 4.LXII. How Gaster invented an art to avoid being hurt or touched by cannon-balls.
+Chapter 4.LXIII. How Pantagruel fell asleep near the island of Chaneph, and of the problems proposed to be solved when he waked.
+Chapter 4.LXIV. How Pantagruel gave no answer to the problems.
+Chapter 4.LXV. How Pantagruel passed the time with his servants.
+Chapter 4.LXVI. How, by Pantagruel's order, the Muses were saluted near the isle of Ganabim.
+Chapter 4.LXVII. How Panurge berayed himself for fear; and of the huge cat Rodilardus, which he took for a puny devil.
+Chapter 4.V. How Pantagruel met a ship with passengers returning from Lantern-land.
+Chapter 4.VI. How, the fray being over, Panurge cheapened one of Dingdong's sheep.
+Chapter 4.VII. Which if you read you'll find how Panurge bargained with Dingdong.
+Chapter 4.VIII. How Panurge caused Dingdong and his sheep to be drowned in the sea.
+Chapter 4.X. How Pantagruel went ashore at the island of Chely, where he saw King St. Panigon.
+Chapter 4.XI. Why monks love to be in kitchens.
+Chapter 4.XII. How Pantagruel passed by the land of Pettifogging, and of the strange way of living among the Catchpoles.
+Chapter 4.XIII. How, like Master Francis Villon, the Lord of Basche commended his servants.
+Chapter 4.XIV. A further account of catchpoles who were drubbed at Basche's house.
+Chapter 4.XIX. What countenances Panurge and Friar John kept during the.
+Chapter 4.XL. How Friar John fitted up the sow; and of the valiant cooks that went into it.
+Chapter 4.XLI. How Pantagruel broke the Chitterlings at the knees.
+Chapter 4.XLII. How Pantagruel held a treaty with Niphleseth, Queen of the Chitterlings.
+Chapter 4.XLIII. How Pantagruel went into the island of Ruach.
+Chapter 4.XLIV. How small rain lays a high wind.
+Chapter 4.XLIX. How Homenas, Bishop of Papimany, showed us the Uranopet decretals .
+Chapter 4.XLV. How Pantagruel went ashore in the island of Pope-Figland.
+Chapter 4.XLVI. How a junior devil was fooled by a husbandman of Pope- Figland.
+Chapter 4.XLVII. How the devil was deceived by an old woman of Pope- Figland.
+Chapter 4.XLVIII. How Pantagruel went ashore at the island of Papimany.
+Chapter 4.XV. How the ancient custom at nuptials is renewed by the catchpole.
+Chapter 4.XVI. How Friar John made trial of the nature of the catchpoles.
+Chapter 4.XVII. How Pantagruel came to the islands of Tohu and Bohu; and of the strange death of Wide-nostrils, the swallower of windmills.
+Chapter 4.XVIII. How Pantagruel met with a great storm at sea.
+Chapter 4.XX. How the pilots were forsaking their ships in the greatest stress of weather.
+Chapter 4.XXI. A continuation of the storm, with a short discourse on the subject of making testaments at sea.
+Chapter 4.XXII. An end of the storm.
+Chapter 4.XXIII. How Panurge played the good fellow when the storm was over.
+Chapter 4.XXIV. How Panurge was said to have been afraid without reason during the storm.
+Chapter 4.XXIX. How Pantagruel sailed by the Sneaking Island, where Shrovetide reigned.
+Chapter 4.XXV. How, after the storm, Pantagruel went on shore in the islands of the Macreons.
+Chapter 4.XXVI. How the good Macrobius gave us an account of the mansion and decease of the heroes.
+Chapter 4.XXVII. Pantagruel's discourse of the decease of heroic souls; and of the dreadful prodigies that happened before the death of the late Lord de Langey.
+Chapter 4.XXVIII. How Pantagruel related a very sad story of the death of the heroes.
+Chapter 4.XXX. How Shrovetide is anatomized and described by Xenomanes.
+Chapter 4.XXXI. Shrovetide's outward parts anatomized.
+Chapter 4.XXXII. A continuation of Shrovetide's countenance.
+Chapter 4.XXXIII. How Pantagruel discovered a monstrous physeter, or whirlpool, near the Wild Island.
+Chapter 4.XXXIV. How the monstrous physeter was slain by Pantagruel.
+Chapter 4.XXXIX. How Friar John joined with the cooks to fight the Chitterlings.
+Chapter 4.XXXV. How Pantagruel went on shore in the Wild Island, the ancient abode of the Chitterlings.
+Chapter 4.XXXVI. How the wild Chitterlings laid an ambuscado for Pantagruel.
+Chapter 4.XXXVII. How Pantagruel sent for Colonel Maul-chitterling and Colonel Cut-pudding; with a discourse well worth your hearing about the names of places and persons.
+Chapter 4.XXXVIII. How Chitterlings are not to be slighted by men.

static/docs/gargantua_book/gargantua_chapter_5.txt

+Chapter 5.I. How Pantagruel arrived at the Ringing Island, and of the noise that we heard.
+Chapter 5.II. How the Ringing Island had been inhabited by the Siticines, who were become birds.
+Chapter 5.III. How there is but one pope-hawk in the Ringing Island.
+Chapter 5.IV. How the birds of the Ringing Island were all passengers.
+Chapter 5.IX. How we arrived at the island of Tools.
+Chapter 5.V. Of the dumb Knight-hawks of the Ringing Island.
+Chapter 5.VI. How the birds are crammed in the Ringing Island.
+Chapter 5.VII. How Panurge related to Master Aedituus the fable of the horse and the ass.
+Chapter 5.VIII. How with much ado we got a sight of the pope-hawk.
+Chapter 5.X. How Pantagruel arrived at the island of Sharping.
+Chapter 5.XI. How we passed through the wicket inhabited by Gripe-men-all, Archduke of the Furred Law-cats.
+Chapter 5.XII. How Gripe-men-all propounded a riddle to us.
+Chapter 5.XIII. How Panurge solved Gripe-men-all's riddle.
+Chapter 5.XIV. How the Furred Law-cats live on corruption.
+Chapter 5.XIX. How we arrived at the queendom of Whims or Entelechy.
+Chapter 5.XL. How the battle in which the good Bacchus overthrew the Indians was represented in mosaic work.
+Chapter 5.XLI. How the temple was illuminated with a wonderful lamp.
+Chapter 5.XLII. How the Priestess Bacbuc showed us a fantastic fountain in the temple, and how the fountain-water had the taste of wine, according to the imagination of those who drank of it.
+Chapter 5.XLIII. How the Priestess Bacbuc equipped Panurge in order to have the word of the Bottle.
+Chapter 5.XLIV. How Bacbuc, the high-priestess, brought Panurge before the Holy Bottle.
+Chapter 5.XLV. How Bacbuc explained the word of the Goddess-Bottle.
+Chapter 5.XLVI. How Panurge and the rest rhymed with poetic fury.
+Chapter 5.XLVII. How we took our leave of Bacbuc, and left the Oracle of the Holy Bottle.
+Chapter 5.XV. How Friar John talks of rooting out the Furred Law-cats.
+Chapter 5.XVI. How Pantagruel came to the island of the Apedefers, or Ignoramuses, with long claws and crooked paws, and of terrible adventures and monsters there.
+Chapter 5.XVII. How we went forwards, and how Panurge had like to have been killed.
+Chapter 5.XVIII. How our ships were stranded, and we were relieved by some people that were subject to Queen Whims (qui tenoient de la Quinte).
+Chapter 5.XX. How the Quintessence cured the sick with a song.
+Chapter 5.XXI. How the Queen passed her time after dinner.
+Chapter 5.XXII. How Queen Whims' officers were employed; and how the said lady retained us among her abstractors.
+Chapter 5.XXIII. How the Queen was served at dinner, and of her way of eating.
+Chapter 5.XXIV. How there was a ball in the manner of a tournament, at which Queen Whims was present.
+Chapter 5.XXIX. How Epistemon disliked the institution of Lent.
+Chapter 5.XXV. How the thirty-two persons at the ball fought.
+Chapter 5.XXVI. How we came to the island of Odes, where the ways go up and down.
+Chapter 5.XXVII. How we came to the island of Sandals; and of the order of Semiquaver Friars.
+Chapter 5.XXVIII. How Panurge asked a Semiquaver Friar many questions, and was only answered in monosyllables.
+Chapter 5.XXX. How we came to the land of Satin.
+Chapter 5.XXXI. How in the land of Satin we saw Hearsay, who kept a school of vouching.
+Chapter 5.XXXII. How we came in sight of Lantern-land.
+Chapter 5.XXXIII. How we landed at the port of the Lychnobii, and came to Lantern-land.
+Chapter 5.XXXIV. How we arrived at the Oracle of the Bottle.
+Chapter 5.XXXIX. How we saw Bacchus's army drawn up in battalia in mosaic work.
+Chapter 5.XXXV. How we went underground to come to the Temple of the Holy Bottle, and how Chinon is the oldest city in the world.
+Chapter 5.XXXVI. How we went down the tetradic steps, and of Panurge's fear.
+Chapter 5.XXXVII. How the temple gates in a wonderful manner opened of themselves.
+Chapter 5.XXXVIII. Of the temple's admirable pavement.

static/js/gargantext/Docs_dyna_chart_and_table.js

@@ -532,7 +532,9 @@ $("#corpusdisplayer").hide()
 // FIRST portion of code to be EXECUTED:
 // (3) Get records and hyperdata for paginator
 $.ajax({
-  url: '/api/nodes?types[]=DOCUMENT&pagination_limit=-1&parent_id=' + corpus_id,
+  url: '/api/nodes?types[]=DOCUMENT&pagination_limit=-1&parent_id='
+        + corpus_id
+        +'&fields[]=parent_id&fields[]=id&fields[]=name&fields[]=typename&fields[]=hyperdata',
   success: function(data){
     $("#content_loader").remove();
     $.each(data.records, function(i, record){

templates/pages/corpora/corpus.html

@@ -12,37 +12,12 @@
 
 <link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.easy-pie-chart.css"%}">
 <link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.dynatable.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/gargantext/tables.css"%}"/>
 
 <script type="text/javascript" src="{% static "js/d3/d3.js"%}"></script>
 <script type="text/javascript" src="{% static "js/d3/dc.js"%}"></script>
 <script type="text/javascript" src="{% static "js/d3/crossfilter.js"%}"></script>
 
-<style>
-.no-transition {
-  -webkit-transition: height 0.1s;
-  -moz-transition: height 0.1s;
-  -ms-transition: height 0.1s;
-  -o-transition: height 0.1s;
-  transition: height 0.1s;
-}
-th { color: #fff; }
-
-th a {
-  color: #fff;
-  font-weight: normal;
-  font-style: italic;
-  font-size: 0.9em;
-}
-
-.dynatable-record-count {
-  font-size: 0.7em;
-}
-.dynatable-pagination-links {
-  font-size: 0.7em;
-}
-
-</style>
-
 {% endblock %}
 
 

templates/pages/corpora/journals.html

+{% extends "pages/menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+
+<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/d3/dc.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.dynatable.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/gargantext/tables.css"%}"/>
+
+<script type="text/javascript" src="{% static "js/d3/d3.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/crossfilter.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/dc.js"%}"></script>
+
+{% endblock %}
+
+
+{% block content %}
+
+<div class="container">
+    <div class="jumbotron">
+        <div class="row">
+            <div id="monthly-move-chart">
+                <center>
+                    Select a frequency group in the chart with blue bars to zoom in
+                <p align="center">
+                <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
+                <a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
+                <!-- <p style="font-size:70%">
+                    <b>x</b>: amount of documents for the journal
+                    <b>y</b>: number of journals with that amount
+                </p> -->
+                <div class="clearfix"></div>
+                </center>
+            </div>
+        </div>
+        <div class="row">
+            <div id="monthly-volume-chart"></div>
+        </div>
+        <div id="content_loader">
+          <br>
+          <center>
+            <!-- <img width="10%" src="{% static "img/ajax-loader.gif"%}"></img> -->
+          </center>
+          <br>
+        </div>
+
+        <div class="row">
+            <div class="panel panel-default">
+
+                <div class="panel-heading">
+                  <h4 class="panel-title">
+                    <a data-toggle="collapse" data-target="#journal_table" href="#">
+                        <!-- Final_UpdateTable redraws the dynatable if necessary -->
+                        <p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg">
+                          Open Folder
+                        </p>
+                    </a>
+                  </h4>
+                </div>
+
+                <div id="journal_table" class="panel-collapse collapse in no-transition" role="tabpanel">
+                    <div class="panel-body">
+                      <div id="div-table">
+                          <!-- (table id="my-ajax-table") dynamically set by Journals_dyna_chart_and_table -->
+                      </div>
+                    </div>
+                </div> <!-- /div panel-collapse -->
+
+            </div> <!-- /div panel -->
+        </div> <!-- /row with the dynatable panels -->
+
+    </div> <!-- /jumbotron -->
+</div> <!-- /container -->
+
+
+
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script src="{% static "js/bootstrap/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
+
+<!-- custom-lib for dynatable.js and dc.js -->
+<script type="text/javascript" src="{% static "js/gargantext/Journals_dyna_chart_and_table.js" %}"></script>
+
+{% endblock %}

templates/pages/menu.html

@@ -38,7 +38,7 @@
                         <li class="dropdown">
                             <a href="#" role="button" class="dropdown-toggle" data-toggle="dropdown" title="That is your login">
                                 <i class="icon-user"></i>
-                                {{ user }}
+                                {{ user.username }}
                                 <i class="caret"></i>
                             </a>
                             <ul class="dropdown-menu">
@@ -58,6 +58,23 @@
         <!--/.nav-collapse -->
 
         {% block corpusBannerTop %}
+
+        {% if corpus %}
+        <div class="container theme-showcase">
+            <div class="jumbotron" style="margin-bottom:0">
+                <div id="corpusbar" class="row">
+                    <div class="col-md-6">
+                        <div class="btn-group btn-group-justified">
+                            <center>
+                                <a type="button" class="btn btn-default {% if view == "documents" %}active{%endif%}" href="/projects/{{project.id}}/corpora/{{ corpus.id }}/">Documents</a>
+                                <a type="button" class="btn btn-default {% if view == "journals" %}active{%endif%}" href="/projects/{{project.id}}/corpora/{{ corpus.id }}/journals">Journals</a>
+                            </center>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        {% endif %}
         {% endblock %}
 
 

templates/pages/projects/overview.html

@@ -4,8 +4,23 @@
 {% block css %}
 {% load staticfiles %}
 <link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/gargantext/garganrest.js" %}"></script>
+<link rel="stylesheet" href="http://code.jquery.com/ui/1.11.2/themes/smoothness/jquery-ui.css">
 
-<script src="{% static "js/jquery/jquery.min.js" %}" type="text/javascript"></script>
+<script type="text/javascript" src="{% static "js/morris.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/morris.min.js" %}"></script>
+<link rel="stylesheet" href="{% static "css/morris.css" %}">
+
+<script src="{% static "js/raphael-min.js"%}"></script>
+<style type="text/css">
+    .ui-autocomplete {
+        z-index: 5000;
+    }
+    .ui-autocomplete .ui-menu-item {
+        font-size:x-small;
+    }
+</style>
 
 {% endblock %}
 
@@ -15,7 +30,7 @@
 
 <div class="container theme-showcase" role="main">
     <div class="jumbotron">
-        <h1>My {{number}} projects</h1>
+        <h1>My projects</h1>
         <p>Template showing my working space</p>
         <p>
             <button
@@ -45,17 +60,19 @@
         {% if projects %}
         {% for project in projects %}
         <!--<div class="col-md-offset-7 col-md-4 content" style="background-color:grey">!-->
-        <div class="col-md-3 content">
-            <h3><a href="/projects/{{ project.id }}">{{ project.name }}</a>
+        <div id="project_{{project.id}}" class="col-md-3 content">
+            <h3><a  href="/projects/{{ project.id }}">{{ project.name }}</a>
 
                 <button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
-                data-content='
-                <ul>
-                    <li> Rename </li>
-                    <li><a href="/projects/{{ project.id }}">Add new corpus</a></li>
-                    <li><a href="/delete/{{ project.id }}">Delete</a></li>
-                </ul>
-                '>Manage</button>
+                data-content="  <ul>
+                                <li>Rename</li>
+                                <li onclick=&quot;
+                                        garganrest.nodes.delete({{project.id}}, function(){$('#project_'+{{project.id}}).remove()});
+                                        $(this).parent().parent().remove();
+                                    &quot;><a href=&quot;#&quot;>Delete</a></li>
+                                </ul>
+                ">Manage</button>
+                </li>
                 {% if common_users %}
                 <a style="cursor:pointer;"><img class="share_button" data-id="{{ project.id }}" title="Share it!" width="20px" src="{% static "img/share.png" %}"></img></a>
                 {% endif %}

templates/pages/projects/project.html

@@ -118,9 +118,9 @@
     </div>
     {% endif %}
 
-    <h2>Lists of Ngrams</h2>
 
     {% if whitelists  %}
+    <h2>Lists of Ngrams</h2>
     <h3>White Lists</h3>
     <ul>
         {% for list in whitelists %}