Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into mat-master

Conflicts:
	gargantext_web/urls.py
	gargantext_web/views.py
	node/admin.py

analysis/InterUnion.py

+import networkx as nx
+from itertools import combinations
+
+class Utils:
+
+	def __init__(self):
+		self.G = nx.Graph()
+
+	def unique(self,a):
+		""" return the list with duplicate elements removed """
+		return list(set(a))
+
+	def intersect(self,a, b):
+		""" return the intersection of two lists """
+		return list(set(a) & set(b))
+
+	def union(self,a, b):
+		""" return the union of two lists """
+		return list(set(a) | set(b))
+
+	def addCompleteSubGraph(self,terms):
+		G=self.G
+		# <addnode> #
+		for i in terms:
+			G.add_node(i)
+		# </addnode> #
+
+		# <addedge> #
+		edges = combinations(terms, 2)
+		for n in edges:
+			n1=n[0]
+			n2=n[1]
+			one=float(1)
+			if G.has_edge(n1,n2):
+				G[n1][n2]['weight']+=one
+			else: G.add_edge(n1,n2,weight=one)
+		self.G = G
\ No newline at end of file

analysis/functions.py

@@ -31,11 +31,11 @@ def create_whitelist(user, corpus, size=100):
     black_list = Node.objects.create(name='BlackList Corpus ' + str(corpus.id), user=user, parent=corpus, type=blacklist_type)
 
     # delete avant pour éviter les doublons
-#    try:
-#        Node_Ngram.objects.filter(node=white_list).all().delete()
-#    except:
-#        print('First time we compute cooc')
-#
+    #    try:
+    #        Node_Ngram.objects.filter(node=white_list).all().delete()
+    #    except:
+    #        print('First time we compute cooc')
+    #
     query_whitelist = """
         INSERT INTO node_node_ngram (node_id, ngram_id, weight)
         SELECT
@@ -68,7 +68,8 @@ def create_whitelist(user, corpus, size=100):
             %d
         ;
     """  % (white_list.id, corpus.id, type_document.id, size)
-
+    print("PRINTING QYERY OF WHITELIST:")
+    print(query_whitelist)
     cursor.execute(query_whitelist)
 
     return white_list
@@ -160,12 +161,11 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
         print("Coocurrences do not exist yet, create it.")
         whitelist = create_whitelist(request.user, corpus, size=n)
         cooccurrence_node = create_cooc(user=request.user, corpus=corpus, whitelist=whitelist, size=n)
-        print(cooccurrence_node.id, "Cooc created")
     else:
         cooccurrence_node = Node.objects.filter(type=type_cooc, parent=corpus).first()
 
     for cooccurrence in NodeNgramNgram.objects.filter(node=cooccurrence_node):
-        
+        # print(cooccurrence.ngramx.terms," <=> ",cooccurrence.ngramy.terms,"\t",cooccurrence.score)
         ids[cooccurrence.ngramx.terms] = cooccurrence.ngramx.id
         ids[cooccurrence.ngramy.terms] = cooccurrence.ngramy.id
 
@@ -178,7 +178,6 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
         weight[cooccurrence.ngramy.terms] = weight.get(cooccurrence.ngramy.terms, 0) + cooccurrence.score
         weight[cooccurrence.ngramx.terms] = weight.get(cooccurrence.ngramx.terms, 0) + cooccurrence.score
 
-
     df = pd.DataFrame(matrix).fillna(0)
     x = copy(df.values)
     x = x / x.sum(axis=1)
@@ -191,11 +190,10 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
     G = nx.from_numpy_matrix(matrix_filtered)
     G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(df.columns)])))
     #G = nx.relabel_nodes(G, dict(enumerate(df.columns)))
-    
     # Removing too connected nodes (find automatic way to do it)
-#    outdeg = G.degree()
-#    to_remove = [n for n in outdeg if outdeg[n] >= 10]
-#    G.remove_nodes_from(to_remove)
+    #    outdeg = G.degree()
+    #    to_remove = [n for n in outdeg if outdeg[n] >= 10]
+    #    G.remove_nodes_from(to_remove)
 
     partition = best_partition(G)
     
@@ -212,7 +210,6 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
                 G.node[node]['size']    = weight[node]
                 G.node[node]['group']   = partition[node]
                 G.add_edge(node, "cluster " + str(partition[node]), weight=3)
-#            G.node[node]['color'] = '19,180,300'
             except Exception as error:
                 print(error)
 
@@ -227,61 +224,46 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
                 #G.node[node]['size']    = weight[node]
                 G.node[node]['group']   = partition[node]
                 #G.add_edge(node, partition[node], weight=3)
-#            G.node[node]['color'] = '19,180,300'
             except Exception as error:
                 print(error)
         data = json_graph.node_link_data(G)
      
 
-#    data = json_graph.node_link_data(G, attrs={\
-#            'source':'source',\
-#            'target':'target',\
-#            'weight':'weight',\
-#            #'label':'label',\
-#            #'color':'color',\
-#            'id':'id',})
+    #    data = json_graph.node_link_data(G, attrs={\
+    #            'source':'source',\
+    #            'target':'target',\
+    #            'weight':'weight',\
+    #            #'label':'label',\
+    #            #'color':'color',\
+    #            'id':'id',})
     #print(data)
     return data
 
 
-#def tfidf(corpus, document, ngram):
-#    '''
-#    Compute TF-IDF (Term Frequency - Inverse Document Frequency)
-#    See: http://en.wikipedia.org/wiki/Tf%E2%80%93idf
-#    '''
-#    try:
-#        occurences_of_ngram = Node_Ngram.objects.get(node=document, ngram=ngram).weight
-#        ngrams_by_document = sum([ x.weight for x in Node_Ngram.objects.filter(node=document)])
-#        term_frequency = occurences_of_ngram / ngrams_by_document
-#    
-#        xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
-#        yy = Node_Ngram.objects.filter(ngram=ngram).count() # filter: ON node.parent=corpus
-#        inverse_document_frequency= log(xx/yy)
-#        
-#        # result = tf * idf
-#        result = term_frequency * inverse_document_frequency
-#    except Exception as error:
-#        print(error, ngram)
-#        result = 0
-#    return result
-
 from analysis.tfidf import tfidf
 
 def do_tfidf(corpus, reset=True):
-    print("doing tfidf")
+    # print("=========== doing tfidf ===========")
     with transaction.atomic():
         if reset==True:
             NodeNodeNgram.objects.filter(nodex=corpus).delete()
         
         if isinstance(corpus, Node) and corpus.type.name == "Corpus":
+            # print("\n- - - - - - - - - - ")
+            # # for i in Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")):
             for document in Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")):
+                # print("the doc:",document)
                 for node_ngram in Node_Ngram.objects.filter(node=document):
+                    # print("\tngram:",node_ngram.ngram)
                     try:
                         nnn = NodeNodeNgram.objects.get(nodex=corpus, nodey=document, ngram=node_ngram.ngram)
+                        # print("\t\tTRY")
                     except:
                         score = tfidf(corpus, document, node_ngram.ngram)
                         nnn = NodeNodeNgram(nodex=corpus, nodey=node_ngram.node, ngram=node_ngram.ngram, score=score)
                         nnn.save()
+                        # print("\t\t",node_ngram.ngram," : ",score)
+            # print("- - - - - - - - - - \n")
         else:
             print("Only corpus implemented yet, you put instead:", type(corpus))
 

analysis/tfidf.py

@@ -60,6 +60,7 @@ def tfidf(corpus, document, ngram):
                 .filter(NodeNgram.ngram_id == ngram.id)\
                 .count()
         
+        # print("\t\t\t","occs:",occurrences_of_ngram," || ngramsbydoc:",ngrams_by_document," || TF = occ/ngramsbydoc:",term_frequency," |||||| x:",xx," || y:",yy," || IDF = log(x/y):",log(xx/yy))
         inverse_document_frequency= log(xx/yy)
 
         # result = tf * idf

backupdb.py

+import os
+command = 'export PGPASSWORD=C8kdcUrAQy66U\npg_dump -U alexandre -h localhost gargandb| gzip > %s' % "mysqldump.db"
+os.system(command)

gargantext_web/team.py → gargantext_web/about.py

@@ -17,11 +17,11 @@ def get_team():
 
     '''
     team = [
-            { 'first_name' : 'Alexandre', 'last_name' : 'Delanoë', 'mail' : 'alexandre+gargantextATdelanoe.org', 'website' : 'http://alexandre.delanoe.org', 'picture' : 'alexandre.jpg'},
-            { 'first_name' : 'David', 'last_name' : 'Chavalarias', 'mail' : '', 'website' : 'http://chavalarias.com', 'picture' : 'david.jpg'},
-            { 'first_name' : 'Mathieu', 'last_name' : 'Rodic', 'mail' : '', 'website'  : 'http://rodic.fr', 'picture' : 'mathieu.jpg'},
-            { 'first_name' : 'Samuel', 'last_name' : 'Castillo J.', 'mail' : 'kaisleanATgmail.com', 'website'  : 'http://www.pksm3.droppages.com', 'picture' : 'samuel.jpg'},
-            { 'first_name' : 'Elias', 'last_name' : 'Showk', 'mail' : '', 'website' : 'https://github.com/elishowk', 'picture' : ''},
+            { 'first_name' : 'Alexandre', 'last_name' : 'Delanoë', 'mail' : 'alexandre+gargantextATdelanoe.org', 'website' : 'http://alexandre.delanoe.org', 'picture' : 'alexandre.jpg', 'role' : 'project manager, scientific board, developer'},
+            { 'first_name' : 'David', 'last_name' : 'Chavalarias', 'mail' : 'david.chavalariasATiscpif.fr', 'website' : 'http://chavalarias.com', 'picture' : 'david.jpg', 'role':'scientific board'},
+            { 'first_name' : 'Mathieu', 'last_name' : 'Rodic', 'mail' : '', 'website'  : 'http://rodic.fr', 'picture' : 'mathieu.jpg', 'role' : 'developer'},
+            { 'first_name' : 'Samuel', 'last_name' : 'Castillo J.', 'mail' : 'kaisleanATgmail.com', 'website'  : 'http://www.pksm3.droppages.com', 'picture' : 'samuel.jpg', 'role' : 'developer'},
+            { 'first_name' : 'Elias', 'last_name' : 'Showk', 'mail' : '', 'website' : 'https://github.com/elishowk', 'picture' : '', 'role' : 'developer'},
             #{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''},
             # copy paste the line above and write your informations please
             ]
@@ -35,7 +35,12 @@ def get_sponsors():
 
     '''
     sponsors = [
-            { 'name' : 'Mines ParisTech', 'website' : 'http://mines-paristech.fr', 'picture' : 'logo.png'},
+            { 'name' : 'Mines ParisTech', 'website' : 'http://mines-paristech.fr', 'picture' : 'mines.png', 'funds':''},
+            { 'name' : 'Institut Pasteur', 'website' : 'http://www.pasteur.fr', 'picture' : 'pasteur.png', 'funds':''},
+            { 'name' : 'Forccast', 'website' : 'http://forccast.hypotheses.org/', 'picture' : 'forccast.png', 'funds':''},
+            { 'name' : 'ADEME', 'website' : 'http://www.ademe.fr', 'picture' : 'ademe.png', 'funds':''},
+            { 'name' : 'EHESS', 'website' : 'http://www.ehess.fr', 'picture' : 'ehess.png', 'funds':''},
+            #{ 'name' : '', 'website' : '', 'picture' : '', 'funds':''},
             # copy paste the line above and write your informations please
             ]
 

gargantext_web/db.py

@@ -134,8 +134,8 @@ class ModelCache(dict):
             for column in self._columns
             if key.__class__ == column.type.python_type
         ]
-        if len(conditions) == 0:
-            raise KeyError
+        # if len(conditions) == 0:
+        #     raise KeyError
         element = session.query(self._model).filter(or_(*conditions)).first()
         if element is None:
             raise KeyError
@@ -149,7 +149,7 @@ class ModelCache(dict):
                 key = getattr(element, column_name)
                 self[key] = element
 
-class Cache:
+class Cache():
 
     def __getattr__(self, key):
         try:

gargantext_web/home.py

+
+import random
+import random_words
+from math import pi
+
+
+
+def paragraph_lorem(size_target=450):
+    '''
+    Function that returns paragraph with false latin language.
+    size_target is the number of random words that will be given.
+    '''
+    
+    lorem = random_words.LoremIpsum()
+    
+    sentences_list = lorem.get_sentences_list(sentences=5)
+    paragraph_size = 0
+    
+    while paragraph_size < size_target :
+        sentences_list.append(lorem.get_sentence())
+        paragraph = ' '.join(sentences_list)
+        paragraph_size = len(paragraph)
+    
+    return(paragraph)
+
+
+def paragraph_gargantua(size_target=500):
+    '''
+    Function that returns paragraph with chapter titles of Gargantua.
+    size_target is the number of random words that will be given.
+    '''
+    
+    paragraph = list()
+    paragraph_size = 0
+    chapter_number = 1
+    
+    while paragraph_size < size_target and chapter_number < 6:
+        chapitre = open('/srv/gargantext/static/docs/gargantua_book/gargantua_chapter_' + str(chapter_number) + '.txt', 'r')
+        paragraph.append(random.choice(chapitre.readlines()).strip())
+        chapitre.close()
+        paragraph_size = len(' '.join(paragraph))
+        chapter_number += 1
+    
+    return(' '.join(paragraph))
+
+
+def random_letter(mot, size_min=5):
+    '''
+    
+    Functions that randomize order letters of a 
+    word which size is greater that size_min.
+
+    '''
+    if len(mot) > size_min:
+        
+        size = round(len(mot) / pi)
+        
+        first_letters = mot[:size]
+        last_letters  = mot[-size:]
+        
+        others_letters = list(mot[size:-size])
+        random.shuffle(others_letters)
+        
+        mot_list = list()
+        mot_list.append(first_letters)
+        
+        for letter in others_letters:
+            mot_list.append(letter)
+            
+        mot_list.append(last_letters)
+        
+        return(''.join(mot_list))  
+        
+    else:
+        return(mot)
+
+
+tutoriel = """Il paraît que l'ordre des lettres dans un mot n'a pas d'importance. La première et la dernière lettre doivent être à la bonne place. Le reste peut être dans un désordre total et on peut toujours lire sans problème. On ne lit donc pas chaque lettre en elle-même, mais le mot comme un tout. Un changement de référentiel et nous transposons ce résultat au texte lui-même: l'ordre des mots est faiblement important comparé au contexte du texte qui, lui, est compté"""
+
+
+def paragraph_tutoreil(tutoriel=tutoriel):
+    '''
+    Functions that returns paragraph of words with words with
+    randomized letters.
+    '''
+    paragraph = ' '.join([ random_letter(mot) for mot in tutoriel.split(" ")]) \
+            + ": comptexter avec Gargantext."
+    return(paragraph)
+

gargantext_web/settings.py

@@ -28,6 +28,7 @@ SECRET_KEY = 'bt)3n9v&a02cu7^^=+u_t2tmn8ex5fvx8$x4r*j*pb1yawd+rz'
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
+MAINTENANCE = False
 
 TEMPLATE_DEBUG = True
 
@@ -66,6 +67,7 @@ INSTALLED_APPS = (
     'cte_tree',
     'node',
     'ngram',
+    'scrap_pubmed',
     'django_hstore',
     'djcelery',
     'aldjemy',

gargantext_web/urls.py

@@ -6,6 +6,7 @@ from django.contrib.auth.views import login
 from gargantext_web import views, views_optimized
 
 import gargantext_web.api
+import scrap_pubmed.views as pubmedscrapper
 
 
 admin.autodiscover()
@@ -19,10 +20,13 @@ urlpatterns = patterns('',
 
     url(r'^auth/$', views.login_user),
     url(r'^auth/logout/$', views.logout_user),
+    url(r'^img/logo.svg$', views.logo),
+    url(r'^css/bootstrap.css$', views.css),
     
     # User Home view
-    url(r'^$', views.home),
-    url(r'^about/', views.about),
+    url(r'^$', views.home_view),
+    url(r'^about/', views.get_about),
+    url(r'^maintenance/', views.get_maintenance),
     
     # Project Management
     url(r'^projects/$', views.projects),
@@ -47,16 +51,18 @@ urlpatterns = patterns('',
     url(r'^corpus/(\d+)/node_link.json$', views.node_link),
     url(r'^corpus/(\d+)/adjacency.json$', views.adjacency),
     url(r'^api/tfidf/(\d+)/(\d+(?:,\d+)+)$', views_optimized.tfidf),
+    # url(r'^api/tfidf/(\d+)/(\w+)$', views.tfidf),
+    url(r'^api/tfidf2/(\d+)/(\w+)$', views.tfidf2),
 
     # Data management
     url(r'^api$', gargantext_web.api.Root),
+    url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
+    url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
     url(r'^api/nodes/(\d+)/children/ngrams$', gargantext_web.api.NodesChildrenNgrams.as_view()),
     url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
     url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
     url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
     # url(r'^api/nodes/(\d+)/children/duplicates/delete$', gargantext_web.api.NodesChildrenDuplicates.delete ),
-    url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
-    url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
 
     url(r'^api/project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpusJSON),
 
@@ -65,7 +71,13 @@ urlpatterns = patterns('',
     url(r'^ngrams$', views.ngrams),
     url(r'^nodeinfo/(\d+)$', views.nodeinfo),
     url(r'^tests/mvc$', views.tests_mvc),
-    url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments)
+    url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments),
+
+    url(r'^tests/istextquery$', pubmedscrapper.getGlobalStatsISTEXT),
+    url(r'^tests/pubmedquery$', pubmedscrapper.getGlobalStats),
+    url(r'^tests/project/(\d+)/pubmedquery/go$', pubmedscrapper.doTheQuery),
+    url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX)
+
 )
 
 

gargantext_web/urls.py~

+from django.conf.urls import patterns, include, url
+
+from django.contrib import admin
+from django.contrib.auth.views import login
+
+from gargantext_web import views
+
+import gargantext_web.api
+
+
+admin.autodiscover()
+
+urlpatterns = patterns('',
+    
+    # Admin views
+    url(r'^admin/', include(admin.site.urls)),
+    url(r'^login/', include(admin.site.urls)),
+    url(r'^grappelli/', include('grappelli.urls')),
+    
+    # User views
+    url(r'^$', views.home),
+    
+    url(r'^projects/$', views.projects),
+    url(r'^project/(\d+)/delete/$', views.delete_project),
+    
+    url(r'^project/(\d+)/$', views.project),
+    
+    url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
+    url(r'^project/(\d+)/corpus/(\d+)/delete/$', views.delete_corpus),
+    
+    # Visualizations
+    url(r'^corpus/(\d+)/explorer$', views.explorer_graph),
+    url(r'^corpus/(\d+)/matrix$', views.explorer_matrix),
+    
+    # Getting data
+    url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
+    url(r'^corpus/(\d+)/node_link.json$', views.node_link),
+    url(r'^corpus/(\d+)/adjacency.json$', views.adjacency),
+
+    url(r'^api$', gargantext_web.api.Root),
+    url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
+    url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
+    
+    #url(r'^api/nodes$', gargantext_web.api.NodesController.get),
+    url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
+    url(r'^api/nodes/(\d+)/data$', gargantext_web.api.CorpusController.data),
+
+    url(r'^graph-it$', views.graph_it),
+    url(r'^ngrams$', views.ngrams),
+)
+
+
+from django.conf import settings
+if settings.DEBUG:
+    urlpatterns += patterns('',
+        url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {
+            'document_root': settings.MEDIA_ROOT,
+        }),
+        url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {
+            'document_root': settings.STATIC_ROOT,
+        }),
+)
+

gargantext_web/views_optimized.py

@@ -83,13 +83,13 @@ def project(request, project_id):
 
     # deal with the form
     if request.method == 'POST':
-        # fomr validation
+        # form validation
         form = CustomForm(request.POST, request.FILES)
         if form.is_valid():
             # extract information from the form
             name = form.cleaned_data['name']
             thefile = form.cleaned_data['file']
-            resourcetype = cache.ResourceType[form.cleaned_data['type']]
+            resourcetype = cache.ResourceType[form.cleaned_data['type']] # e.g: here it converts to "pubmed" idk why
             # which default language shall be used?
             if resourcetype.name == "europress_french":
                 language_id = cache.Language['fr'].id

init/README.rst

@@ -97,7 +97,6 @@ Extras
 
 Last steps of configuration:
 ----------------------------
-
 1) If your project is not in /srv/gargantext:
     ln -s [the project folder] /srv/gargantext
 
@@ -126,9 +125,14 @@ patch /srv/gargantext_env/lib/python3.4/site-packages/cte_tree/models.py /srv/ga
 /srv/gargantext/manage.py shell < /srv/gargantext/init/init.py
 
 
+Start Turbo parser server
+-------------------------
+See dependences in init/dependences.sh
+See README for install instructions /srv/gargantext/parsing/Taggers/nlpserver/README.rst
+
+
 Start the Python Notebook server
 --------------------------------
-
 1)  In Pyvenv:
     python manage.py shell_plus --notebook
 
@@ -144,7 +148,6 @@ $ python manage.py runserver
 
 For Production Server
 ---------------------
-
 git checkout stable
 
 $ sudo aptitude install rabbitmq-server

init/index.sql

+-- Indexing text fields
+CREATE INDEX node_node_name ON node_node (name);
+CREATE INDEX node_node_metadata_valuetext ON node_node_metadata (value_text);
+CREATE INDEX node_ngram_terms ON node_ngram (terms);
+
+-- indexing ALL foreing keys
+CREATE INDEX node_ngram__language_id ON node_ngram (language_id);
+CREATE INDEX node_node__type_id ON node_node (type_id);
+CREATE INDEX node_node__user_id ON node_node (user_id);
+CREATE INDEX node_node__language_id ON node_node (language_id);
+CREATE INDEX node_node__parent_id ON node_node (parent_id);
+CREATE INDEX node_node_metadata__node_id ON node_node_metadata (node_id);
+CREATE INDEX node_node_metadata__metadata_id ON node_node_metadata (metadata_id);
+CREATE INDEX node_node_ngram__ngram_id ON node_node_ngram (ngram_id);
+CREATE INDEX node_node_ngram__node_id ON node_node_ngram (node_id);
+CREATE INDEX node_nodengramngram__node_id ON node_nodengramngram (node_id);
+CREATE INDEX node_nodengramngram__ngramx_id ON node_nodengramngram (ngramx_id);
+CREATE INDEX node_nodengramngram__ngramy_id ON node_nodengramngram (ngramy_id);
+CREATE INDEX node_nodenodengram__nodey_id ON node_nodenodengram (nodey_id);
+CREATE INDEX node_nodenodengram__ngram_id ON node_nodenodengram (ngram_id);
+CREATE INDEX node_nodenodengram__nodex_id ON node_nodenodengram (nodex_id);
+CREATE INDEX node_node_resource__node_id ON node_node_resource (node_id);
+CREATE INDEX node_node_resource__resource_id ON node_node_resource (resource_id);
+CREATE INDEX node_resource__user_id ON node_resource (user_id);
+CREATE INDEX node_resource__type_id ON node_resource (type_id);
+

init/requirements.txt

@@ -4,11 +4,13 @@ Jinja2==2.7.3
 MarkupSafe==0.23
 Pillow==2.5.3
 Pygments==1.6
+RandomWords==0.1.12
 SQLAlchemy==0.9.8
 South==1.0
 aldjemy==0.3.10
 amqp==1.4.6
 anyjson==0.3.3
+bibtexparser==0.6.0
 billiard==3.3.0.18
 celery==3.1.15
 certifi==14.05.14
@@ -23,17 +25,19 @@ django-cte-trees==0.9.2
 django-extensions==1.4.0
 django-grappelli==2.5.3
 django-hstore==1.3.1
+django-maintenance==0.1
 django-mptt==0.6.1
 django-nested-inlines==0.1
+django-pgjson==0.2.2
 django-treebeard==2.0
 djangorestframework==3.0.0
 graphviz==0.4
 ipython==2.2.0
 kombu==3.0.23
-lxml==3.3.6
+lxml==3.4.1
 matplotlib==1.4.0
 networkx==1.9
-nltk==3.0a4
+# nltk==3.0a4
 nose==1.3.4
 numpy==1.8.2
 pandas==0.14.1
@@ -44,13 +48,16 @@ pycparser==2.10
 pydot2==1.0.33
 pyparsing==2.0.2
 python-dateutil==2.2
+python-igraph==0.7
 pytz==2014.7
 pyzmq==14.3.1
 readline==6.2.4.1
 redis==2.10.3
 scikit-learn==0.15.1
 scipy==0.14.0
+simplerandom==0.12.1
 six==1.7.3
 sympy==0.7.5
 tornado==4.0.1
 uWSGI==2.0.7
+ujson==1.33

init_cooc.py

+# Without this, we couldn't use the Django environment
+
+import os
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext_web.settings")
+os.environ.setdefault("DJANGO_HSTORE_GLOBAL_REGISTER", "False")
+
+
+# We're gonna use all the models!
+
+from node.models import User, NodeType, Node
+
+
+user = User.objects.get(username = 'contro2015.lait')
+
+# Reset: all data
+
+try:
+    typeDoc     = NodeType.objects.get(name='Cooccurrence')
+except Exception as error:
+    print(error)
+
+Node.objects.filter(user=user, type=typeDoc).all().delete()
+
+exit()

init_gargantext.py

@@ -217,10 +217,10 @@ corpus_pubmed.add_resource(
 for resource in corpus_pubmed.get_resources():
     print('Resource #%d - %s - %s' % (resource.id, resource.digest, resource.file))
     
-print('Parse corpus #%d...' % (corpus_pubmed.id, ))
-corpus_pubmed.parse_resources(verbose=True)
-print('Extract corpus #%d...' % (corpus_pubmed.id, ))
-corpus_pubmed.children.all().extract_ngrams(['title',])
-print('Parsed corpus #%d.' % (corpus_pubmed.id, ))
+# print('Parse corpus #%d...' % (corpus_pubmed.id, ))
+# corpus_pubmed.parse_resources(verbose=True)
+# print('Extract corpus #%d...' % (corpus_pubmed.id, ))
+# corpus_pubmed.children.all().extract_ngrams(['title',])
+# print('Parsed corpus #%d.' % (corpus_pubmed.id, ))
 
 exit()

node/admin.py

@@ -98,13 +98,10 @@ from django import forms
 
 from django.utils.translation import ugettext_lazy as _
 class CustomForm(forms.Form):
-    name = forms.CharField( label='Name', max_length=199 , required=True)
-    parsing_options = ResourceType.objects.all().values_list('id', 'name')
-    type = forms.IntegerField( widget=forms.Select( choices= parsing_options) , required=True )
+    name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
+    type = ModelChoiceField( ResourceType.objects.all() , widget=forms.Select(attrs={'onchange':'CustomForSelect( $("option:selected", this).text() );'}) )
     file = forms.FileField()
 
-
-
     # Description: clean_file()
     """
         * file_.content_type - Example: ['application/pdf', 'image/jpeg']
@@ -120,21 +117,16 @@ class CustomForm(forms.Form):
     """
     def clean_file(self):
         file_ = self.cleaned_data.get('file')
-        #Filename length
-        if len(file_.name)>30:
-            from datetime import datetime
-            file_.name = str(datetime.now().microsecond)
-            # raise forms.ValidationError(_('Come on dude, name too long. Now is:'+file_.name))
+        # #Filename length
+        # if len(file_.name)>30:
+        #     from datetime import datetime
+        #     file_.name = str(datetime.now().microsecond)
+        #     # raise forms.ValidationError(_('Come on dude, name too long. Now is:'+file_.name))
         #File size
-        if len(file_)>128 * 1024 * 1024:
-            raise forms.ValidationError(_('File too heavy! (<128MB).'))
-        ## File type:
-        # if file_.content_type == "application/zip":
-        #     raise forms.ValidationError(_('We need a zip pls.'))
+        if len(file_)>1024 ** 3:
+            raise forms.ValidationError(_('File too heavy! (>1GB).'))
         return file_
 
-
-
 class CorpusForm(ModelForm):
     #parent = ModelChoiceField(EmptyQuerySet)
     def __init__(self, *args, **kwargs):

parsing/FileParsers/EuropressFileParser.py

@@ -41,12 +41,43 @@ class EuropressFileParser(FileParser):
             html = etree.fromstring(contents, html_parser)
             
             try:
+                
+                format_europresse = 50
                 html_articles = html.xpath('/html/body/table/tbody')
+
                 if len(html_articles) < 1:
                     html_articles = html.xpath('/html/body/table')
+                    
+                    if len(html_articles) < 1:
+                        format_europresse = 1
+                        html_articles = html.xpath('//div[@id="docContain"]')
             except Exception as error:
                 print(error)
             
+            if format_europresse == 50:
+                name_xpath = "./tr/td/span[@class = 'DocPublicationName']"
+                header_xpath = "//span[@class = 'DocHeader']"
+                title_xpath = "string(./tr/td/span[@class = 'TitreArticleVisu'])"
+                text_xpath  = "./tr/td/descendant-or-self::*[not(self::span[@class='DocHeader'])]/text()"
+            elif format_europresse == 1:
+                name_xpath = "//span[@class = 'DocPublicationName']"
+                header_xpath = "//span[@class = 'DocHeader']"
+                title_xpath = "string(//div[@class = 'titreArticleVisu'])"
+                text_xpath  = "./descendant::*[\
+                        not(\
+                           self::div[@class='Doc-SourceText'] \
+                        or self::span[@class='DocHeader'] \
+                        or self::span[@class='DocPublicationName'] \
+                        or self::span[@id='docNameVisu'] \
+                        or self::span[@class='DocHeader'] \
+                        or self::div[@class='titreArticleVisu'] \
+                        or self::span[@id='docNameContType'] \
+                        or descendant-or-self::span[@id='ucPubliC_lblCertificatIssuedTo'] \
+                        or descendant-or-self::span[@id='ucPubliC_lblEndDate'] \
+                        or self::td[@class='txtCertificat'] \
+                        )]/text()"
+                doi_xpath  = "//span[@id='ucPubliC_lblNodoc']/text()"
+                
 
         except Exception as error:
             print(error)
@@ -58,19 +89,20 @@ class EuropressFileParser(FileParser):
                 metadata = {}
                 
                 if len(html_article):
-                    for name in html_article.xpath("./tr/td/span[@class = 'DocPublicationName']"):
+                    for name in html_article.xpath(name_xpath):
                         if name.text is not None:
                             format_journal = re.compile('(.*), (.*)', re.UNICODE)
                             test_journal = format_journal.match(name.text)
                             if test_journal is not None:
-                                metadata['source'] = test_journal.group(1)
+                                metadata['journal'] = test_journal.group(1)
                                 metadata['volume'] = test_journal.group(2)
                             else:
-                                metadata['source'] = name.text.encode(codif)
+                                metadata['journal'] = name.text.encode(codif)
 
-                    for header in html_article.xpath("./tr/td/span[@class = 'DocHeader']"):
+                    for header in html_article.xpath(header_xpath):
                         try:
                             text = header.text
+                            #print("header", text)
                         except Exception as error:
                             print(error)
 
@@ -136,8 +168,8 @@ class EuropressFileParser(FileParser):
                         if test_page is not None:
                             metadata['page'] = test_page.group(1).encode(codif)
 
-                    metadata['title'] = html_article.xpath("string(./tr/td/span[@class = 'TitreArticleVisu'])").encode(codif)
-                    metadata['text']  = html_article.xpath("./tr/td/descendant-or-self::*[not(self::span[@class='DocHeader'])]/text()")
+                    metadata['title'] = html_article.xpath(title_xpath).encode(codif)
+                    metadata['abstract']  = html_article.xpath(text_xpath)
                    
                     line = 0
                     br_tag = 10
@@ -183,16 +215,26 @@ class EuropressFileParser(FileParser):
                     metadata['publication_year']  = metadata['publication_date'].strftime('%Y')
                     metadata['publication_month'] = metadata['publication_date'].strftime('%m')
                     metadata['publication_day']  = metadata['publication_date'].strftime('%d')
-                    metadata['publication_date'] = ""
+                    metadata.pop('publication_date')
+                    
+                    if len(metadata['abstract'])>0 and format_europresse == 50: 
+                        metadata['doi'] = str(metadata['abstract'][-9])
+                        metadata['abstract'].pop()
+# Here add separator for paragraphs
+                        metadata['abstract'] = str(' '.join(metadata['abstract']))
+                        metadata['abstract'] = str(re.sub('Tous droits réservés.*$', '', metadata['abstract']))
+                    elif format_europresse == 1:
+                        metadata['doi'] = ' '.join(html_article.xpath(doi_xpath))
+                        metadata['abstract'] = metadata['abstract'][:-9]
+# Here add separator for paragraphs
+                        metadata['abstract'] = str(' '.join(metadata['abstract']))
+
+                    else: 
+                        metadata['doi'] = "not found"
+                    
+                    metadata['length_words'] = len(metadata['abstract'].split(' '))
+                    metadata['length_letters'] = len(metadata['abstract'])
                     
-                    if len(metadata['text'])>0: 
-                        metadata['doi'] = str(metadata['text'][-9])
-                        metadata['text'].pop()
-                        metadata['text'] = str(' '.join(metadata['text']))
-                        metadata['text'] = str(re.sub('Tous droits réservés.*$', '', metadata['text']))
-
-                    else: metadata['doi'] = "not found"
-
                     metadata['bdd']  = u'europresse'
                     metadata['url']  = u''
                     
@@ -201,7 +243,8 @@ class EuropressFileParser(FileParser):
                         metadata[key] = value.decode() if isinstance(value, bytes) else value
                     yield metadata
                     count += 1
-        
+            file.close()
+
         except Exception as error:
             print(error)
             pass

parsing/FileParsers/ISText.py

+from django.db import transaction
+from lxml import etree
+from .FileParser import FileParser
+from ..NgramsExtractors import *
+from datetime import datetime
+from io import BytesIO
+import json
+
+class ISText(FileParser):
+    
+    def _parse(self, thefile):
+        json_data=open(thefile,"r")
+        data = json.load(json_data)
+        json_data.close()
+        json_docs = data["hits"]
+        metadata_list = []
+        metadata_path = {
+            "id"                : "id",
+            "source"           : 'corpusName',
+            "title"             : 'title',
+            "genre"             : "genre",
+            # "language_iso3"     : 'MedlineCitation/Article/Language',
+            "doi"               : 'doi',
+            "host"              : 'host',
+            "publication_date"  : 'pubdate',
+            # "authors"           : 'author',
+            "authorsRAW"        : 'author',
+            "keywords"          : "keywords"
+        }
+        metadata = {}
+        import pprint
+        import datetime
+        for json_doc in json_docs:
+            for key, path in metadata_path.items():
+                try:
+                    # print(path," ==> ",len(json_doc[path]))
+                    metadata[key] = json_doc[path]
+                except: pass
+
+            # print("|",metadata["publication_date"])
+
+            if "doi" in metadata: metadata["doi"] = metadata["doi"][0]
+            
+            keywords = []
+            if "keywords" in metadata:
+                for keyw in metadata["keywords"]:
+                    keywords.append(keyw["value"] )
+                metadata["keywords"] = ", ".join( keywords )
+
+            moredate=False
+            moresource=False
+            if "host" in metadata:
+
+                if "genre" in metadata["host"] and len(metadata["host"]["genre"])>0:
+                    if "genre" in metadata and len(metadata["genre"])==0:
+                        metadata["genre"] = metadata["host"]["genre"]
+
+                # print(metadata["host"])
+                if "pubdate" in metadata["host"]:
+                    onebuffer = metadata["publication_date"]
+                    metadata["publication_date"] = []
+                    metadata["publication_date"].append(onebuffer)
+                    metadata["publication_date"].append( metadata["host"]["pubdate"] )
+
+                if "title" in metadata["host"]:
+                    metadata["journal"] = metadata["host"]["title"]
+
+            authors=False
+            if "authorsRAW" in metadata:
+                names = []
+                for author in metadata["authorsRAW"]: 
+                    names.append(author["name"])
+                metadata["authors"] = ", ".join(names)
+
+            if "host" in metadata: metadata.pop("host")
+            if "genre" in metadata:
+                if len(metadata["genre"])==0:
+                    metadata.pop("genre")
+            
+            if "publication_date" in metadata and isinstance(metadata["publication_date"], list):
+                if len(metadata["publication_date"])>1:
+                    d1 = metadata["publication_date"][0]
+                    d2 = metadata["publication_date"][1]
+                    # print("date1:",d1)
+                    # print("date2:",d2)
+                    if len(d1)==len(d2):
+                        metadata["publication_date"] = d2
+                        # if int(d1)>int(d2): metadata["publication_date"] = d2
+                    else:
+                        fulldate = ""
+                        year = d2[:4]
+                        fulldate+=year
+                        if len(d2)>4:
+                            month = d2[4:6]
+                            fulldate+="-"+month
+                            if len(d2)>6:
+                                day = d2[6:8]
+                                fulldate+="-"+day
+                        metadata["publication_date"] = fulldate
+                else:
+                    if "copyrightdate" in json_doc: 
+                        metadata["publication_date"] = json_doc["copyrightdate"]
+            else:
+                if "copyrightdate" in json_doc:
+                    metadata["publication_date"] = json_doc["copyrightdate"]
+            
+            print("||",metadata["title"])
+            metadata_list.append(metadata)
+            print("=============================")
+
+        print("\nlen list:",len(metadata_list))
+        return metadata_list

parsing/FileParsers/__init__.py

@@ -2,3 +2,4 @@ from .RisFileParser import RisFileParser
 from .IsiFileParser import IsiFileParser
 from .PubmedFileParser import PubmedFileParser
 from .EuropressFileParser import EuropressFileParser
+from .ISText import ISText

parsing/NgramsExtractors/NgramsExtractor.py

-from ..Taggers import Tagger
+from ..Taggers import TurboTagger
 import nltk
 
 
@@ -19,7 +19,7 @@ class NgramsExtractor:
         self.stop()
         
     def start(self):
-        self.tagger = Tagger()
+        self.tagger = TurboTagger()
         
     def stop(self):
         pass

parsing/Taggers/Tagger.py

@@ -58,9 +58,11 @@ class Tagger:
         if single:
             self.tagging_end()
         return []
-        
+    
+
     """Send a text to be tagged.
     """
+    # Not used right now
     def tag_text(self, text):
         tokens_tags = []
         self.tagging_start()

parsing/Taggers/nlpserver/README.md

@@ -15,6 +15,10 @@ CONFIGURATION
 The settings for the server can be found in `settings.py`.
 Please ensure the TCP port is not already in use on your machine, and that the path to the models are correct.
 
+START for tests
+===============
+python3 server.py
+"CTRL + c" to shut down
 
 START/STOP THE SERVER
 =====================

parsing/corpustools.py

@@ -123,7 +123,7 @@ def parse_resources(corpus, user=None, user_id=None):
                 language_id = None
             # create new node
             node = Node(
-                name = metadata_dict.get('title', '')[:255],
+                name = metadata_dict.get('title', '')[:200],
                 parent_id = corpus_id,
                 user_id = user_id,
                 type_id = type_id,
@@ -224,7 +224,7 @@ def extract_ngrams(corpus, keys):
         ngramsextractor = ngramsextractors[language_iso2]
         for text in nodeinfo[2:]:
             if text is not None and len(text):
-                ngrams = ngramsextractor.extract_ngrams(text)
+                ngrams = ngramsextractor.extract_ngrams(text.replace("[","").replace("]",""))
                 for ngram in ngrams:
                     terms = ' '.join([token for token, tag in ngram]).lower()
                     n = len(ngram)
@@ -287,8 +287,11 @@ def extract_ngrams(corpus, keys):
     node_ngram_data = list()
     for node_id, ngrams in node_ngram_list.items():
         for terms, weight in ngrams.items():
-            ngram_id = ngram_ids[terms]
-            node_ngram_data.append((node_id, ngram_id, weight, ))
+            try:
+                ngram_id = ngram_ids[terms]
+                node_ngram_data.append((node_id, ngram_id, weight, ))
+            except Exception as e:
+                print("err01:",e)
     bulk_insert(Node_Ngram, ['node_id', 'ngram_id', 'weight'], node_ngram_data, cursor=cursor)
     dbg.message = 'insert %d associations' % len(node_ngram_data)
     # commit to database
@@ -370,41 +373,42 @@ def compute_tfidf(corpus):
     ''' % (Node.__table__.name, Node_Ngram.__table__.name, corpus.id, ))
     cursor.execute('SELECT COUNT(*) FROM tmp__st')
     D = cursor.fetchone()[0]
-    lnD = log(D)
-    cursor.execute('UPDATE tmp__idf SET idf = idf + %f' % (lnD, ))
-    # show off
-    dbg.show('insert tfidf for %d documents' % D)
-    cursor.execute('''
-        INSERT INTO
-            %s (nodex_id, nodey_id, ngram_id, score)
-        SELECT
-            %d AS nodex_id,
-            tf.node_id AS nodey_id,
-            tf.ngram_id AS ngram_id,
-            (tf.frequency * idf.idf) AS score
-        FROM
-            tmp__idf AS idf
-        INNER JOIN
-            tmp__tf AS tf ON tf.ngram_id = idf.ngram_id
-    ''' % (NodeNodeNgram.__table__.name, corpus.id, ))
-    # # show off
-    # cursor.execute('''
-    #     SELECT
-    #         node.name,
-    #         ngram.terms,
-    #         node_node_ngram.score AS tfidf
-    #     FROM
-    #         %s AS node_node_ngram
-    #     INNER JOIN
-    #         %s AS node ON node.id = node_node_ngram.nodey_id
-    #     INNER JOIN
-    #         %s AS ngram ON ngram.id = node_node_ngram.ngram_id
-    #     WHERE
-    #         node_node_ngram.nodex_id = %d
-    #     ORDER BY
-    #         score DESC
-    # ''' % (NodeNodeNgram.__table__.name, Node.__table__.name, Ngram.__table__.name, corpus.id, ))
-    # for row in cursor.fetchall():
-    #     print(row)
-    # the end!
-    db.commit()
+    if D>0:
+        lnD = log(D)
+        cursor.execute('UPDATE tmp__idf SET idf = idf + %f' % (lnD, ))
+        # show off
+        dbg.show('insert tfidf for %d documents' % D)
+        cursor.execute('''
+            INSERT INTO
+                %s (nodex_id, nodey_id, ngram_id, score)
+            SELECT
+                %d AS nodex_id,
+                tf.node_id AS nodey_id,
+                tf.ngram_id AS ngram_id,
+                (tf.frequency * idf.idf) AS score
+            FROM
+                tmp__idf AS idf
+            INNER JOIN
+                tmp__tf AS tf ON tf.ngram_id = idf.ngram_id
+        ''' % (NodeNodeNgram.__table__.name, corpus.id, ))
+        # # show off
+        # cursor.execute('''
+        #     SELECT
+        #         node.name,
+        #         ngram.terms,
+        #         node_node_ngram.score AS tfidf
+        #     FROM
+        #         %s AS node_node_ngram
+        #     INNER JOIN
+        #         %s AS node ON node.id = node_node_ngram.nodey_id
+        #     INNER JOIN
+        #         %s AS ngram ON ngram.id = node_node_ngram.ngram_id
+        #     WHERE
+        #         node_node_ngram.nodex_id = %d
+        #     ORDER BY
+        #         score DESC
+        # ''' % (NodeNodeNgram.__table__.name, Node.__table__.name, Ngram.__table__.name, corpus.id, ))
+        # for row in cursor.fetchall():
+        #     print(row)
+        # the end!
+        db.commit()

scrap_pubmed/MedlineFetcherDavid2015.py

+# ****************************
+# *****  Medline Fetcher *****
+# ****************************
+
+# MEDLINE USER REQUIREMENT : Run retrieval scripts on weekends or between 9 pm and 5 am Eastern Time weekdays
+import sys
+if sys.version_info >= (3, 0): from urllib.request import urlopen
+else: from urllib import urlopen
+import os
+import time
+# import libxml2
+from lxml import etree
+import datetime
+from django.core.files import File
+import codecs
+
+import threading
+from queue import Queue
+# import time
+
+class MedlineFetcher:
+
+    def __init__(self):
+        self.queue_size = 8
+        self.q = Queue()
+        self.firstResults = []
+        self.lock = threading.Lock() # lock to serialize console output
+        self.pubMedEutilsURL = 'http://www.ncbi.nlm.nih.gov/entrez/eutils'
+        self.pubMedDB = 'Pubmed'
+        self.reportType = 'medline'
+
+
+    # Return the globalResults!:
+    # - count = 
+    # - queryKey = 
+    # - webEnv = 
+    def medlineEsearch(self , query):
+
+        # print ("MedlineFetcher::medlineEsearch :")
+
+        "Get number of results for query 'query' in variable 'count'"
+        "Get also 'queryKey' and 'webEnv', which are used by function 'medlineEfetch'"
+        print(query)
+        origQuery = query
+        query = query.replace(' ', '%20')
+            
+        eSearch = '%s/esearch.fcgi?db=%s&retmax=1&usehistory=y&term=%s' %(self.pubMedEutilsURL, self.pubMedDB, query)
+        eSearchResult = urlopen(eSearch)
+        data = eSearchResult.read()
+
+        root = etree.XML(data)
+
+        findcount = etree.XPath("/eSearchResult/Count/text()")
+        count = findcount(root)[0]
+        
+        findquerykey = etree.XPath("/eSearchResult/QueryKey/text()")
+        queryKey = findquerykey(root)[0]
+
+        findwebenv = etree.XPath("/eSearchResult/WebEnv/text()")
+        webEnv = findwebenv(root)[0]
+
+        values = { "query":origQuery , "count": int(str(count)), "queryKey": queryKey , "webEnv":webEnv }
+        return values
+
+
+    # RETMAX:
+    # Total number of UIDs from the retrieved set to be shown in the XML output (default=20)
+    # maximum of 100,000 records
+    def medlineEfetchRAW( self , fullquery):
+        
+
+        query = fullquery["string"]
+        retmax = fullquery["retmax"]
+        count = fullquery["count"]
+        queryKey = fullquery["queryKey"]
+        webEnv = fullquery["webEnv"]
+
+        "Fetch medline result for query 'query', saving results to file every 'retmax' articles"
+
+        queryNoSpace = query.replace(' ', '') # No space in directory and file names, avoids stupid errors
+        
+        print ("LOG::TIME: ",'medlineEfetchRAW :Query "' , query , '"\t:\t' , count , ' results')
+
+        retstart = 0
+        eFetch = '%s/efetch.fcgi?email=youremail@example.org&rettype=%s&retmode=xml&retstart=%s&retmax=%s&db=%s&query_key=%s&WebEnv=%s' %(self.pubMedEutilsURL, self.reportType, retstart, retmax, self.pubMedDB, queryKey, webEnv)
+        return eFetch
+
+    def ensure_dir(self , f):
+        d = os.path.dirname(f)
+        if not os.path.exists(d):
+            os.makedirs(d)
+
+    # generic!
+    def downloadFile(self, item):
+        url = item[0]
+        filename = item[1]
+        print("\tin test_downloadFile:")
+        # print(url,filename)
+        data = urlopen(url)
+        f = codecs.open(filename, "w" ,encoding='utf-8')
+        myfile = File(f)
+        myfile.write( data.read().decode('utf-8') )
+        myfile.close()
+        f.close()
+        with self.lock:
+            print(threading.current_thread().name, filename+" OK")
+            return filename
+
+    # generic!
+    def test_downloadFile(self, item):
+        url = item[0]
+        filename = item[1]
+        print("\tin downloadFile:")
+        data = urlopen(url)
+        return data
+
+    # generic!
+    def do_work(self,item):
+        # time.sleep(1) # pretend to do some lengthy work.
+        returnvalue = self.medlineEsearch(item)
+        with self.lock:
+            print(threading.current_thread().name, item)
+            return returnvalue
+
+    # The worker thread pulls an item from the queue and processes it
+    def worker(self):
+        while True:
+            item = self.q.get()
+            self.firstResults.append(self.do_work(item))
+            self.q.task_done()
+
+    def worker2(self):
+        while True:
+            item = self.q.get()
+            results = []
+            try: result = self.downloadFile(item)
+            except: result = False
+            self.firstResults.append(result)
+            self.q.task_done()
+
+    def chunks(self , l , n):
+        print("chunks:")
+        for i in range(0, len(l), n):
+            yield l[i:i+n]
+
+    # GLOBALLIMIT:
+    # I will retrieve this exact amount of publications.
+    # The publications per year i'll retrieve per year will be = (k/N)*GlobalLimit <- i'll use this as RETMAX
+    # - k : Number of publications of x year (according to pubmed)
+    # - N : Sum of every k belonging to {X} (total number of pubs according to pubmed)
+    # - GlobalLimit : Number of publications i want.
+    def serialFetcher(self , yearsNumber , query, globalLimit):
+
+        # Create the queue and thread pool.
+        for i in range(self.queue_size):
+             t = threading.Thread(target=self.worker)
+             t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
+             t.start()
+        start = time.perf_counter()
+
+        N = 0
+
+        print ("MedlineFetcher::serialFetcher :")
+        thequeries = []
+        globalresults = []
+        for i in range(yearsNumber):
+            year = str(2015 - i)
+            print ('YEAR ' + year)
+            print ('---------\n')
+            pubmedquery = str(year) + '[dp] '+query
+            self.q.put( pubmedquery ) #put task in the queue
+        
+        self.q.join()
+        print('time:',time.perf_counter() - start)
+
+        for globalresults in self.firstResults:
+            # globalresults = self.medlineEsearch(pubmedquery)
+            if globalresults["count"]>0:
+                N+=globalresults["count"]
+                querymetadata = { 
+                    "string": globalresults["query"] , 
+                    "count": globalresults["count"] , 
+                    "queryKey":globalresults["queryKey"] , 
+                    "webEnv":globalresults["webEnv"] , 
+                    "retmax":0 
+                }
+                thequeries.append ( querymetadata )
+
+        print("Total Number:", N,"publications")
+        print("And i want just:",globalLimit,"publications")
+        print("---------------------------------------\n")
+
+        for i,query in enumerate(thequeries):
+            k = query["count"]
+            proportion = k/float(N)
+            retmax_forthisyear = int(round(globalLimit*proportion))
+            query["retmax"] = retmax_forthisyear
+            if query["retmax"]==0: query["retmax"]+=1
+
+        return thequeries

scrap_pubmed/admin.py

+from django.contrib import admin
+from gargantext_web.settings import STATIC_ROOT
+# Register your models here.
+import os
+import datetime
+
+class Logger():
+	
+	def write(msg):
+		path = "Logs/"
+		Logger.ensure_dir(path)
+		
+		nowfull = datetime.datetime.now().isoformat().split("T")
+		date = nowfull[0]
+		time = nowfull[1]
+
+		return path
+
+	def ensure_dir(f):
+		d = os.path.dirname(f)
+		if not os.path.exists(d):
+			os.makedirs(d)

scrap_pubmed/models.py

+from django.db import models
+
+# Create your models here.

scrap_pubmed/tests.py

+from django.test import TestCase
+
+# Create your tests here.

scrap_pubmed/views.py

+from django.shortcuts import redirect
+from django.shortcuts import render
+
+from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.template.loader import get_template
+from django.template import Context
+from django.contrib.auth.models import User, Group
+
+from scrap_pubmed.MedlineFetcherDavid2015 import MedlineFetcher
+
+from gargantext_web.api import JsonHttpResponse
+from urllib.request import urlopen, urlretrieve
+import json
+
+from gargantext_web.settings import MEDIA_ROOT
+# from datetime import datetime
+import time
+import datetime
+import os
+import threading
+from django.core.files import File
+from gargantext_web.settings import DEBUG
+
+from node.models import Language, ResourceType, Resource, \
+        Node, NodeType, Node_Resource, Project, Corpus, \
+        Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram
+
+
+def getGlobalStats(request ):
+	print(request.method)
+	alist = ["bar","foo"]
+
+	if request.method == "POST":
+		N = 100
+		query = request.POST["query"]
+		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
+		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
+		instancia = MedlineFetcher()
+		# alist = instancia.serialFetcher( 5, query , int(request.POST["N"]) )
+		alist = instancia.serialFetcher( 5, query , N )
+
+	data = alist
+	return JsonHttpResponse(data)
+
+
+
+def getGlobalStatsISTEXT(request ):
+	print(request.method)
+	alist = ["bar","foo"]
+
+	if request.method == "POST":
+		N = 100
+		query = request.POST["query"]
+		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
+		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
+		query_string = query.replace(" ","+")
+		url = "http://api.istex.fr/document/?q="+query_string
+
+		tasks = MedlineFetcher()
+
+		filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.datetime.now().isoformat()))
+
+		try: 
+			thedata = tasks.test_downloadFile( [url,filename] )
+			alist = thedata.read().decode('utf-8')
+		except Exception as error:
+			alist = [str(error)]
+	data = alist
+	return JsonHttpResponse(data)
+
+
+def doTheQuery(request , project_id):
+	alist = ["hola","mundo"]
+
+	if request.method == "POST":
+		query = request.POST["query"]
+		name = request.POST["string"]
+
+		instancia = MedlineFetcher()
+		thequeries = json.loads(query)
+
+		urlreqs = []
+		for yearquery in thequeries:
+			urlreqs.append( instancia.medlineEfetchRAW( yearquery ) )
+		alist = ["tudo fixe" , "tudo bem"]
+
+		"""
+		urlreqs: List of urls to query.
+		- Then, to each url in urlreqs you do:
+			eFetchResult = urlopen(url)
+			eFetchResult.read()  # this will output the XML... normally you write this to a XML-file.
+		"""
+
+		thefile = "how we do this here?"
+		resource_type = ResourceType.objects.get(name="pubmed" )
+
+		parent      = Node.objects.get(id=project_id)
+		node_type   = NodeType.objects.get(name='Corpus')
+		type_id = NodeType.objects.get(name='Document').id
+		user_id = User.objects.get( username=request.user ).id
+
+		corpus = Node(
+			user=request.user,
+			parent=parent,
+			type=node_type,
+			name=name,
+		)
+		corpus.save()
+
+		tasks = MedlineFetcher()
+		for i in range(8):
+			t = threading.Thread(target=tasks.worker2) #thing to do
+			t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
+			t.start()
+		for url in urlreqs:
+			filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.datetime.now().isoformat()))
+			tasks.q.put( [url , filename]) #put a task in th queue
+		tasks.q.join() # wait until everything is finished
+
+		dwnldsOK = 0
+		for filename in tasks.firstResults:
+			if filename!=False:
+				corpus.add_resource( user=request.user, type=resource_type, file=filename )
+				dwnldsOK+=1
+			
+		if dwnldsOK == 0: return JsonHttpResponse(["fail"])
+
+		# do the WorkFlow
+		try:
+			if DEBUG is True:
+				corpus.workflow() # old times...
+				# corpus.workflow__MOV()
+				# corpus.write_everything_to_DB()
+			else:
+				corpus.workflow.apply_async((), countdown=3)
+				# corpus.workflow__MOV() # synchronous! because is faaast
+				# corpus.write_everything_to_DB.apply_async((), countdown=3) # asynchronous
+
+
+			return JsonHttpResponse(["workflow","finished"])
+		except Exception as error:
+			print(error)
+
+		return JsonHttpResponse(["workflow","finished","outside the try-except"])
+
+	data = alist
+	return JsonHttpResponse(data)
+
+
+def testISTEX(request , project_id):
+	print(request.method)
+	alist = ["bar","foo"]
+
+
+
+	if request.method == "POST":
+		# print(alist)
+		query = "-"
+		query_string = "-"
+		N = 60
+		if "query" in request.POST: query = request.POST["query"]
+		if "string" in request.POST: query_string = request.POST["string"].replace(" ","+")
+		# if "N" in request.POST: N = request.POST["N"]
+		print(query_string , query , N)
+
+
+		urlreqs = []
+		pagesize = 50
+		tasks = MedlineFetcher()
+		chunks = list(tasks.chunks(range(N), pagesize))
+		for k in chunks:
+			if (k[0]+pagesize)>N: pagesize = N-k[0]
+			urlreqs.append("http://api.istex.fr/document/?q="+query_string+"&output=*&"+"from="+str(k[0])+"&size="+str(pagesize))
+		print(urlreqs)
+
+		urlreqs = ["http://localhost/374255" , "http://localhost/374278" ]
+		print(urlreqs)
+
+		resource_type = ResourceType.objects.get(name="istext" )
+
+		parent      = Node.objects.get(id=project_id)
+		node_type   = NodeType.objects.get(name='Corpus')
+		type_id = NodeType.objects.get(name='Document').id
+		user_id = User.objects.get( username=request.user ).id
+
+		corpus = Node(
+			user=request.user,
+			parent=parent,
+			type=node_type,
+			name=query,
+		)
+
+		corpus.save()
+
+		# configuring your queue with the event
+		for i in range(8):
+			t = threading.Thread(target=tasks.worker2) #thing to do
+			t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
+			t.start()
+		for url in urlreqs:
+			filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.now().microsecond))
+			tasks.q.put( [url , filename]) #put a task in th queue
+		tasks.q.join() # wait until everything is finished
+		for filename in tasks.firstResults:
+			corpus.add_resource( user=request.user, type=resource_type, file=filename )
+
+
+		corpus.save()
+		print("DEBUG:",DEBUG)
+		# do the WorkFlow
+		try:
+			if DEBUG is True:
+				corpus.workflow()
+			else:
+				corpus.workflow.apply_async((), countdown=3)
+
+			return JsonHttpResponse(["workflow","finished"])
+		except Exception as error:
+			print(error)
+
+	data = [query_string,query,N]
+	return JsonHttpResponse(data)
+

static/docs/gargantua_book/Source.txt

+Project Gutenberg's Gargantua and Pantagruel, Complete., by Francois Rabelais
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever.  You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.net
+
+
+Title: Gargantua and Pantagruel, Complete.
+       Five Books Of The Lives, Heroic Deeds And Sayings Of Gargantua And
+       His Son Pantagruel
+       
+
+Author: Francois Rabelais
+
+Release Date: August 8, 2004 [EBook #1200]
+
+Language: English
+
+
+*** START OF THIS PROJECT GUTENBERG EBOOK GARGANTUA AND PANTAGRUEL, ***
+
+
+
+
+Produced by Sue Asscher and David Widger
+
+
+
+
+
+MASTER FRANCIS RABELAIS
+
+
+FIVE BOOKS OF THE LIVES, HEROIC DEEDS AND SAYINGS OF
+
+GARGANTUA AND HIS SON PANTAGRUEL
+
+
+
+Translated into English by
+
+Sir Thomas Urquhart of Cromarty
+
+and
+
+Peter Antony Motteux
+
+
+
+
+The text of the first Two Books of Rabelais has been reprinted from the
+first edition (1653) of Urquhart's translation.  Footnotes initialled 'M.'
+are drawn from the Maitland Club edition (1838); other footnotes are by the
+translator.  Urquhart's translation of Book III. appeared posthumously in
+1693, with a new edition of Books I. and II., under Motteux's editorship.
+Motteux's rendering of Books IV. and V. followed in 1708.  Occasionally (as
+the footnotes indicate) passages omitted by Motteux have been restored from
+the 1738 copy edited by Ozell.
+
+
+

static/docs/gargantua_book/gargantua_chapter_1.txt

+Chapter 1.I. Of the Genealogy and Antiquity of Gargantua.
+Chapter 1.II. The Antidoted Fanfreluches:  or, a Galimatia of extravagant Conceits found in an ancient Monument.
+Chapter 1.III. How Gargantua was carried eleven months in his mother's belly.
+Chapter 1.IV. How Gargamelle, being great with Gargantua, did eat a huge deal of tripes.
+Chapter 1.IX. The colours and liveries of Gargantua.
+Chapter 1.L. Gargantua's speech to the vanquished.
+Chapter 1.LI. How the victorious Gargantuists were recompensed after the battle.
+Chapter 1.LII. How Gargantua caused to be built for the Monk the Abbey of Theleme.
+Chapter 1.LIII. How the abbey of the Thelemites was built and endowed.
+Chapter 1.LIV. The inscription set upon the great gate of Theleme.
+Chapter 1.LV. What manner of dwelling the Thelemites had.
+Chapter 1.LVI. How the men and women of the religious order of Theleme were apparelled.
+Chapter 1.LVII. How the Thelemites were governed, and of their manner of living.
+Chapter 1.LVIII. A prophetical Riddle.
+Chapter 1.V. The Discourse of the Drinkers.
+Chapter 1.VI. How Gargantua was born in a strange manner.
+Chapter 1.VII. After what manner Gargantua had his name given him, and how he tippled, bibbed, and curried the can.
+Chapter 1.VIII. How they apparelled Gargantua.
+Chapter 1.X. Of that which is signified by the colours white and blue.
+Chapter 1.XI. Of the youthful age of Gargantua.
+Chapter 1.XII. Of Gargantua's wooden horses.
+Chapter 1.XIII. How Gargantua's wonderful understanding became known to his father Grangousier, by the invention of a torchecul or wipebreech.
+Chapter 1.XIV. How Gargantua was taught Latin by a Sophister.
+Chapter 1.XIX. The oration of Master Janotus de Bragmardo for recovery of the bells.
+Chapter 1.XL. Why monks are the outcasts of the world; and wherefore some have bigger noses than others.
+Chapter 1.XLI. How the Monk made Gargantua sleep, and of his hours and breviaries.
+Chapter 1.XLII. How the Monk encouraged his fellow-champions, and how he hanged upon a tree.
+Chapter 1.XLIII. How the scouts and fore-party of Picrochole were met with by Gargantua, and how the Monk slew Captain Drawforth, and then was taken prisoner by his enemies.
+Chapter 1.XLIV. How the Monk rid himself of his keepers, and how Picrochole's forlorn hope was defeated.
+Chapter 1.XLIX. How Picrochole in his flight fell into great misfortunes, and what Gargantua did after the battle.
+Chapter 1.XLV. How the Monk carried along with him the Pilgrims, and of the good words that Grangousier gave them.
+Chapter 1.XLVI. How Grangousier did very kindly entertain Touchfaucet his prisoner.
+Chapter 1.XLVII. How Grangousier sent for his legions, and how Touchfaucet slew Rashcalf, and was afterwards executed by the command of Picrochole.
+Chapter 1.XLVIII. How Gargantua set upon Picrochole within the rock Clermond, and utterly defeated the army of the said Picrochole.
+Chapter 1.XV. How Gargantua was put under other schoolmasters.
+Chapter 1.XVI. How Gargantua was sent to Paris, and of the huge great mare that he rode on; how she destroyed the oxflies of the Beauce.
+Chapter 1.XVII. How Gargantua paid his welcome to the Parisians, and how he took away the great bells of Our Lady's Church.
+Chapter 1.XVIII. How Janotus de Bragmardo was sent to Gargantua to recover the great bells.
+Chapter 1.XX. How the Sophister carried away his cloth, and how he had a suit in law against the other masters.
+Chapter 1.XXI. The study of Gargantua, according to the discipline of his schoolmasters the Sophisters.
+Chapter 1.XXII. The games of Gargantua.
+Chapter 1.XXIII. How Gargantua was instructed by Ponocrates, and in such sort disciplinated, that he lost not one hour of the day.
+Chapter 1.XXIV. How Gargantua spent his time in rainy weather.
+Chapter 1.XXIX. The tenour of the letter which Grangousier wrote to his son Gargantua.
+Chapter 1.XXV. How there was great strife and debate raised betwixt the cake-bakers of Lerne, and those of Gargantua's country, whereupon were waged great wars.
+Chapter 1.XXVI. How the inhabitants of Lerne, by the commandment of Picrochole their king, assaulted the shepherds of Gargantua unexpectedly and on a sudden.
+Chapter 1.XXVII. How a monk of Seville saved the close of the abbey from being ransacked by the enemy.
+Chapter 1.XXVIII. How Picrochole stormed and took by assault the rock Clermond, and of Grangousier's unwillingness and aversion from the undertaking of war.
+Chapter 1.XXX. How Ulric Gallet was sent unto Picrochole.
+Chapter 1.XXXI. The speech made by Gallet to Picrochole.
+Chapter 1.XXXII. How Grangousier, to buy peace, caused the cakes to be restored.
+Chapter 1.XXXIII. How some statesmen of Picrochole, by hairbrained counsel, put him in extreme danger.
+Chapter 1.XXXIV. How Gargantua left the city of Paris to succour his country, and how Gymnast encountered with the enemy.
+Chapter 1.XXXIX. How the Monk was feasted by Gargantua, and of the jovial discourse they had at supper.
+Chapter 1.XXXV. How Gymnast very souply and cunningly killed Captain Tripet and others of Picrochole's men.
+Chapter 1.XXXVI. How Gargantua demolished the castle at the ford of Vede, and how they passed the ford.
+Chapter 1.XXXVII. How Gargantua, in combing his head, made the great cannon-balls fall out of his hair.
+Chapter 1.XXXVIII. How Gargantua did eat up six pilgrims in a salad.

static/docs/gargantua_book/gargantua_chapter_2.txt

+Chapter 2.I. Of the original and antiquity of the great Pantagruel.
+Chapter 2.II. Of the nativity of the most dread and redoubted Pantagruel.
+Chapter 2.III. Of the grief wherewith Gargantua was moved at the decease of his wife Badebec.
+Chapter 2.IV. Of the infancy of Pantagruel.
+Chapter 2.IX. How Pantagruel found Panurge, whom he loved all his lifetime.
+Chapter 2.V. Of the acts of the noble Pantagruel in his youthful age.
+Chapter 2.VI. How Pantagruel met with a Limousin, who too affectedly did counterfeit the French language.
+Chapter 2.VII. How Pantagruel came to Paris, and of the choice books of the Library of St. Victor.
+Chapter 2.VIII. How Pantagruel, being at Paris, received letters from his father Gargantua, and the copy of them.
+Chapter 2.X. How Pantagruel judged so equitably of a controversy, which was wonderfully obscure and difficult, that, by reason of his just decree therein, he was reputed to have a most admirable judgment.
+Chapter 2.XI. How the Lords of Kissbreech and Suckfist did plead before Pantagruel without an attorney.
+Chapter 2.XII. How the Lord of Suckfist pleaded before Pantagruel.
+Chapter 2.XIII. How Pantagruel gave judgment upon the difference of the two lords.
+Chapter 2.XIV. How Panurge related the manner how he escaped out of the hands of the Turks.
+Chapter 2.XIX. How Panurge put to a nonplus the Englishman that argued by signs.
+Chapter 2.XV. How Panurge showed a very new way to build the walls of Paris.
+Chapter 2.XVI. Of the qualities and conditions of Panurge.
+Chapter 2.XVII. How Panurge gained the pardons, and married the old women, and of the suit in law which he had at Paris.
+Chapter 2.XVIII. How a great scholar of England would have argued against Pantagruel, and was overcome by Panurge.
+Chapter 2.XX. How Thaumast relateth the virtues and knowledge of Panurge.
+Chapter 2.XXI. How Panurge was in love with a lady of Paris.
+Chapter 2.XXII. How Panurge served a Parisian lady a trick that pleased her not very well.
+Chapter 2.XXIII. How Pantagruel departed from Paris, hearing news that the Dipsodes had invaded the land of the Amaurots; and the cause wherefore the leagues are so short in France.
+Chapter 2.XXIV. A letter which a messenger brought to Pantagruel from a lady of Paris, together with the exposition of a posy written in a gold ring.
+Chapter 2.XXIX. How Pantagruel discomfited the three hundred giants armed.
+Chapter 2.XXV. How Panurge, Carpalin, Eusthenes, and Epistemon, the gentlemen attendants of Pantagruel, vanquished and discomfited six hundred and threescore horsemen very cunningly.
+Chapter 2.XXVI. How Pantagruel and his company were weary in eating still salt meats; and how Carpalin went a-hunting to have some venison.
+Chapter 2.XXVII. How Pantagruel set up one trophy in memorial of their valour, and Panurge another in remembrance of the hares.  How Pantagruel likewise with his farts begat little men, and with his fisgs little women; and how Panurge broke a great staff over two glasses.
+Chapter 2.XXVIII. How Pantagruel got the victory very strangely over the Dipsodes and the Giants.
+Chapter 2.XXX. How Epistemon, who had his head cut off, was finely healed by Panurge, and of the news which he brought from the devils, and of the damned people in hell.
+Chapter 2.XXXI. How Pantagruel entered into the city of the Amaurots, and how Panurge married King Anarchus to an old lantern-carrying hag, and made him a crier of green sauce.
+Chapter 2.XXXII. How Pantagruel with his tongue covered a whole army, and what the author saw in his mouth.
+Chapter 2.XXXIII. How Pantagruel became sick, and the manner how he was recovered.
+Chapter 2.XXXIV. The conclusion of this present book, and the excuse of the author.

static/docs/gargantua_book/gargantua_chapter_3.txt

+Chapter 3.I. How Pantagruel transported a colony of Utopians into Dipsody.
+Chapter 3.II. How Panurge was made Laird of Salmigondin in Dipsody, and did waste his revenue before it came in.
+Chapter 3.III. How Panurge praiseth the debtors and borrowers.
+Chapter 3.IV. Panurge continueth his discourse in the praise of borrowers and lenders.
+Chapter 3.IX. How Panurge asketh counsel of Pantagruel whether he should marry, yea, or no.
+Chapter 3.L. How the famous Pantagruelion ought to be prepared and wrought.
+Chapter 3.LI. Why it is called Pantagruelion, and of the admirable virtues.
+Chapter 3.LII. How a certain kind of Pantagruelion is of that nature that the fire is not able to consume it.
+Chapter 3.V. How Pantagruel altogether abhorreth the debtors and borrowers.
+Chapter 3.VI. Why new married men were privileged from going to the wars.
+Chapter 3.VII. How Panurge had a flea in his ear, and forbore to wear any longer his magnificent codpiece.
+Chapter 3.VIII. Why the codpiece is held to be the chief piece of armour amongst warriors.
+Chapter 3.X. How Pantagruel representeth unto Panurge the difficulty of giving advice in the matter of marriage; and to that purpose mentioneth somewhat of the Homeric and Virgilian lotteries.
+Chapter 3.XI. How Pantagruel showeth the trial of one's fortune by the throwing of dice to be unlawful.
+Chapter 3.XII. How Pantagruel doth explore by the Virgilian lottery what fortune Panurge shall have in his marriage.
+Chapter 3.XIII. How Pantagruel adviseth Panurge to try the future good or bad luck of his marriage by dreams.
+Chapter 3.XIV. Panurge's dream, with the interpretation thereof.
+Chapter 3.XIX. How Pantagruel praiseth the counsel of dumb men.
+Chapter 3.XL. How Bridlegoose giveth reasons why he looked upon those law- actions which he decided by the chance of the dice.
+Chapter 3.XLI. How Bridlegoose relateth the history of the reconcilers of parties at variance in matters of law.
+Chapter 3.XLII. How suits at law are bred at first, and how they come afterwards to their perfect growth.
+Chapter 3.XLIII. How Pantagruel excuseth Bridlegoose in the matter of sentencing actions at law by the chance of the dice.
+Chapter 3.XLIV. How Pantagruel relateth a strange history of the perplexity of human judgment.
+Chapter 3.XLIX. How Pantagruel did put himself in a readiness to go to sea; and of the herb named Pantagruelion.
+Chapter 3.XLV. How Panurge taketh advice of Triboulet.
+Chapter 3.XLVI. How Pantagruel and Panurge diversely interpret the words of Triboulet.
+Chapter 3.XLVII. How Pantagruel and Panurge resolved to make a visit to the Oracle of the Holy Bottle.
+Chapter 3.XLVIII. How Gargantua showeth that the children ought not to marry without the special knowledge and advice of their fathers and mothers.
+Chapter 3.XV. Panurge's excuse and exposition of the monastic mystery concerning powdered beef.
+Chapter 3.XVI. How Pantagruel adviseth Panurge to consult with the Sibyl of Panzoust.
+Chapter 3.XVII. How Panurge spoke to the Sibyl of Panzoust.
+Chapter 3.XVIII. How Pantagruel and Panurge did diversely expound the verses of the Sibyl of Panzoust.
+Chapter 3.XX. How Goatsnose by signs maketh answer to Panurge.
+Chapter 3.XXI. How Panurge consulteth with an old French poet, named Raminagrobis.
+Chapter 3.XXII. How Panurge patrocinates and defendeth the Order of the Begging Friars.
+Chapter 3.XXIII. How Panurge maketh the motion of a return to Raminagrobis.
+Chapter 3.XXIV. How Panurge consulteth with Epistemon.
+Chapter 3.XXIX. How Pantagruel convocated together a theologian, physician, lawyer, and philosopher, for extricating Panurge out of the perplexity wherein he was.
+Chapter 3.XXV. How Panurge consulteth with Herr Trippa.
+Chapter 3.XXVI. How Panurge consulteth with Friar John of the Funnels.
+Chapter 3.XXVII. How Friar John merrily and sportingly counselleth Panurge.
+Chapter 3.XXVIII. How Friar John comforteth Panurge in the doubtful matter of cuckoldry.
+Chapter 3.XXX. How the theologue, Hippothadee, giveth counsel to Panurge in the matter and business of his nuptial enterprise.
+Chapter 3.XXXI. How the physician Rondibilis counselleth Panurge.
+Chapter 3.XXXII. How Rondibilis declareth cuckoldry to be naturally one of the appendances of marriage.
+Chapter 3.XXXIII. Rondibilis the physician's cure of cuckoldry.
+Chapter 3.XXXIV. How women ordinarily have the greatest longing after things prohibited.
+Chapter 3.XXXIX. How Pantagruel was present at the trial of Judge Bridlegoose, who decided causes and controversies in law by the chance and fortune of the dice.
+Chapter 3.XXXV. How the philosopher Trouillogan handleth the difficulty of marriage.
+Chapter 3.XXXVI. A continuation of the answer of the Ephectic and Pyrrhonian philosopher Trouillogan.
+Chapter 3.XXXVII. How Pantagruel persuaded Panurge to take counsel of a fool.
+Chapter 3.XXXVIII. How Triboulet is set forth and blazed by Pantagruel and Panurge.

static/docs/gargantua_book/gargantua_chapter_4.txt

+Chapter 4.I. How Pantagruel went to sea to visit the oracle of Bacbuc, alias the Holy Bottle.
+Chapter 4.II. How Pantagruel bought many rarities in the island of Medamothy.
+Chapter 4.III. How Pantagruel received a letter from his father Gargantua, and of the strange way to have speedy news from far distant places.
+Chapter 4.IV. How Pantagruel writ to his father Gargantua, and sent him several curiosities.
+Chapter 4.IX. How Pantagruel arrived at the island of Ennasin, and of the strange ways of being akin in that country.
+Chapter 4.L. How Homenas showed us the archetype, or representation of a pope.
+Chapter 4.LI. Table-talk in praise of the decretals.
+Chapter 4.LII. A continuation of the miracles caused by the decretals.
+Chapter 4.LIII. How, by the virtue of the decretals, gold is subtilely drawn out of France to Rome.
+Chapter 4.LIV. How Homenas gave Pantagruel some bon-Christian pears.
+Chapter 4.LIX. Of the ridiculous statue Manduce; and how and what the Gastrolaters sacrifice to their ventripotent god.
+Chapter 4.LV. How Pantagruel, being at sea, heard various unfrozen words.
+Chapter 4.LVI. How among the frozen words Pantagruel found some odd ones.
+Chapter 4.LVII. How Pantagruel went ashore at the dwelling of Gaster, the first master of arts in the world.
+Chapter 4.LVIII. How, at the court of the master of ingenuity, Pantagruel detested the Engastrimythes and the Gastrolaters.
+Chapter 4.LX. What the Gastrolaters sacrificed to their god on interlarded fish-days.
+Chapter 4.LXI. How Gaster invented means to get and preserve corn.
+Chapter 4.LXII. How Gaster invented an art to avoid being hurt or touched by cannon-balls.
+Chapter 4.LXIII. How Pantagruel fell asleep near the island of Chaneph, and of the problems proposed to be solved when he waked.
+Chapter 4.LXIV. How Pantagruel gave no answer to the problems.
+Chapter 4.LXV. How Pantagruel passed the time with his servants.
+Chapter 4.LXVI. How, by Pantagruel's order, the Muses were saluted near the isle of Ganabim.
+Chapter 4.LXVII. How Panurge berayed himself for fear; and of the huge cat Rodilardus, which he took for a puny devil.
+Chapter 4.V. How Pantagruel met a ship with passengers returning from Lantern-land.
+Chapter 4.VI. How, the fray being over, Panurge cheapened one of Dingdong's sheep.
+Chapter 4.VII. Which if you read you'll find how Panurge bargained with Dingdong.
+Chapter 4.VIII. How Panurge caused Dingdong and his sheep to be drowned in the sea.
+Chapter 4.X. How Pantagruel went ashore at the island of Chely, where he saw King St. Panigon.
+Chapter 4.XI. Why monks love to be in kitchens.
+Chapter 4.XII. How Pantagruel passed by the land of Pettifogging, and of the strange way of living among the Catchpoles.
+Chapter 4.XIII. How, like Master Francis Villon, the Lord of Basche commended his servants.
+Chapter 4.XIV. A further account of catchpoles who were drubbed at Basche's house.
+Chapter 4.XIX. What countenances Panurge and Friar John kept during the.
+Chapter 4.XL. How Friar John fitted up the sow; and of the valiant cooks that went into it.
+Chapter 4.XLI. How Pantagruel broke the Chitterlings at the knees.
+Chapter 4.XLII. How Pantagruel held a treaty with Niphleseth, Queen of the Chitterlings.
+Chapter 4.XLIII. How Pantagruel went into the island of Ruach.
+Chapter 4.XLIV. How small rain lays a high wind.
+Chapter 4.XLIX. How Homenas, Bishop of Papimany, showed us the Uranopet decretals .
+Chapter 4.XLV. How Pantagruel went ashore in the island of Pope-Figland.
+Chapter 4.XLVI. How a junior devil was fooled by a husbandman of Pope- Figland.
+Chapter 4.XLVII. How the devil was deceived by an old woman of Pope- Figland.
+Chapter 4.XLVIII. How Pantagruel went ashore at the island of Papimany.
+Chapter 4.XV. How the ancient custom at nuptials is renewed by the catchpole.
+Chapter 4.XVI. How Friar John made trial of the nature of the catchpoles.
+Chapter 4.XVII. How Pantagruel came to the islands of Tohu and Bohu; and of the strange death of Wide-nostrils, the swallower of windmills.
+Chapter 4.XVIII. How Pantagruel met with a great storm at sea.
+Chapter 4.XX. How the pilots were forsaking their ships in the greatest stress of weather.
+Chapter 4.XXI. A continuation of the storm, with a short discourse on the subject of making testaments at sea.
+Chapter 4.XXII. An end of the storm.
+Chapter 4.XXIII. How Panurge played the good fellow when the storm was over.
+Chapter 4.XXIV. How Panurge was said to have been afraid without reason during the storm.
+Chapter 4.XXIX. How Pantagruel sailed by the Sneaking Island, where Shrovetide reigned.
+Chapter 4.XXV. How, after the storm, Pantagruel went on shore in the islands of the Macreons.
+Chapter 4.XXVI. How the good Macrobius gave us an account of the mansion and decease of the heroes.
+Chapter 4.XXVII. Pantagruel's discourse of the decease of heroic souls; and of the dreadful prodigies that happened before the death of the late Lord de Langey.
+Chapter 4.XXVIII. How Pantagruel related a very sad story of the death of the heroes.
+Chapter 4.XXX. How Shrovetide is anatomized and described by Xenomanes.
+Chapter 4.XXXI. Shrovetide's outward parts anatomized.
+Chapter 4.XXXII. A continuation of Shrovetide's countenance.
+Chapter 4.XXXIII. How Pantagruel discovered a monstrous physeter, or whirlpool, near the Wild Island.
+Chapter 4.XXXIV. How the monstrous physeter was slain by Pantagruel.
+Chapter 4.XXXIX. How Friar John joined with the cooks to fight the Chitterlings.
+Chapter 4.XXXV. How Pantagruel went on shore in the Wild Island, the ancient abode of the Chitterlings.
+Chapter 4.XXXVI. How the wild Chitterlings laid an ambuscado for Pantagruel.
+Chapter 4.XXXVII. How Pantagruel sent for Colonel Maul-chitterling and Colonel Cut-pudding; with a discourse well worth your hearing about the names of places and persons.
+Chapter 4.XXXVIII. How Chitterlings are not to be slighted by men.

static/docs/gargantua_book/gargantua_chapter_5.txt

+Chapter 5.I. How Pantagruel arrived at the Ringing Island, and of the noise that we heard.
+Chapter 5.II. How the Ringing Island had been inhabited by the Siticines, who were become birds.
+Chapter 5.III. How there is but one pope-hawk in the Ringing Island.
+Chapter 5.IV. How the birds of the Ringing Island were all passengers.
+Chapter 5.IX. How we arrived at the island of Tools.
+Chapter 5.V. Of the dumb Knight-hawks of the Ringing Island.
+Chapter 5.VI. How the birds are crammed in the Ringing Island.
+Chapter 5.VII. How Panurge related to Master Aedituus the fable of the horse and the ass.
+Chapter 5.VIII. How with much ado we got a sight of the pope-hawk.
+Chapter 5.X. How Pantagruel arrived at the island of Sharping.
+Chapter 5.XI. How we passed through the wicket inhabited by Gripe-men-all, Archduke of the Furred Law-cats.
+Chapter 5.XII. How Gripe-men-all propounded a riddle to us.
+Chapter 5.XIII. How Panurge solved Gripe-men-all's riddle.
+Chapter 5.XIV. How the Furred Law-cats live on corruption.
+Chapter 5.XIX. How we arrived at the queendom of Whims or Entelechy.
+Chapter 5.XL. How the battle in which the good Bacchus overthrew the Indians was represented in mosaic work.
+Chapter 5.XLI. How the temple was illuminated with a wonderful lamp.
+Chapter 5.XLII. How the Priestess Bacbuc showed us a fantastic fountain in the temple, and how the fountain-water had the taste of wine, according to the imagination of those who drank of it.
+Chapter 5.XLIII. How the Priestess Bacbuc equipped Panurge in order to have the word of the Bottle.
+Chapter 5.XLIV. How Bacbuc, the high-priestess, brought Panurge before the Holy Bottle.
+Chapter 5.XLV. How Bacbuc explained the word of the Goddess-Bottle.
+Chapter 5.XLVI. How Panurge and the rest rhymed with poetic fury.
+Chapter 5.XLVII. How we took our leave of Bacbuc, and left the Oracle of the Holy Bottle.
+Chapter 5.XV. How Friar John talks of rooting out the Furred Law-cats.
+Chapter 5.XVI. How Pantagruel came to the island of the Apedefers, or Ignoramuses, with long claws and crooked paws, and of terrible adventures and monsters there.
+Chapter 5.XVII. How we went forwards, and how Panurge had like to have been killed.
+Chapter 5.XVIII. How our ships were stranded, and we were relieved by some people that were subject to Queen Whims (qui tenoient de la Quinte).
+Chapter 5.XX. How the Quintessence cured the sick with a song.
+Chapter 5.XXI. How the Queen passed her time after dinner.
+Chapter 5.XXII. How Queen Whims' officers were employed; and how the said lady retained us among her abstractors.
+Chapter 5.XXIII. How the Queen was served at dinner, and of her way of eating.
+Chapter 5.XXIV. How there was a ball in the manner of a tournament, at which Queen Whims was present.
+Chapter 5.XXIX. How Epistemon disliked the institution of Lent.
+Chapter 5.XXV. How the thirty-two persons at the ball fought.
+Chapter 5.XXVI. How we came to the island of Odes, where the ways go up and down.
+Chapter 5.XXVII. How we came to the island of Sandals; and of the order of Semiquaver Friars.
+Chapter 5.XXVIII. How Panurge asked a Semiquaver Friar many questions, and was only answered in monosyllables.
+Chapter 5.XXX. How we came to the land of Satin.
+Chapter 5.XXXI. How in the land of Satin we saw Hearsay, who kept a school of vouching.
+Chapter 5.XXXII. How we came in sight of Lantern-land.
+Chapter 5.XXXIII. How we landed at the port of the Lychnobii, and came to Lantern-land.
+Chapter 5.XXXIV. How we arrived at the Oracle of the Bottle.
+Chapter 5.XXXIX. How we saw Bacchus's army drawn up in battalia in mosaic work.
+Chapter 5.XXXV. How we went underground to come to the Temple of the Holy Bottle, and how Chinon is the oldest city in the world.
+Chapter 5.XXXVI. How we went down the tetradic steps, and of Panurge's fear.
+Chapter 5.XXXVII. How the temple gates in a wonderful manner opened of themselves.
+Chapter 5.XXXVIII. Of the temple's admirable pavement.

templates/about.html

@@ -25,7 +25,9 @@
 				<div class="panel-heading">
 						<h2 class="panel-title">
 								<a data-toggle="collapse" data-parent="#accordion" href="#collapseVersions">
+										<center>
 										<h2>Versions</h2>
+										</center>
 								</a>
 						</h2>
 				</div>
@@ -35,15 +37,20 @@
 										<ul>
 												<li>Version 1.0</li>
 														<ul>
-																<li>Beta Version </li>
-																<li>Licence of Gargantext is GPL v3+ </li>
+																<li>[Start] Beta Version </li>
+																<li>[Law] Licence of Gargantext is GPL v3+ </li>
 														</ul>
 												<li>Version 1.0.5</li>
 														<ul>
-																<li>Bug resolution: xml zipped from Mac</li>
-																<li>Bug resolution: french accents in filenames</li>
+																<li>Bug resolution: [Import] xml zipped from Mac</li>
+																<li>Bug resolution: [Import] french accents in filenames</li>
 																<li>New features: [Advanced chart] ngrams completion</li>
-																<li>New features: button to delete all duplicates</li>
+																<li>New features: [Duplicates management] button to delete all duplicates</li>
+														</ul>
+												<li>Version 1.0.6</li>
+														<ul>
+																<li>Bug resolution: [Advanced chart] one can make comparisons with different corpora at different scales</li>
+																<li>Bug resolution: [Graph] Graph link can not be executed until workflow is finished.</li>
 														</ul>
 										</ul>
 								</div>
@@ -56,7 +63,9 @@
 				<div class="panel-heading">
 						<h2 class="panel-title">
 								<a data-toggle="collapse" data-parent="#accordion" href="#collapseCommunity">
+										<center>
 										<h2>Community</h2>
+										</center>
 								</a>
 						</h2>
 				</div>
@@ -94,7 +103,9 @@
 				<div class="panel-heading">
 						<h2 class="panel-title">
 								<a data-toggle="collapse" data-parent="#accordion" href="#collapseTeam">
+										<center>
 										<h2>Core team</h2>
+										</center>
 								</a>
 						</h2>
 				</div>
@@ -130,12 +141,34 @@
 								</div>
 						</div>
 				</div>
-
 {% endif %}
 
 		</div>
 </div>
 
+{% if sponsors %}
+<div class="panel panel-default">
+				<div class="panel-heading">
+						<h2 class="panel-title">
+										<center>
+										<h2>Sponsors</h2>
+										<a href="http://www.cnrs.fr" target="_blank" >
+										<img src="{% static "img/sponsors/cnrs.png"%}" alt="CNRS"  style="height:100px">
+										</a>
+										<a href="http://www.iscpif.fr" target="_blank" >
+										<img src="{% static "img/sponsors/iscpif.svg"%}" style="height:100px">
+										</a>
+										{% for sponsor in sponsors %}
+										<a href="{{ sponsor.website }}" target="_blank" >
+										<img src="{% static "img/sponsors/"%}{{ sponsor.picture }}" style="height:100px">
+										</a>
+										{% endfor %}
+										</center>
+</div>
+</div>
+</div>
+{% endif %}
+
 
 
 </div>

templates/chart.html

@@ -294,7 +294,7 @@
                     <br/>
                     Interpolation:
                     <select ng-model="graph.options.lineMode">
-                        <option ng-repeat="mode in ['bundle', 'linear']" value="{{ mode }}">{{ mode }}</option>
+                        <option ng-repeat="mode in ['linear', 'bundle']" value="{{ mode }}">{{ mode }}</option>
                     </select>
                     <span ng-if="graph.options.lineMode != 'linear'">
                         with a tension of

templates/corpus.html

@@ -131,7 +131,11 @@
 
 				<div class="col-md-4">
 						<div class="jumbotron">
-						<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% if processing == "1" %}
+                <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
+                {% else %}
+								<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% endif %}
 						<ol>
 								<li>Visualize</li>
 								<li>Explore</li>

templates/explorer.html

@@ -260,8 +260,29 @@
 
         </div>
         
-        
         <div id="topPapers"></div>
+        <!--
+        <div id="tab-container-top" class='tab-container'>
+
+           <ul class='etabs'>
+             <li id="tabmed" class='tab active'><a href="#tabs3">Medline Pubs</a></li>
+             <li id="tabgps" class='tab'><a href="#tabs4">+</a></li>
+           </ul>
+
+           <div class='panel-container'>
+              <div id="tabs3">
+                <div id="topPapers"></div>
+              </div>
+              <div id="tabs4">
+                <div id="topProposals"></div>
+              </div>
+           </div>
+        </div>
+        -->
+
+
+
+
         <div id="information"></div>
       </div>
 

templates/home.html

@@ -15,9 +15,24 @@
 
 <div class="container theme-showcase" role="main">
         <div class="jumbotron">
+				<div class="row">
+						<div class="col-md-4 content">
 						<h1>Gargantext</h1>
 						<p>A web platform to explore text-mining</p>
 						<a class="btn btn-primary btn-lg" href="/projects">Test Gargantext</a>
+						</div>
+						<div class="col-md-3 content">
+						</div>
+						<div class="col-md-5 content">
+								<!--
+								<h3>Project Manager:</h3> 
+								<h4><a href="http://alexandre.delanoe.org" target="blank">Alexandre Delanoë</a></h4>
+								<h3>Scientific board:</h3> 
+								<h4><a href="http://chavalarias.com" target="blank">David Chavalarias</a> and <a href="http://alexandre.delanoe.org" target="blank">Alexandre Delanoë</a></h4>
+						<h3><a href="/about/#collapseTeam" target="blank">Thanks to all the team</a></h3>
+						--!>
+						</div>
+				</div>
         </div>
 
 		<div class="container">
@@ -48,57 +63,23 @@
 <div class="row">
 
 <div class="col-md-4 content">
-				<h3><a href="#">Historic</a></h3>
-												<p>
-
-    Chapter 1.VI. -- How Gargantua was born in a strange manner.
-    Chapter 2.XXIII. -- How Pantagruel departed from Paris, hearing
-    news that the Dipsodes had invaded the land of the Amaurots; and
-    the cause wherefore the leagues are so short in France. Chapter
-    3.XLVI. -- How Pantagruel and Panurge diversely interpret the
-    words of Triboulet. Chapter 4.LV. -- How Pantagruel, being at sea,
-    heard various unfrozen words. Chapter 5.IX. -- How we arrived at
-    the island of Tools.
-
-												</p>
+				<h3><a href="#">Historic: random sentences</a></h3>
+												<p> {{ paragraph_gargantua }}</p>
 										</div>
 
 <div class="col-md-4 content">
-<h3><a href="#">Presentation</a></h3>
-												<p>
-
-										Lorem ipsum dolor sit amet, consectetur adipiscing elit,
-				sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
-				Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris
-				nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
-				reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
-				pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
-				culpa qui officia deserunt mollit anim id est laborum.
-
+<h3><a href="#">Presentation: random words</a></h3>
+												<p> {{ paragraph_lorem }}
 												</p>
 										</div>
 
   
 <div class="col-md-4 content">
-<h3><a href="#">Tutoreil</a></h3>
+<h3><a href="#">Tutoreil: random letters</a></h3>
 												<p>
+                        {{ paragraph_tutoreil }}
 												<!-- Why not French ? -->
 												<!-- find Cambridge source which inspired this --!>
-
-        Il praaît que l'odrre des ltetres dnas un mot n'a pas
-        d'iprnorotncae. La pmeirère et la drenèire letrte diovent
-        êrte à la bnnoe pclae. Le rsete peut êrte dnas un dsérorde
-        ttoal et on puet tujoruos lrie snas poribême. On ne lit
-        donc pas chuaqe ltetre en elle-mmêe, mias le mot cmome un
-        tuot. Un chnagmnet de réfretniel et nuos tarnsposns ce
-        rselutat au txete lui-mmêe: l'odrre des mtos est faiblement
-        imoprtnat copmraé au cnotxete du txete qui, lui, est copmté:
-        comptexter avec Gargantext.
-
-												
-												
-												
-												
 												</p>
 										</div>
         

static/img/logo.svg → templates/logo.svg

@@ -66,7 +66,7 @@
        inkscape:export-ydpi="454.50735"
        inkscape:export-xdpi="454.50735"
        inkscape:export-filename="/srv/gargantext/static/img/logo.png"
-       style="fill:#ff8080;fill-opacity:0.82014388"
+       style="fill:{{color}};fill-opacity:0.82014388"
        id="g3835"
        transform="matrix(0.2422549,0,0,0.23374214,-49.789462,-7.9055988)">
       <path
@@ -75,7 +75,7 @@
          inkscape:export-filename="/home/alexandre/projets/gargantext.py/gargantext_core/shared/LogoSimple.png"
          id="path3837"
          d="m 206.24721,35.28586 0,129.5 67.78125,0 0,-8.625 c -9.86526,-0.47262 -18.57934,-2.63259 -25.5625,-6.28125 -18.65918,-9.74237 -29.875,-28.26535 -29.875,-49.1875 0,-31.71741 21.11877,-52.8149 55.4375,-55.1875 l 0,-10.21875 -67.78125,0 z m 67.78125,10.21875 0,8.5 c 1.74191,-0.16369 3.53543,-0.28125 5.37499,-0.28125 6.91081,0 13.295,1.44116 19.6875,4.15625 l 2.40625,2.875 2.59375,14.53125 9.6875,0 0,-25.375 c -11.40283,-3.03451 -22.61727,-4.65625 -33.15625,-4.65625 -2.24526,0 -4.44959,0.10177 -6.59374,0.25 z m 0,8.5 c -23.28864,2.18852 -37.65625,18.81513 -37.65625,45.562503 0,27.600037 14.44681,45.025437 37.65625,47.812497 l 0,-93.375 z m 0,93.375 0,8.78125 c 1.36224,0.0653 2.75177,0.0937 4.15624,0.0937 10.19344,0 22.1324,-1.88915 35.78125,-5.5625 l 0,-38.1875 2.9375,-2.21875 9.5,-0.8125 0,-6.5625 -43.21875,0 0,6.5625 12.28125,0.8125 2.9375,2.21875 0,33.21875 c -6.73804,1.4374 -12.61466,2.09375 -17.625,2.09375 -2.32322,0 -4.57592,-0.17643 -6.74999,-0.4375 z"
-         style="font-size:166.11251831px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#ff8080;fill-opacity:0.82014388;stroke:none;font-family:Bitstream Charter;-inkscape-font-specification:Bitstream Charter"
+         style="font-size:166.11251831px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:{{color}};fill-opacity:0.82014388;stroke:none;font-family:Bitstream Charter;-inkscape-font-specification:Bitstream Charter"
          inkscape:connector-curvature="0" />
       <path
          inkscape:export-ydpi="100"
@@ -90,7 +90,7 @@
          sodipodi:cy="480.17926"
          sodipodi:cx="-321.88605"
          id="path3839"
-         style="fill:#ff8080;fill-opacity:0.82014388;stroke:none"
+         style="fill:{{color}};fill-opacity:0.82014388;stroke:none"
          sodipodi:type="arc" />
     </g>
   </g>

templates/maintenance.html

+
+{% extends "menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+<link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
+
+<script src="{% static "js/jquery/jquery.min.js" %}" type="text/javascript"></script>
+
+{% endblock %}
+
+
+
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+		<div class="jumbotron">
+				<h1>Gargantext in maintenance</h1>
+				<h2>Thanks for your comprehension</h2>
+		</div>
+
+
+
+
+
+</div>
+
+{% endblock %}

templates/menu.html

@@ -17,7 +17,7 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </button>
-					<a class="navbar-brand" style="line-height:15px; height:10px; padding: 10px 10px;" href="/"><img src="{% static "img/logo.svg" %}"></a>
+					<a class="navbar-brand" style="line-height:15px; height:10px; padding: 10px 10px;" href="/"><img src="/img/logo.svg"></a>
         </div>
         
         <div class="navbar-collapse collapse">
@@ -40,19 +40,20 @@
 
 				<ul class="nav pull-right">
 						<li class="dropdown">
-								<a href="#" role="button" class="dropdown-toggle" data-toggle="dropdown">:<i class="icon-user"></i> {{ user }}<i class="caret"></i>
+								<a href="#" role="button" class="dropdown-toggle" data-toggle="dropdown"><i class="icon-user"></i> {{ user }}<i class="caret"></i>
 
 								</a>
 								<ul class="dropdown-menu">
-										<li><a tabindex="-1" href="/auth/">Login</a></li>
-										<li><a tabindex="-1" href="#">Profile</a></li>
+										<li><a tabindex="-1" href="http://www.iscpif.fr/tiki-index.php?page=gargantext_feedback">Report Feedback</a></li>
 										<li class="divider"></li>
+								{% if user.is_authenticated %}
 										<li><a tabindex="-1" href="/auth/logout">Logout</a></li>
+										{% else %}
+										<li><a tabindex="-1" href="/auth/">Login</a></li>
+								{% endif %}
 								</ul>
 						</li>
 				</ul>
-
-          
       </div>
       </div>
     </div>
@@ -65,7 +66,8 @@
 <hr>
 
 <footer>
-        <p>Gargantext, version 1.0.5, Copyrights CNRS {{ date.year }}.</p>
+		<p>Gargantext, version 1.0.6, <a href="http://www.cnrs.fr" target="blank">Copyrights CNRS {{ date.year }}</a>, 
+		<a href="http://www.gnu.org/licenses/agpl-3.0.html" target="blank">Licence aGPLV3</a>.</p>
 </footer>
 
 

templates/project.html_backup

+
+{% extends "menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+<link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
+
+<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+
+<script src="{% static "js/raphael-min.js"%}"></script>
+<script src="{% static "js/morris.min.js"%}"></script>
+
+
+{% endblock %}
+
+
+
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+        <div class="jumbotron">
+				<div class="row">
+        <div class="col-md-6">
+            {% if project %}
+            <h1>{{ project.name }}</h1>
+						<!--<h3> {{number}} corpora </h3>-->
+            {% endif %}
+        </div>
+
+        <div class="col-md-4">
+				<p>
+				{% if donut %}
+	  		<div id="hero-donut" style="height: 200px;"></div>
+				{% endif %}
+        <center>
+            <button 
+								type="button" 
+								class="btn btn-primary btn-lg" 
+								data-container="body" 
+								data-toggle="popover" 
+								data-placement="bottom"
+								>Add a corpus</button>
+								<div id="popover-content" class="hide">
+
+					<form enctype="multipart/form-data" action="/project/{{project.id}}/" method="post">
+    {% csrf_token %}
+		{{ form.non_field_errors }}
+		{{ form.as_p}}
+		
+		{{ formResource.non_field_errors }}
+		{{ formResource.as_p}}
+		<input onclick='$("#semLoader").css("visibility", "visible"); $("#semLoader").show();' type="submit" name="submit" id="submit" class="btn" value="Add this corpus" /><div>
+    </center>
+				</p>
+
+								</div>
+		        </div>
+
+        </div>
+
+
+        </div>
+        </div>
+</div>
+<!-- Add jumbotron container for each type of corpus (presse, science etc.) -->
+
+          <div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
+							<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
+          </div>
+
+<div class="container">
+				{% if list_corpora  %}
+								<h1>Resources</h1>
+								<h2>Corpora</h2>
+										<ul>
+										{% for key, corpora in list_corpora.items %}
+										<li>{{ key }}</li>
+												<ul>
+														{% for corpus in corpora %}
+														<li> {% ifnotequal corpus.count 0 %}
+																		<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> 
+																			{{corpus.name}}
+																		</a>
+																		, {{ corpus.count }} Documents 
+																 {% else %}
+																 {{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
+																 {% endifnotequal %}
+																		<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" 
+																		data-content='
+																		<ul>
+																		<li> Rename </li>
+																		<li> Add new documents </li>
+																		<li><a href="/project/{{ project.id }}/corpus/{{ corpus.id}}/delete">Delete</a></li>
+																		</ul>
+																		'>Manage</button>
+																</li>
+														{% endfor %}
+												</ul>
+										{% endfor %}
+										</ul>
+				{% endif %}
+
+
+								{% if list_corporax  %}
+										<div class="col-md-4">
+                        <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}">{{corpus.name}}</a>
+												</h3>
+												<h4>{{ corpus.count }} Documents </h4>
+												<h5>Activity:</h5>
+												<div class="chart" data-percent="73">73%</div>
+										</div>
+								{% endif %}
+				
+						{% if whitelists  %}
+						<h2>Lists of Ngrams</h2>
+								<h3>White Lists</h2>
+								{% for list in whitelists %}
+										<ul>
+                    <li> {{list.name }}
+										</ul>
+								{% endfor %}
+						{% endif %}
+						
+            {% if whitelists  %}
+						<h3>Black Lists</h2>
+								{% for list in blacklists %}
+										<ul>
+                    <li> {{list.name }}
+										</ul>
+								{% endfor %}
+						{% endif %}
+						
+
+				
+        {% if cooclists  %}
+						<h2>Results (graphs)</h2>
+						<h3>Cooccurrences Lists</h2>
+								{% for list in cooclists %}
+										<ul>
+                    <li> {{list.name }}
+										</ul>
+								{% endfor %}
+						{% endif %}
+			  
+</div>
+
+
+<script>
+        // Morris Donut Chart
+        Morris.Donut({
+            element: 'hero-donut',
+            data: [
+            {% if donut %}
+            {% for part in donut %}
+						{label: '{{ part.source }}', value: {{ part.part }} },
+            {% endfor %}
+            {% endif %}
+
+            ],
+            colors: ["@white", "@white"],
+            //colors: ["#30a1ec", "#76bdee"],
+            formatter: function (y) { return y + "%" }
+        });
+</script>
+
+
+
+
+{% endblock %}

templates/subcorpus.html

@@ -63,7 +63,7 @@ function deleteDuplicates(url) {
       success: function(data) {
         console.log("in DeleteDuplicates")
         console.log(data)
-        $("#delAll").remove();
+        location.reload();
       },
         error: function(result) {
             console.log("Data not found");

templates/tests/mvc.html

@@ -240,6 +240,7 @@
                             <li ng-repeat="filter in filters">
                                 <button ng-click="removeFilter($index)" title="remove this filter">x</button>
                                 <span>...where the </span>
+
                                 <select ng-model="filter.entity" ng-options="entity as entity.key for entity in entities"></select>
                                 <span ng-if="filter.entity.key != 'ngrams'">
                                     <select ng-if="filter.entity" ng-model="filter.column" ng-options="column as column.key for column in filter.entity.columns | orderBy:'key'"></select>