[FIX] merge in README.

analysis/diachronic_specificity.py

+import sqlalchemy
+from gargantext_web import api
+from node import models
+from sqlalchemy import create_engine
+from sqlalchemy.sql import func
+import numpy as np
+import collections
+
+NodeType = models.NodeType.sa
+NodeNgram = models.Node_Ngram.sa
+NodeNodeNgram = models.NodeNgramNgram.sa
+Ngram = models.Ngram.sa
+Node = models.Node.sa
+Corpus = models.Corpus.sa
+
+def get_session():
+    import sqlalchemy.orm
+    from django.db import connections
+    from sqlalchemy.orm import sessionmaker
+    from aldjemy.core import get_engine
+    alias = 'default'
+    connection = connections[alias]
+    engine = create_engine("postgresql+psycopg2://alexandre:C8kdcUrAQy66U@localhost/gargandb",
+            use_native_hstore=True)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+session = get_session()
+
+
+def result2dict(query):
+    results = dict()
+    for result in query:
+        if result[0] is not None:
+            results[result[0]] = result[1]
+    return(results)
+
+
+def diachronic_specificity(corpus_id, string, order=True):
+    ''' 
+    Take as parameter Corpus primary key and text of ngrams.
+    Result is a dictionnary.
+    Keys are period (years for now)
+    Values are measure to indicate diachronic specificity.
+    Nowadays, the measure is rather simple: distance of frequency of period from mean of frequency of all corpus.
+    '''
+    corpus = session.query(Node).get(int(corpus_id))
+    ngram = session.query(Ngram).filter(Ngram.terms == string).first()
+    
+    ngram_frequency_query = session.query(Node.metadata['publication_year'], func.count('*'))                        .join(NodeNgram, Node.id == NodeNgram.node_id)                        .filter( NodeNgram.ngram == ngram)                        .filter(Node.parent_id == corpus.id)                        .group_by(Node.metadata['publication_year']).all()
+
+    document_year_sum_query = session.query(Node.metadata['publication_year'], func.count('*'))            .filter(Node.parent_id == corpus.id)            .group_by(Node.metadata['publication_year']).all()
+            
+            
+    document_filterByngram_year = result2dict(ngram_frequency_query)
+    document_all_year = result2dict(document_year_sum_query)
+    #print(document_all_year)
+    data = dict()
+    
+    for year in document_all_year.keys():
+       data[year] = document_filterByngram_year.get(year, 0) / document_all_year[year]
+    
+    mean = np.mean(list(data.values()))
+    
+    data_dict = dict(zip(data.keys(), list(map(lambda x: x - mean, data.values()))))
+    
+    if order == True:
+        return collections.OrderedDict(sorted(data_dict.items()))
+    else:
+        return data_dict
+
+
+# For tests
+#diachronic_specificity(102750, "bayer", order=True)
+
+

analysis/functions.py

@@ -54,6 +54,9 @@ def create_whitelist(user, corpus, size=100):
             n.type_id = %d
         AND
         ngX.n >= 2
+        AND
+        ngX.n <= 3
+
 
         GROUP BY
             ngX.id
@@ -241,27 +244,31 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', n=150
     return data
 
 
-def tfidf(corpus, document, ngram):
-    try:
-        occurences_of_ngram = Node_Ngram.objects.get(node=document, ngram=ngram).weight
-        ngrams_by_document = sum([ x.weight for x in Node_Ngram.objects.filter(node=document)])
-        term_frequency = occurences_of_ngram / ngrams_by_document
-    
-        xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
-        yy = Node_Ngram.objects.filter(ngram=ngram).count()
-        inverse_d_frequency= log(xx/yy)
-        
-        # result = tf * idf
-        result = term_frequency * inverse_d_frequency
-    except Exception as error:
-        print(error)
-        result = 0
-    return result
-
-
+#def tfidf(corpus, document, ngram):
+#    '''
+#    Compute TF-IDF (Term Frequency - Inverse Document Frequency)
+#    See: http://en.wikipedia.org/wiki/Tf%E2%80%93idf
+#    '''
+#    try:
+#        occurences_of_ngram = Node_Ngram.objects.get(node=document, ngram=ngram).weight
+#        ngrams_by_document = sum([ x.weight for x in Node_Ngram.objects.filter(node=document)])
+#        term_frequency = occurences_of_ngram / ngrams_by_document
+#    
+#        xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
+#        yy = Node_Ngram.objects.filter(ngram=ngram).count() # filter: ON node.parent=corpus
+#        inverse_document_frequency= log(xx/yy)
+#        
+#        # result = tf * idf
+#        result = term_frequency * inverse_document_frequency
+#    except Exception as error:
+#        print(error, ngram)
+#        result = 0
+#    return result
+
+from analysis.tfidf import tfidf
 
 def do_tfidf(corpus, reset=True):
-
+    print("doing tfidf")
     with transaction.atomic():
         if reset==True:
             NodeNodeNgram.objects.filter(nodex=corpus).delete()

analysis/tfidf.py

+
+import sqlalchemy
+from gargantext_web import api
+from node import models
+from sqlalchemy import create_engine
+from sqlalchemy.sql import func
+import numpy as np
+import collections
+from math import log
+NodeType = models.NodeType.sa
+NodeNgram = models.Node_Ngram.sa
+NodeNodeNgram = models.NodeNgramNgram.sa
+Ngram = models.Ngram.sa
+Node = models.Node.sa
+Corpus = models.Corpus.sa
+
+def get_session():
+    import sqlalchemy.orm
+    from django.db import connections
+    from sqlalchemy.orm import sessionmaker
+    from aldjemy.core import get_engine
+    alias = 'default'
+    connection = connections[alias]
+    engine = create_engine("postgresql+psycopg2://alexandre:C8kdcUrAQy66U@localhost/gargandb",
+            use_native_hstore=True)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+session = get_session()
+
+type_doc = session.query(NodeType).filter(NodeType.name == "Document").first()
+
+def tfidf(corpus, document, ngram):
+    ''' 
+    Compute TF-IDF (Term Frequency - Inverse Document Frequency)
+    See: http://en.wikipedia.org/wiki/Tf%E2%80%93idf
+    '''
+    try:
+        #occurences_of_ngram = Node_Ngram.objects.get(node=document, ngram=ngram).weight
+        occurrences_of_ngram = session.query(NodeNgram)\
+                .filter(NodeNgram.node_id == document.id)\
+                .filter(NodeNgram.ngram_id == ngram.id)\
+                .first().weight
+        
+        #return(type(occurrences_of_ngram))
+        #ngrams_by_document = np.sum([ x.weight for x in Node_Ngram.objects.filter(node=document)])
+        ngrams_by_document = session.query(NodeNgram).filter(NodeNgram.node_id == document.id).count()
+        
+        term_frequency = occurrences_of_ngram / ngrams_by_document
+        #return term_frequency 
+        
+        #xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
+        xx = session.query(Node)\
+                .filter(Node.parent_id == corpus.id)\
+                .filter(Node.type_id   == type_doc.id)                .count()
+        #yy = Node_Ngram.objects.filter(ngram=ngram).count() # filter: ON node.parent=corpus
+        yy = session.query(NodeNgram)\
+                .join(Node, NodeNgram.node_id == Node.id)\
+                .filter(Node.parent_id == corpus.id)\
+                .filter(NodeNgram.ngram_id == ngram.id)\
+                .count()
+        
+        inverse_document_frequency= log(xx/yy)
+
+        # result = tf * idf
+        result = term_frequency * inverse_document_frequency
+ 
+        return result
+    except Exception as error:
+        print(error)
+
+
+#corpus = session.query(Node).get(int(102750))
+#ngram  = session.query(Ngram).get(10885)
+##ngram = session.query(Ngram).filter(Ngram.terms == "bayer").first()
+#type_doc = session.query(NodeType).filter(NodeType.name == "Document").first()
+#doc_id  = session.query(NodeNgram.node, NodeNgram.node_id)\
+#        .join(Node, Node.id == NodeNgram.node_id)\
+#        .filter(NodeNgram.ngram == ngram)\
+#        .filter(Node.type_id == type_doc.id)\
+#        .first()
+#document = session.query(Node).get(doc_id[1])
+#
+#result = tfidf(corpus,document, ngram)
+#print(result)
+#
+
+
+

gargantext.ini

@@ -40,7 +40,7 @@ vacuum          = true
 pidfile = /tmp/gargantext.pid
 
 # respawn processes taking more than 20 seconds
-harakiri = 20
+harakiri = 120
 
 # limit the project to 128 MB
 #limit-as = 128
@@ -51,4 +51,5 @@ max-requests = 5000
 # background the process & log
 #daemonize = /var/log/uwsgi/gargantext.log
 
-
+uid = 1000
+gid = 1000

gargantext_web/api.py

@@ -277,20 +277,23 @@ class NodesChildrenDuplicates(APIView):
         # get the minimum ID for each of the nodes sharing the same metadata
         kept_node_ids_query = self._fetch_duplicates(request, node_id, [func.min(Node.id).label('id')], 0)
         kept_node_ids = [kept_node.id for kept_node in kept_node_ids_query]
-        # delete the stuff
-        delete_query = (session
-            .query(Node)
-            .filter(Node.parent_id == node_id)
-            .filter(~Node.id.in_(kept_node_ids))
-        )
-        count = delete_query.count()
-        delete_query.delete(synchronize_session=False)
-        session.flush()
-        # return the result
+        duplicate_nodes =  models.Node.objects.filter( parent_id=node_id ).exclude(id__in=kept_node_ids)
+        # # delete the stuff
+        # delete_query = (session
+        #     .query(Node)
+        #     .filter(Node.parent_id == node_id)
+        #     .filter(~Node.id.in_(kept_node_ids))
+        # )
+        count = len(duplicate_nodes)
+        for node in duplicate_nodes:
+            print("deleting node ",node.id)
+            node.delete()
+        # print(delete_query)
+        # # delete_query.delete(synchronize_session=True)
+        # session.flush()
         return JsonHttpResponse({
-            'deleted': count,
+            'deleted': count
         })
-        # return duplicates_query
 
 
 
@@ -663,7 +666,21 @@ class Nodes(APIView):
             'metadata': dict(node.metadata),
         })
 
+    # deleting node by id
+    # currently, very dangerous
+    def delete(self, request, node_id):
+        session = get_session()
+        node = models.Node.objects.filter(id = node_id)
+        msgres = ""
+        try:
+            node.delete()
+            msgres = node_id+" deleted!"
+        except:
+            msgres ="error deleting: "+node_id
 
+        return JsonHttpResponse({
+            'deleted': msgres,
+        })
 
 class CorpusController:
 

gargantext_web/settings.py

@@ -10,7 +10,7 @@ https://docs.djangoproject.com/en/1.6/ref/settings/
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
-BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 PROJECT_PATH = os.path.join(BASE_DIR, os.pardir)
 PROJECT_PATH = os.path.abspath(PROJECT_PATH)
 
@@ -33,7 +33,16 @@ TEMPLATE_DEBUG = True
 
 
 TEMPLATE_DIRS = (
-   os.path.join(BASE_DIR, 'templates'),
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes
+    # Don't forget to use absolute paths, not relative paths.
+    '/srv/gargantext/templates',
+
+#import os.path
+#
+#TEMPLATE_DIRS = (
+#    os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/'),
+#)
 )
 
 
@@ -112,11 +121,11 @@ USE_TZ = True
 
 ROOT_URLCONF = 'gargantext_web.urls'
 
-STATIC_ROOT = os.path.join(PROJECT_PATH, 'static')
+STATIC_ROOT = '/var/www/gargantext/static/'
 STATIC_URL = '/static/'
 
-# MEDIA_ROOT = '/var/www/gargantext/media'
-MEDIA_ROOT = os.path.join(PROJECT_PATH, 'media')
+MEDIA_ROOT = '/var/www/gargantext/media'
+#MEDIA_ROOT = os.path.join(PROJECT_PATH, 'media')
 MEDIA_URL   = '/media/'
 
 
@@ -127,8 +136,11 @@ STATICFILES_FINDERS = (
 
 
 STATICFILES_DIRS = (
-    os.path.join(BASE_DIR, "static"),
-)
+            #os.path.join(BASE_DIR, "static"),
+                '/srv/gargantext/static',
+                #'/var/www/www/alexandre/media',
+                #'/var/www/alexandre.delanoe.org/',
+                )
 
 TEMPLATE_CONTEXT_PROCESSORS = (
     "django.contrib.auth.context_processors.auth",

gargantext_web/team.py

+# -*- coding: utf-8 -*-
+
+# Order in script: Alphabetical order (first_name, name, mail, website)
+# Order in public: Shuffled order
+
+import random
+
+def get_team():
+    ''' 
+    Function to get list of each member as dict of personal informations.
+    
+    You are free to fill the form which is verbose indeed but clear enough for
+    manual entries (I could zip lists but not clear enough).
+    
+    For your picture, please ask Alexandre to take your picture with his camera
+    in order to follow the design shape of the website.
+
+    '''
+    team = [
+            { 'first_name' : 'Alexandre', 'last_name' : 'Delanoë', 'mail' : 'alexandre+gargantextATdelanoe.org', 'website' : 'http://alexandre.delanoe.org', 'picture' : 'alexandre.jpg'},
+            { 'first_name' : 'David', 'last_name' : 'Chavalarias', 'mail' : '', 'website' : 'http://chavalarias.com', 'picture' : 'david.jpg'},
+            { 'first_name' : 'Mathieu', 'last_name' : 'Rodic', 'mail' : '', 'website'  : 'http://rodic.fr', 'picture' : 'mathieu.jpg'},
+            { 'first_name' : 'Samuel', 'last_name' : 'Castillo J.', 'mail' : 'kaisleanATgmail.com', 'website'  : 'http://www.pksm3.droppages.com', 'picture' : 'samuel.jpg'},
+            { 'first_name' : 'Elias', 'last_name' : 'Showk', 'mail' : '', 'website' : 'https://github.com/elishowk', 'picture' : ''},
+            #{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''},
+            # copy paste the line above and write your informations please
+            ]
+
+    random.shuffle(team)
+    return(team)
+
+def get_sponsors():
+    ''' 
+    Function to get list of each sponsor as dict of institutional informations.
+
+    '''
+    sponsors = [
+            { 'name' : 'Mines ParisTech', 'website' : 'http://mines-paristech.fr', 'picture' : 'logo.png'},
+            # copy paste the line above and write your informations please
+            ]
+
+    random.shuffle(sponsors)
+    return(sponsors)
+

gargantext_web/urls.py

@@ -16,9 +16,13 @@ urlpatterns = patterns('',
     url(r'^admin/', include(admin.site.urls)),
     url(r'^login/', include(admin.site.urls)),
     url(r'^grappelli/', include('grappelli.urls')),
+
+    url(r'^auth/$', views.login_user),
+    url(r'^auth/logout/$', views.logout_user),
     
     # User Home view
     url(r'^$', views.home),
+    url(r'^about/', views.about),
     
     # Project Management
     url(r'^projects/$', views.projects),
@@ -35,8 +39,8 @@ urlpatterns = patterns('',
 
     # Visualizations
     url(r'^project/(\d+)/corpus/(\d+)/chart$', views.chart),
-    url(r'^corpus/(\d+)/explorer$', views.graph),
-    url(r'^corpus/(\d+)/matrix$', views.matrix),
+    url(r'^project/(\d+)/corpus/(\d+)/explorer$', views.graph),
+    url(r'^project/(\d+)/corpus/(\d+)/matrix$', views.matrix),
     
     # Data management
     url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
@@ -50,9 +54,10 @@ urlpatterns = patterns('',
     url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
     url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
     url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
+    # url(r'^api/nodes/(\d+)/children/duplicates/delete$', gargantext_web.api.NodesChildrenDuplicates.delete ),
     url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
     url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
-    
+
     url(r'^api/project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpusJSON),
 
     url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
@@ -60,7 +65,7 @@ urlpatterns = patterns('',
     url(r'^ngrams$', views.ngrams),
     url(r'^nodeinfo/(\d+)$', views.nodeinfo),
     url(r'^tests/mvc$', views.tests_mvc),
-    url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments),
+    url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments)
 )
 
 

gargantext_web/views.py

@@ -9,7 +9,7 @@ from node.models import Language, ResourceType, Resource, \
         Node, NodeType, Node_Resource, Project, Corpus, \
         Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram
 
-from node.admin import CorpusForm, ProjectForm, ResourceForm
+from node.admin import CorpusForm, ProjectForm, ResourceForm, CustomForm
 
 from django.contrib.auth.models import User
 
@@ -27,6 +27,35 @@ from parsing.FileParsers import *
 
 # SOME FUNCTIONS
 
+from gargantext_web.settings import DEBUG
+from django.http import *
+from django.shortcuts import render_to_response,redirect
+from django.template import RequestContext
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth import authenticate, login, logout
+
+def login_user(request):
+    logout(request)
+    username = password = ''
+    print(request)
+    if request.POST:
+        username = request.POST['username']
+        password = request.POST['password']
+
+        user = authenticate(username=username, password=password)
+        if user is not None:
+
+            if user.is_active:
+                login(request, user)
+                return HttpResponseRedirect('/projects/')
+    return render_to_response('authentication.html', context_instance=RequestContext(request))
+
+
+def logout_user(request):
+    logout(request)
+    return HttpResponseRedirect('/')
+    # Redirect to a success page.
+
 def query_to_dicts(query_string, *query_args):
     """Run a simple query and produce a generator
     that returns the results as a bunch of dictionaries
@@ -68,6 +97,25 @@ def date_range(start_dt, end_dt = None, format=None):
 
 # SOME VIEWS
 
+from gargantext_web import team
+def about(request):
+    '''
+    About Gargantext, the team and sponsors
+    '''
+    template = get_template('about.html')
+    user = request.user
+    date = datetime.datetime.now()
+    members = team.get_team()
+
+    html = template.render(Context({\
+            'user': user,\
+            'date': date,\
+            'team': members,\
+            }))
+    
+    return HttpResponse(html)
+
+
 def home(request):
     '''
     Home describes the platform.
@@ -92,7 +140,7 @@ def projects(request):
     To each project, we can link a resource that can be an image.
     '''
     if not request.user.is_authenticated():
-        return redirect('/admin/logout/?next=%s' % request.path)
+        return redirect('/auth/')
     
     t = get_template('projects.html')
     
@@ -121,6 +169,7 @@ def projects(request):
         'projects': projects
         })
 
+
 def project(request, project_id):
     '''
     This view represents all corpora in a panoramic way.
@@ -186,76 +235,172 @@ def project(request, project_id):
 
 
     if request.method == 'POST':
-        #form = CorpusForm(request.POST, request.FILES)
-        #print(str(request.POST))
-        name        = str(request.POST['name'])
-
-        try:
-            resource_type = ResourceType.objects.get(id=str(request.POST['type']))
-        except Exception as error:
-            print(error)
-            resource_type = None
-        
-        try:
-            file = request.FILES['file']
-        except Exception as error:
-            print(error)
-            file = None
-
-        #if name != "" and resource_type is not None and file is not None:
-        try:
-            parent      = Node.objects.get(id=project_id)
-            node_type   = NodeType.objects.get(name='Corpus')
-            
-            if resource_type.name == "europress_french":
-                language    = Language.objects.get(iso2='fr')
-            elif resource_type.name == "europress_english":
-                language    = Language.objects.get(iso2='en')
+        print("original file:")
+        print(request.FILES)
+
+        form = CustomForm(request.POST, request.FILES)
+        if form.is_valid():
+
+
+            name = form.cleaned_data['name']
+            thefile = form.cleaned_data['file']
+            resource_type = ResourceType.objects.get(id=str( form.cleaned_data['type'] ))
+
+            print("-------------")
+            print(name,"|",resource_type,"|",thefile)
+            print("-------------")
             
+            print("new file:")
+            print(thefile)
+
             try:
-                corpus = Node(
-                        user=request.user,
-                        parent=parent,
-                        type=node_type,
-                        language=language,
-                        name=name,
-                        )
-            except:
-                corpus = Node(
+                parent      = Node.objects.get(id=project_id)
+                node_type   = NodeType.objects.get(name='Corpus')
+                
+                if resource_type.name == "europress_french":
+                    language    = Language.objects.get(iso2='fr')
+                elif resource_type.name == "europress_english":
+                    language    = Language.objects.get(iso2='en')
+                
+                try:
+                    corpus = Node(
+                            user=request.user,
+                            parent=parent,
+                            type=node_type,
+                            language=language,
+                            name=name,
+                            )
+                except:
+                    corpus = Node(
+                            user=request.user,
+                            parent=parent,
+                            type=node_type,
+                            name=name,
+                            )
+
+                corpus.save()
+
+                print(request.user, resource_type , thefile )
+
+                corpus.add_resource(
                         user=request.user,
-                        parent=parent,
-                        type=node_type,
-                        name=name,
+                        type=resource_type,
+                        file=thefile
                         )
 
-            corpus.save()
-            print(corpus.language)
-            corpus.add_resource(
-                    user=request.user,
-                    type=resource_type,
-                    file=file
-                    )
-
-            try:
-                corpus.workflow()
-                #corpus.workflow((), countdown=3)
+                try:
+                    #corpus.parse_and_extract_ngrams()
+                    #corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
+                    if DEBUG is True:
+                        corpus.workflow()
+                    else:
+                        corpus.workflow.apply_async((), countdown=3)
 
-            except Exception as error:
-                print(error)
+                except Exception as error:
+                    print(error)
 
-            return HttpResponseRedirect('/project/' + str(project_id))
-        except Exception as error:
-            print('ee', error)
-            form = CorpusForm(request=request)
-            formResource = ResourceForm()
+                return HttpResponseRedirect('/project/' + str(project_id))
 
+                
+            except Exception as error:
+                print('ee', error)
+                form = CorpusForm(request=request)
+                formResource = ResourceForm()
+
+
+        else:
+            print("bad form, bad form")
+            return render(request, 'project.html', {
+                    'form'          : form,
+                    'user'          : user,
+                    'date'          : date,
+                    'project'       : project,
+                    'donut'         : donut,
+                    'list_corpora'  : list_corpora,
+                    'whitelists'    : whitelists,
+                    'blacklists'    : blacklists,
+                    'cooclists'     : cooclists,
+                    'number'        : number,
+                })
     else:
-        form = CorpusForm(request=request)
-        formResource = ResourceForm()
+        form = CustomForm()
+
+
+    # if request.method == 'POST':
+    #     #form = CorpusForm(request.POST, request.FILES)
+    #     #print(str(request.POST))
+    #     name        = str(request.POST['name'])
+    #     try:
+    #         resource_type = ResourceType.objects.get(id=str(request.POST['type']))
+    #     except Exception as error:
+    #         print(error)
+    #         resource_type = None
+        
+    #     try:
+    #         file = request.FILES['file']
+    #     except Exception as error:
+    #         print(error)
+    #         file = None
+
+    #     #if name != "" and resource_type is not None and file is not None:
+    #     try:
+    #         parent      = Node.objects.get(id=project_id)
+    #         node_type   = NodeType.objects.get(name='Corpus')
+            
+    #         if resource_type.name == "europress_french":
+    #             language    = Language.objects.get(iso2='fr')
+    #         elif resource_type.name == "europress_english":
+    #             language    = Language.objects.get(iso2='en')
+            
+    #         try:
+    #             corpus = Node(
+    #                     user=request.user,
+    #                     parent=parent,
+    #                     type=node_type,
+    #                     language=language,
+    #                     name=name,
+    #                     )
+    #         except:
+    #             corpus = Node(
+    #                     user=request.user,
+    #                     parent=parent,
+    #                     type=node_type,
+    #                     name=name,
+    #                     )
+
+    #         corpus.save()
+
+    #         print(request.user, resource_type , file )
+    #         print(corpus.language)
+    #         corpus.add_resource(
+    #                 user=request.user,
+    #                 type=resource_type,
+    #                 file=file
+    #                 )
+
+    #         try:
+    #             #corpus.parse_and_extract_ngrams()
+    #             #corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
+    #             if DEBUG is True:
+    #                 corpus.workflow()
+    #             else:
+    #                 corpus.workflow.apply_async((), countdown=3)
+
+    #         except Exception as error:
+    #             print(error)
+
+    #         return HttpResponseRedirect('/project/' + str(project_id))
+    #     except Exception as error:
+    #         print('ee', error)
+    #         form = CorpusForm(request=request)
+    #         formResource = ResourceForm()
+
+    # else:
+    #     form = CorpusForm(request=request)
+    #     formResource = ResourceForm()
        
     return render(request, 'project.html', {
             'form'          : form,
-            'formResource'  : formResource,
             'user'          : user,
             'date'          : date,
             'project'       : project,
@@ -307,41 +452,41 @@ def corpus(request, project_id, corpus_id):
 #    except:
 #        sources_donut = []
     # Do a javascript query/api for that
-    query_date = """
-        SELECT
-            id,
-            metadata -> 'publication_year' as year,
-            metadata -> 'publication_month' as month, 
-            metadata -> 'publication_day' as day,
-            metadata -> 'title'
-        FROM
-            node_node AS n
-        WHERE
-            n.parent_id = %d
-        ORDER BY
-            year, month, day DESC
-        LIMIT
-            20
-        OFFSET
-            %d
-    """ % (corpus.id, 0)
-    try:
-        cursor = connection.cursor()
-
-        cursor.execute(query_date)
-        documents = list()
-        while True:
-            document = dict()
-            row = cursor.fetchone()
-            
-            if row is None:
-                break
-            document['id']      = row[0]
-            document['date']    = row[1] + '/' + row[2] + '/' + row[3]
-            document['title']   = row[4]
-            documents.append(document)
-    except Exception as error:
-        print(error)
+#    query_date = """
+#        SELECT
+#            id,
+#            metadata -> 'publication_year' as year,
+#            metadata -> 'publication_month' as month, 
+#            metadata -> 'publication_day' as day,
+#            metadata -> 'title'
+#        FROM
+#            node_node AS n
+#        WHERE
+#            n.parent_id = %d
+#        ORDER BY
+#            year, month, day DESC
+#        LIMIT
+#            20
+#        OFFSET
+#            %d
+#    """ % (corpus.id, 0)
+#    try:
+#        cursor = connection.cursor()
+#
+#        cursor.execute(query_date)
+#        documents = list()
+#        while True:
+#            document = dict()
+#            row = cursor.fetchone()
+#            
+#            if row is None:
+#                break
+#            document['id']      = row[0]
+#            document['date']    = row[1] + '/' + row[2] + '/' + row[3]
+#            document['title']   = row[4]
+#            documents.append(document)
+#    except Exception as error:
+#        print(error)
 
     try:
         chart = dict()
@@ -356,7 +501,7 @@ def corpus(request, project_id, corpus_id):
             'date': date,\
             'project': project,\
             'corpus' : corpus,\
-            'documents': documents,\
+#            'documents': documents,\
             'number' : number,\
             'dates' : chart,\
             }))
@@ -364,8 +509,6 @@ def corpus(request, project_id, corpus_id):
     return HttpResponse(html)
 
 
-
-
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 def subcorpus(request, project_id, corpus_id, start , end ):
     if not request.user.is_authenticated():
@@ -390,10 +533,11 @@ def subcorpus(request, project_id, corpus_id, start , end ):
     
     project = Node.objects.get(id=project_id)
     corpus = Node.objects.get(id=corpus_id)
-    
+    type_document = NodeType.objects.get(name="Document")
     # retrieving all the documents
-    documents  = corpus.children.all()
-    number = corpus.children.count()
+    # documents  = corpus.children.all()
+    documents  = corpus.__class__.objects.filter(parent_id=corpus_id , type = type_document )
+    number = len(documents)
 
     filtered_docs = []
     # filtering documents by range-date
@@ -406,7 +550,7 @@ def subcorpus(request, project_id, corpus_id, start , end ):
                 filtered_docs.append(doc)
 
     # ordering from most recent to the older.
-    ordered = sorted(filtered_docs, key=lambda x: x.date, reverse=True)
+    ordered = sorted(filtered_docs, key=lambda x: x.date)
 
     # pages of 10 elements. Like a sir.
     paginator = Paginator(ordered, 10)
@@ -458,10 +602,11 @@ def subcorpusJSON(request, project_id, corpus_id, start , end ):
     
     project = Node.objects.get(id=project_id)
     corpus = Node.objects.get(id=corpus_id)
-    
+    type_document = NodeType.objects.get(name="Document")
     # retrieving all the documents
-    documents  = corpus.children.all()
-    number = corpus.children.count()
+    # documents  = corpus.children.all()
+    documents  = corpus.__class__.objects.filter(parent_id=corpus_id , type = type_document )
+    number = len(documents)
 
     filtered_docs = []
     # filtering documents by range-date
@@ -518,38 +663,48 @@ def chart(request, project_id, corpus_id):
     t = get_template('chart.html')
     user = request.user
     date = datetime.datetime.now()
+    
     project = Node.objects.get(id=project_id)
+    corpus  = Node.objects.get(id=corpus_id)
+    
     html = t.render(Context({
-        'user': user,
-        'date': date,
-        'project' : project,
+        'user'      : user,
+        'date'      : date,
+        'project'   : project,
+        'corpus'    : corpus,
     }))    
     return HttpResponse(html)
 
-def matrix(request, corpus_id):
+def matrix(request, project_id, corpus_id):
     t = get_template('matrix.html')
     user = request.user
     date = datetime.datetime.now()
+    
+    project = Node.objects.get(id=project_id)
     corpus = Node.objects.get(id=corpus_id)
 
     html = t.render(Context({\
-            'user': user,\
-            'date': date,\
-            'corpus': corpus,\
+            'user'      : user,\
+            'date'      : date,\
+            'corpus'    : corpus,\
+            'project'   : project,\
             }))
     
     return HttpResponse(html)
 
-def graph(request, corpus_id):
+def graph(request, project_id, corpus_id):
     t = get_template('explorer.html')
     user = request.user
     date = datetime.datetime.now()
+    
+    project = Node.objects.get(id=project_id)
     corpus = Node.objects.get(id=corpus_id)
 
     html = t.render(Context({\
-            'user': user,\
-            'date': date,\
-            'corpus': corpus,\
+            'user'      : user,\
+            'date'      : date,\
+            'corpus'    : corpus,\
+            'project'   : project,\
             }))
     
     return HttpResponse(html)
@@ -666,8 +821,9 @@ def node_link(request, corpus_id):
     Create the HttpResponse object with the node_link dataset.
     '''
 
-   
+    print("In node_link() START")
     data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
+    print("In node_link() END")
     return JsonHttpResponse(data)
 
 def adjacency(request, corpus_id):

init/README.rst

@@ -139,9 +139,20 @@ Start the Django server
 -----------------------
 in bash to launch python env : /srv/gargantext_env/bin/activate
 In Pyvenv:
-python manage.py runserver
+$ python manage.py runserver
 
 
+For Production Server
+---------------------
+
+git checkout stable
+
+$ sudo aptitude install rabbitmq-server
+$ sudo aptitude install tmux
+# In your python envrionment:
+$ tmux -c ./manage.py celery worker --loglevel=info
+$ python manage.py runserver
+
 Versions on git
 ---------------
 
@@ -149,4 +160,3 @@ stable branch    : current version  for production server with nginx config (and
 testing branch   : current version for users' tests
 unstable branch  : current version for developers
 
-

init/README_old.txt

-Here informations for installation:
------------------------------------
-
-
-I - Local Installation
-All informations are in init folder
-
-1) Create virtualenv with python version 3.4
-
-2) pip install -r requirements.txt
-
-3) Manually install nltk (inside the sources)
-	nltk==3.0a4
-
-4) adapt database configuration in gargantext_web settings 
-(Development done on postgresql)
-
-5) source env/bin/activate
-
-6) ./manage.py runserver
-
-That's all

init_accounts.py

+from node.models import User
+from django.core.mail import send_mail
+
+def notify_user(username, email, password):
+    message = '''
+    Bonjour,
+    votre compte vient d'être créé.
+
+    Vous pouvez désormais vous connecter ici:
+    http://mines.gargantext.org
+
+    Votre login est: %s
+    Votre mot de passe est : %s
+
+    Nous restons votre disposition pour tout complément d'information.
+    Cordialement
+    --
+        L'équipe de Gargantext (CNRS)
+
+    ''' % (username, password)
+
+    #send_mail('[Gargantext] Votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], fail_silently=False )
+    send_mail('[Gargantext] Votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], ['alexandre@delanoe.org'] )
+    # add option for mass sending email
+
+def create_user(username, email, password=None, active=False, notify=True):
+    user = User()
+    user.username = username
+    user.email = email
+    user.active_user = active
+    if password is None:
+        password = User.objects.make_random_password()
+        print(password)
+    user.set_password(password)
+    user.save()
+    
+    if notify == True:
+        notify_user(username, email, password)
+    
+    return user
+
+def delete_user(username):
+    user = User.objects.get(username=username)
+    user.delete()
+
+def active_user(username, active=True):
+    '''
+    To get inactive, active=False
+    '''
+    user = User.objects.get(username=username)
+    user.active_user = active
+    user.save()
+
+def mines_account_creation(fichier=None):
+    if fichier is None:
+        fichier = "/home/alexandre/projets/forccast/Tutorat/2014-2015/comptes_gargantext.csv"
+    accounts = open(fichier, "r")
+    for line in accounts.readlines():
+        username, email, password = line.split(',')
+        create_user(username, email, password=password, notify=True)
+        #delete_user(username)
+    accounts.close()
+

node/admin.py

@@ -92,6 +92,49 @@ class ResourceForm(ModelForm):
         model = Resource
         exclude = ['user', 'guid', 'digest']
 
+
+# for formexample.html
+from django import forms
+
+from django.utils.translation import ugettext_lazy as _
+class CustomForm(forms.Form):
+    name = forms.CharField( label='Name', max_length=199 , required=True)
+    parsing_options = ResourceType.objects.all().values_list('id', 'name')
+    type = forms.IntegerField( widget=forms.Select( choices= parsing_options) , required=True )
+    file = forms.FileField()
+
+
+
+    # Description: clean_file()
+    """
+        * file_.content_type - Example: ['application/pdf', 'image/jpeg']
+        * len(file_) - file size.
+            2.5MB - 2621440
+            5MB - 5242880
+            10MB - 10485760
+            20MB - 20971520
+            50MB - 5242880
+            100MB 104857600
+            250MB - 214958080
+            500MB - 429916160
+    """
+    def clean_file(self):
+        file_ = self.cleaned_data.get('file')
+        #Filename length
+        if len(file_.name)>30:
+            from datetime import datetime
+            file_.name = str(datetime.now().microsecond)
+            # raise forms.ValidationError(_('Come on dude, name too long. Now is:'+file_.name))
+        #File size
+        if len(file_)>104857600:
+            raise forms.ValidationError(_('File to heavy! (<100MB).'))
+        ## File type:
+        # if file_.content_type == "application/zip":
+        #     raise forms.ValidationError(_('We need a zip pls.'))
+        return file_
+
+
+
 class CorpusForm(ModelForm):
     #parent = ModelChoiceField(EmptyQuerySet)
     def __init__(self, *args, **kwargs):
@@ -99,12 +142,6 @@ class CorpusForm(ModelForm):
             self.request = kwargs.pop('request', None)
             super(CorpusForm, self).__init__(*args, **kwargs)
             parent_type = NodeType.objects.get(name="Project")
-            #parent_type = NodeType.objects.get(name=self._parent_nodetype_name)
-#            self.fields['parent'].queryset = Node.objects.filter(
-#                    user_id=self.request.user.id, 
-#                    type_id=parent_type.id
-#                    )
-            #self.fields['language'].queryset = Language.objects.filter(implemented=1)
         except Exception as error:
             print("Error with", error)
     

node/models.py

@@ -131,9 +131,9 @@ class Node(CTENode):
     def add_resource(self, **kwargs):
         # only for tests
         resource = Resource(guid=str(time()), digest=str(time()), **kwargs )
-
         #resource = Resource(**kwargs)
         resource.save()
+
         # User
         if 'user' not in kwargs and 'user_id' not in kwargs:
             resource.user = self.user
@@ -236,11 +236,13 @@ class Node(CTENode):
 
     @current_app.task(filter=task_method)
     def workflow(self, keys=None, ngramsextractorscache=None, ngramscaches=None, verbose=False):
+        print("In workflow() START")
         self.parse_resources()
         type_document   = NodeType.objects.get(name='Document')
         self.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
         from analysis.functions import do_tfidf
         do_tfidf(self)
+        print("In workflow() END")
 
 class Node_Metadata(models.Model):
     node        = models.ForeignKey(Node, on_delete=models.CASCADE)

parsing/FileParsers/EuropressFileParser.py

@@ -3,6 +3,7 @@ import locale
 from lxml import etree
 from datetime import datetime, date
 from django.utils import timezone
+import dateutil.parser
 
 from .FileParser import FileParser
 from ..NgramsExtractors import *
@@ -12,161 +13,201 @@ from ..NgramsExtractors import *
 class EuropressFileParser(FileParser):
   
     def _parse(self, file):
+
         localeEncoding = "fr_FR"
         codif      = "UTF-8"
         count = 0
 
         if isinstance(file, str):
             file = open(file, 'rb')
-        #print(file)
+        # print(file)
         contents = file.read()
         #print(len(contents))
         #return []
         encoding = self.detect_encoding(contents)
-        print(encoding)
+        #print(encoding)
         if encoding != "utf-8":
-            contents = contents.decode(encoding, errors='replace').encode(codif)
+            try:
+                contents = contents.decode("latin1", errors='replace').encode(codif)
+            except Exception as error:
+                print(error)
+#                try:
+#                    contents = contents.decode(encoding, errors='replace').encode(codif)
+#                except Exception as error:
+#                    print(error)
 
         try:
             html_parser = etree.HTMLParser(encoding=codif)
             html = etree.fromstring(contents, html_parser)
-            html_articles = html.xpath('/html/body/table')
+            
+            try:
+                html_articles = html.xpath('/html/body/table/tbody')
+                if len(html_articles) < 1:
+                    html_articles = html.xpath('/html/body/table')
+            except Exception as error:
+                print(error)
+            
+
         except:
             return []
 
         # initialize the list of metadata
         metadata_list = []
         # parse all the articles, one by one
-
-        for html_article in html_articles:
-            
-            metadata = {}
-            
-            if len(html_article):
-                for name in html_article.xpath("./tr/td/span[@class = 'DocPublicationName']"):
-                    if name.text is not None:
-                        format_journal = re.compile('(.*), (.*)', re.UNICODE)
-                        test_journal = format_journal.match(name.text)
-                        if test_journal is not None:
-                            metadata['source'] = test_journal.group(1)
-                            metadata['volume'] = test_journal.group(2)
+        try:
+            for html_article in html_articles:
+                
+                metadata = {}
+                
+                if len(html_article):
+                    for name in html_article.xpath("./tr/td/span[@class = 'DocPublicationName']"):
+                        if name.text is not None:
+                            format_journal = re.compile('(.*), (.*)', re.UNICODE)
+                            test_journal = format_journal.match(name.text)
+                            if test_journal is not None:
+                                metadata['source'] = test_journal.group(1)
+                                metadata['volume'] = test_journal.group(2)
+                            else:
+                                metadata['source'] = name.text.encode(codif)
+
+                    for header in html_article.xpath("./tr/td/span[@class = 'DocHeader']"):
+                        try:
+                            text = header.text
+                        except Exception as error:
+                            print(error)
+
+                        
+                        if isinstance(text, bytes):
+                            text = text.decode(encoding)
+                        format_date_fr = re.compile('\d*\s*\w+\s+\d{4}', re.UNICODE)
+                        if text is not None:
+                            test_date_fr = format_date_fr.match(text)
+                            format_date_en = re.compile('\w+\s+\d+,\s+\d{4}', re.UNICODE)
+                            test_date_en = format_date_en.match(text)
+                            format_sect = re.compile('(\D+),', re.UNICODE)
+                            test_sect = format_sect.match(text)
+                            format_page = re.compile(', p. (\w+)', re.UNICODE)
+                            test_page = format_page.match(text)
                         else:
-                            metadata['source'] = name.text.encode(codif)
-
-                for header in html_article.xpath("./tr/td/span[@class = 'DocHeader']"):
-                    text = header.text
-                    if isinstance(text, bytes):
-                        text = text.decode(encoding)
+                            test_date_fr = None
+                            test_date_en = None
+                            test_sect = None
+                            test_page = None
+                        
+                        
+                        
+                        if test_date_fr is not None:
+                            self.localeEncoding = "fr_FR"
+                            locale.setlocale(locale.LC_ALL, localeEncoding)
+                            if encoding != "utf-8":
+                                text = text.replace('י', 'é')
+                                text = text.replace('ű', 'û')
+                                text = text.replace(' aot ', ' août ')
 
-                    format_date_fr = re.compile('\d*\s*\w+\s+\d{4}', re.UNICODE)
-                    test_date_fr = format_date_fr.match(text)
-                    
-                    format_date_en = re.compile('\w+\s+\d+,\s+\d{4}', re.UNICODE)
-                    test_date_en = format_date_en.match(text)
-
-                    format_sect = re.compile('(\D+),', re.UNICODE)
-                    test_sect = format_sect.match(text)
+                            try :
+                                metadata['publication_date'] = datetime.strptime(text, '%d %B %Y')
+                            except :
+                                try:
+                                    metadata['publication_date'] = datetime.strptime(text, '%B %Y')
+                                except :
+                                    try:
+                                        locale.setlocale(locale.LC_ALL, "fr_FR")
+                                        metadata['publication_date'] = datetime.strptime(text, '%d %B %Y')
+                                        # metadata['publication_date'] = dateutil.parser.parse(text)
+                                    except Exception as error:
+                                        print(error)
+                                        print(text)
+                                        pass
+                        
+                        
+                        
+                        if test_date_en is not None:
+                            localeEncoding = "en_GB.UTF-8"
+                            locale.setlocale(locale.LC_ALL, localeEncoding)
+                            try :
+                                metadata['publication_date'] = datetime.strptime(text, '%B %d, %Y')
+                            except :
+                                try :
+                                    metadata['publication_date'] = datetime.strptime(text, '%B %Y')
+                                except :
+                                    pass
+
+                        if test_sect is not None:
+                            metadata['section'] = test_sect.group(1).encode(codif)
+                        
+                        if test_page is not None:
+                            metadata['page'] = test_page.group(1).encode(codif)
+
+                    metadata['title'] = html_article.xpath("string(./tr/td/span[@class = 'TitreArticleVisu'])").encode(codif)
+                    metadata['text']  = html_article.xpath("./tr/td/descendant-or-self::*[not(self::span[@class='DocHeader'])]/text()")
+                   
+                    line = 0
+                    br_tag = 10
+                    for i in html_articles[count].iter():
+                       # print line, br, i, i.tag, i.attrib, i.tail
+                        if i.tag == "span":
+                            if "class" in i.attrib:
+                                if i.attrib['class'] == 'TitreArticleVisu':
+                                    line = 1
+                                    br_tag = 2
+                        if line == 1 and i.tag == "br":
+                            br_tag -= 1
+                        if line == 1 and br_tag == 0:
+                            try:
+                                metadata['authors'] = str.title(etree.tostring(i, method="text", encoding=codif)).encode(codif)#.split(';')
+                            except:
+                                metadata['authors'] = 'not found'
+                            line = 0
+                            br_tag = 10
                     
-                    format_page = re.compile(', p. (\w+)', re.UNICODE)
-                    test_page = format_page.match(text)
                     
-                    if test_date_fr is not None:
-                        self.localeEncoding = "fr_FR"
-                        locale.setlocale(locale.LC_ALL, localeEncoding)
-                        if encoding != "utf-8":
-                            text = text.replace('י', 'é')
-                            text = text.replace('ű', 'û')
-                            text = text.replace(' aot ', ' août ')
-
-                        try :
-                            metadata['publication_date'] = datetime.strptime(text, '%d %B %Y')
-                        except :
+                    try:
+                        if metadata['publication_date'] is not None or metadata['publication_date'] != '':
                             try:
-                                metadata['publication_date'] = datetime.strptime(text, '%B %Y')
-                            except :
-                                print(text)
+                                back = metadata['publication_date']
+                            except Exception as e: 
+                                #print(e)
                                 pass
+                        else:
+                            try:
+                                metadata['publication_date'] = back
+                            except Exception as e:
+                                print(e)
+                    except :
+                        metadata['publication_date'] = timezone.now()
+
+                    #if lang == 'fr':
+                    #metadata['language_iso2'] = 'fr'
+                    #elif lang == 'en':
+                    #    metadata['language_iso2'] = 'en'
                     
-                    if test_date_en is not None:
-                        localeEncoding = "en_GB.UTF-8"
-                        locale.setlocale(locale.LC_ALL, localeEncoding)
-                        try :
-                            metadata['publication_date'] = datetime.strptime(text, '%B %d, %Y')
-                        except :
-                            try :
-                                metadata['publication_date'] = datetime.strptime(text, '%B %Y')
-                            except :
-                                pass
+                    
+                    metadata['publication_year']  = metadata['publication_date'].strftime('%Y')
+                    metadata['publication_month'] = metadata['publication_date'].strftime('%m')
+                    metadata['publication_day']  = metadata['publication_date'].strftime('%d')
+                    metadata['publication_date'] = ""
+                    
+                    if len(metadata['text'])>0: 
+                        metadata['doi'] = str(metadata['text'][-9])
+                        metadata['text'].pop()
+                        metadata['text'] = str(' '.join(metadata['text']))
+                        metadata['text'] = str(re.sub('Tous droits réservés.*$', '', metadata['text']))
+
+                    else: metadata['doi'] = "not found"
 
-                    if test_sect is not None:
-                        metadata['section'] = test_sect.group(1).encode(codif)
+                    metadata['bdd']  = u'europresse'
+                    metadata['url']  = u''
                     
-                    if test_page is not None:
-                        metadata['page'] = test_page.group(1).encode(codif)
-
-                metadata['title'] = html_article.xpath("string(./tr/td/span[@class = 'TitreArticleVisu'])").encode(codif)
-                metadata['text']  = html_article.xpath("./tr/td/descendant-or-self::*[not(self::span[@class='DocHeader'])]/text()")
-               
-                line = 0
-                br_tag = 10
-                for i in html_articles[count].iter():
-                   # print line, br, i, i.tag, i.attrib, i.tail
-                    if i.tag == "span":
-                        if "class" in i.attrib:
-                            if i.attrib['class'] == 'TitreArticleVisu':
-                                line = 1
-                                br_tag = 2
-                    if line == 1 and i.tag == "br":
-                        br_tag -= 1
-                    if line == 1 and br_tag == 0:
-                        try:
-                            metadata['authors'] = str.title(etree.tostring(i, method="text", encoding=codif)).encode(codif)#.split(';')
-                        except:
-                            metadata['authors'] = 'not found'
-                        line = 0
-                        br_tag = 10
-                
-                
-                try:
-                    if metadata['publication_date'] is not None or metadata['publication_date'] != '':
-                        try:
-                            back = metadata['publication_date']
-                        except Exception as e: 
-                            #print(e)
-                            pass
-                    else:
-                        try:
-                            metadata['publication_date'] = back
-                        except Exception as e:
-                            print(e)
-                except :
-                    metadata['publication_date'] = timezone.now()
-
-                #if lang == 'fr':
-                #metadata['language_iso2'] = 'fr'
-                #elif lang == 'en':
-                #    metadata['language_iso2'] = 'en'
-                
-                
-                metadata['publication_year']  = metadata['publication_date'].strftime('%Y')
-                metadata['publication_month'] = metadata['publication_date'].strftime('%m')
-                metadata['publication_day']  = metadata['publication_date'].strftime('%d')
-                metadata['publication_date'] = ""
-
-                metadata['object_id'] = str(metadata['text'][-9])
-                metadata['text'].pop()
-                metadata['text'] = str(' '.join(metadata['text']))
-                metadata['text'] = str(re.sub('Tous droits réservés.*$', '', metadata['text']))
-
-                metadata['bdd']  = u'europresse'
-                metadata['url']  = u''
-                
-              #metadata_str = {}
-                for key, value in metadata.items():
-                    metadata[key] = value.decode() if isinstance(value, bytes) else value
-                metadata_list.append(metadata)
-                count += 1
-         
+                  #metadata_str = {}
+                    for key, value in metadata.items():
+                        metadata[key] = value.decode() if isinstance(value, bytes) else value
+                    metadata_list.append(metadata)
+                    count += 1
+        
+        except Exception as error:
+            print(error)
+            pass
+
 #       from pprint import pprint
 #       pprint(metadata_list)
 #       return []

parsing/FileParsers/FileParser.py

@@ -77,9 +77,9 @@ class FileParser:
                 if language:
                     break
         if language:
-            metadata["language_iso2"] = language.iso2
-            metadata["language_iso3"] = language.iso3
-            metadata["language_fullname"] = language.fullname
+            metadata["language_iso2"]       = language.iso2
+            metadata["language_iso3"]       = language.iso3
+            metadata["language_fullname"]   = language.fullname
         return metadata
         
     def format_metadata(self, metadata):
@@ -102,10 +102,16 @@ class FileParser:
         if zipfile.is_zipfile(file):
             zipArchive = zipfile.ZipFile(file)
             for filename in zipArchive.namelist():
-                metadata_list += self.parse(zipArchive.open(filename, "r"))
+                try:
+                    metadata_list += self.parse(zipArchive.open(filename, "r"))
+                except Exception as error:
+                    print(error)
         # ...otherwise, let's parse it directly!
         else:
-            metadata_list += self._parse(file)
+            try:
+                metadata_list += self._parse(file)
+            except Exception as error:
+                print(error)
         # return the list of formatted metadata
         return map(self.format_metadata, metadata_list)
 

parsing/FileParsers/RisFileParser.py

@@ -36,6 +36,10 @@ class RisFileParser(FileParser):
                             #print(metadata)
                             try:
                                 #print("append")
+                                if 'language_fullname' not in metadata.keys():
+                                    if 'language_iso3' not in metadata.keys():
+                                        if 'language_iso2' not in metadata.keys():
+                                            metadata['language_iso2'] = 'en'
                                 metadata_list.append(metadata)
                                 metadata = {}
                                 #print("append succeeded")

static/css/bootstrap.css

@@ -743,6 +743,7 @@ pre code {
   min-height: 1px;
   padding-left: 15px;
   padding-right: 15px;
+  padding-bottom: 15px;
 }
 .col-xs-1,
 .col-xs-2,

static/img/logo.svg

@@ -31,7 +31,7 @@
      showgrid="true"
      inkscape:grid-bbox="true"
      inkscape:document-units="px"
-     inkscape:window-width="1360"
+     inkscape:window-width="881"
      inkscape:window-height="762"
      inkscape:window-x="0"
      inkscape:window-y="0"
@@ -60,11 +60,11 @@
        x="0"
        y="-0.1566938"
        inkscape:export-filename="/srv/gargantext/static/img/logo.png"
-       inkscape:export-xdpi="53"
-       inkscape:export-ydpi="53" />
+       inkscape:export-xdpi="454.50735"
+       inkscape:export-ydpi="454.50735" />
     <g
-       inkscape:export-ydpi="53.799999"
-       inkscape:export-xdpi="53.799999"
+       inkscape:export-ydpi="454.50735"
+       inkscape:export-xdpi="454.50735"
        inkscape:export-filename="/srv/gargantext/static/img/logo.png"
        style="fill:#ff8080;fill-opacity:0.82014388"
        id="g3835"

static/img/team/logo.svg

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="48px"
+   height="48px"
+   id="svg4362"
+   version="1.1"
+   inkscape:version="0.48.5 r10040"
+   sodipodi:docname="logo.svg">
+  <defs
+     id="defs4364" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="5.6897594"
+     inkscape:cx="-11.235831"
+     inkscape:cy="3.8560006"
+     inkscape:current-layer="layer1"
+     showgrid="true"
+     inkscape:grid-bbox="true"
+     inkscape:document-units="px"
+     inkscape:window-width="1360"
+     inkscape:window-height="762"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="0" />
+  <metadata
+     id="metadata4367">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     id="layer1"
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer">
+    <rect
+       style="fill:#fffcfc;fill-opacity:1;stroke:none"
+       id="rect3755"
+       width="29.70249"
+       height="31.108515"
+       x="0"
+       y="-0.1566938"
+       inkscape:export-filename="/srv/gargantext/static/img/logo.png"
+       inkscape:export-xdpi="53"
+       inkscape:export-ydpi="53" />
+    <g
+       inkscape:export-ydpi="53.799999"
+       inkscape:export-xdpi="53.799999"
+       inkscape:export-filename="/srv/gargantext/static/img/logo.png"
+       style="fill:#ff8080;fill-opacity:0.82014388"
+       id="g3835"
+       transform="matrix(0.2422549,0,0,0.23374214,-49.789462,-7.9055988)">
+      <path
+         inkscape:export-ydpi="100"
+         inkscape:export-xdpi="100"
+         inkscape:export-filename="/home/alexandre/projets/gargantext.py/gargantext_core/shared/LogoSimple.png"
+         id="path3837"
+         d="m 206.24721,35.28586 0,129.5 67.78125,0 0,-8.625 c -9.86526,-0.47262 -18.57934,-2.63259 -25.5625,-6.28125 -18.65918,-9.74237 -29.875,-28.26535 -29.875,-49.1875 0,-31.71741 21.11877,-52.8149 55.4375,-55.1875 l 0,-10.21875 -67.78125,0 z m 67.78125,10.21875 0,8.5 c 1.74191,-0.16369 3.53543,-0.28125 5.37499,-0.28125 6.91081,0 13.295,1.44116 19.6875,4.15625 l 2.40625,2.875 2.59375,14.53125 9.6875,0 0,-25.375 c -11.40283,-3.03451 -22.61727,-4.65625 -33.15625,-4.65625 -2.24526,0 -4.44959,0.10177 -6.59374,0.25 z m 0,8.5 c -23.28864,2.18852 -37.65625,18.81513 -37.65625,45.562503 0,27.600037 14.44681,45.025437 37.65625,47.812497 l 0,-93.375 z m 0,93.375 0,8.78125 c 1.36224,0.0653 2.75177,0.0937 4.15624,0.0937 10.19344,0 22.1324,-1.88915 35.78125,-5.5625 l 0,-38.1875 2.9375,-2.21875 9.5,-0.8125 0,-6.5625 -43.21875,0 0,6.5625 12.28125,0.8125 2.9375,2.21875 0,33.21875 c -6.73804,1.4374 -12.61466,2.09375 -17.625,2.09375 -2.32322,0 -4.57592,-0.17643 -6.74999,-0.4375 z"
+         style="font-size:166.11251831px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#ff8080;fill-opacity:0.82014388;stroke:none;font-family:Bitstream Charter;-inkscape-font-specification:Bitstream Charter"
+         inkscape:connector-curvature="0" />
+      <path
+         inkscape:export-ydpi="100"
+         inkscape:export-xdpi="100"
+         transform="translate(611.62306,-400.10238)"
+         sodipodi:open="true"
+         sodipodi:end="6.1660663"
+         sodipodi:start="0"
+         d="m -312.87112,480.17926 c 0,4.97881 -4.03612,9.01493 -9.01493,9.01493 -4.97881,0 -9.01493,-4.03612 -9.01493,-9.01493 0,-4.97881 4.03612,-9.01493 9.01493,-9.01493 4.57131,0 8.41901,3.42153 8.95317,7.96152"
+         sodipodi:ry="9.0149298"
+         sodipodi:rx="9.0149298"
+         sodipodi:cy="480.17926"
+         sodipodi:cx="-321.88605"
+         id="path3839"
+         style="fill:#ff8080;fill-opacity:0.82014388;stroke:none"
+         sodipodi:type="arc" />
+    </g>
+  </g>
+</svg>

static/js/gargantext.angular.js

@@ -381,7 +381,7 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
                 y: {key: 'y', type: 'linear', type: 'numeric'},
             },
             tension: 1.0,
-            lineMode: 'bundle',
+            lineMode: 'linear',
             tooltip: {mode: 'scrubber', formatter: function(x, y, series) {
                 var grouping = groupings.datetime[$scope.groupingKey];
                 return grouping.representation(x) + ' → ' + y;
@@ -442,6 +442,13 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
             angular.forEach(results, function(result, r){
                 var x = grouping.truncate(result[0]);
                 var y = parseFloat(result[1]);
+                if (dataObject[x] === undefined) {
+                    var row = [];
+                    angular.forEach($scope.datasets, function(){
+                        row.push(0);
+                    });
+                    dataObject[x] = row;
+                }
                 dataObject[x][datasetIndex] += y;
             });
         });

static/js/jquery/jquery.ba-dotimeout.min.js

+/*
+ * jQuery doTimeout: Like setTimeout, but better! - v1.0 - 3/3/2010
+ * http://benalman.com/projects/jquery-dotimeout-plugin/
+ * 
+ * Copyright (c) 2010 "Cowboy" Ben Alman
+ * Dual licensed under the MIT and GPL licenses.
+ * http://benalman.com/about/license/
+ */
+(function($){var a={},c="doTimeout",d=Array.prototype.slice;$[c]=function(){return b.apply(window,[0].concat(d.call(arguments)))};$.fn[c]=function(){var f=d.call(arguments),e=b.apply(this,[c+f[0]].concat(f));return typeof f[0]==="number"||typeof f[1]==="number"?this:e};function b(l){var m=this,h,k={},g=l?$.fn:$,n=arguments,i=4,f=n[1],j=n[2],p=n[3];if(typeof f!=="string"){i--;f=l=0;j=n[1];p=n[2]}if(l){h=m.eq(0);h.data(l,k=h.data(l)||{})}else{if(f){k=a[f]||(a[f]={})}}k.id&&clearTimeout(k.id);delete k.id;function e(){if(l){h.removeData(l)}else{if(f){delete a[f]}}}function o(){k.id=setTimeout(function(){k.fn()},j)}if(p){k.fn=function(q){if(typeof p==="string"){p=g[p]}p.apply(m,d.call(n,i))===true&&!q?o():e()};o()}else{if(k.fn){j===undefined?e():k.fn(j===false);return true}else{e()}}}})(jQuery);
\ No newline at end of file

templates/about.html

+
+{% extends "menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+<link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
+
+<script src="{% static "js/jquery/jquery.min.js" %}" type="text/javascript"></script>
+
+{% endblock %}
+
+
+
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+		<div class="jumbotron">
+				<h1>About Gargantext</h1>
+		</div>
+
+
+<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+
+		<div class="panel panel-default">
+				<div class="panel-heading">
+						<h2 class="panel-title">
+								<a data-toggle="collapse" data-parent="#accordion" href="#collapseVersions">
+										<h2>Versions</h2>
+								</a>
+						</h2>
+				</div>
+				<div id="collapseVersions" class="panel-collapse collapse" role="tabpanel">
+						<div class="panel-body">
+								<div class="container">
+										<ul>
+												<li>Version 1.0</li>
+														<ul>
+																<li>Beta Version </li>
+																<li>Licence of Gargantext is GPL v3+ </li>
+														</ul>
+												<li>Version 1.0.5</li>
+														<ul>
+																<li>Bug resolution: xml zipped from Mac</li>
+																<li>Bug resolution: french accents in filenames</li>
+																<li>New features: [Advanced chart] ngrams completion</li>
+																<li>New features: button to delete all duplicates</li>
+														</ul>
+										</ul>
+								</div>
+						</div>
+				</div>
+		</div>
+
+
+		<div class="panel panel-default">
+				<div class="panel-heading">
+						<h2 class="panel-title">
+								<a data-toggle="collapse" data-parent="#accordion" href="#collapseCommunity">
+										<h2>Community</h2>
+								</a>
+						</h2>
+				</div>
+				<div id="collapseCommunity" class="panel-collapse collapse" role="tabpanel">
+						<div class="panel-body">
+								<div class="container">
+										<ul>
+												<li>Mailing-lists</li>
+														<ul>
+																<li>User mailing-list: soon</li>
+																<li>Devel mailing-list: soon</li>
+														</ul>
+												<li>Code and repository access</li>
+														<ul>
+																<li>You are free to participate. Present yourself on mailing-lists or irc.</li>
+														</ul>
+												<li>IRC</li>
+														<ul>
+																<li>#gargantext on OFTC:</li>
+																<li> 
+																		<p><iframe src="http://webchat.oftc.net/?channels=#gargantext" width="500" height="350"></iframe></p>
+																<p>If the window is too small, just click <a href="http://webchat.oftc.net/?channels=#gargantext">here</a></p>
+																</li>
+														</ul>
+										</ul>
+								</div>
+						</div>
+				</div>
+		</div>
+
+
+
+{% if team %}
+		<div class="panel panel-default">
+				<div class="panel-heading">
+						<h2 class="panel-title">
+								<a data-toggle="collapse" data-parent="#accordion" href="#collapseTeam">
+										<h2>Core team</h2>
+								</a>
+						</h2>
+				</div>
+				<div id="collapseTeam" class="panel-collapse collapse" role="tabpanel">
+						<div class="panel-body">
+								<div class="container">
+										<div class="row">
+												<div class="thumbnails">
+												{% for member in team %}
+												<div class="col-md-3 ">
+														<div class="thumbnail">
+															{% if member.picture %}
+																<img src="{% static "img/team/"%}{{ member.picture }}" style="100px; height:150px">
+																{% else %}
+																<img src="{% static "img/logo.png" %}" style="100px; height:150px">
+															{% endif %}
+																		<div class="caption">
+																				<h3>{{ member.first_name }} {{member.last_name }}</h3>
+																				{% if member.role %}
+																						<p class="description">{{ member.role }}</p>
+																				{% endif %}
+																				{% if member.mail %}
+																					<a href="mailto:{{ member.mail }}" class="btn btn-primary btn-xs">Mail</a>
+																				{% endif %}
+																				{% if member.website %}
+																					<a href="{{ member.website }}" class="btn btn-primary btn-xs">Website</a>
+																				{% endif %}
+																		</div>
+														</div>
+												</div>
+												{% endfor %}
+										</div>
+								</div>
+						</div>
+				</div>
+
+{% endif %}
+
+		</div>
+</div>
+
+
+
+</div>
+
+{% endblock %}

templates/add_corpus.html

@@ -12,7 +12,6 @@
 {% block content %}
 
 <div id="loading" style="position:absolute; top:50%; left:40%; width:80px; display: none;" >
-
     <img src="{% static "img/ajax-loader.gif" %}" alt="" />
 </div>
 
@@ -29,11 +28,5 @@
 </div>
 </div>
 
-<script>
-function showLoader(){
-  $('#loading').setStyle('display', 'block');
-  $('submit_btn').disabled = true;
-};
-</script>
 
 {% endblock %}

templates/add_corpus_as_p.html

@@ -24,5 +24,4 @@
 </form>
 </div>
 </div>
-
 {% endblock %}

templates/authentication.html

+<html>
+<body>
+
+<head>
+
+        <link href="/static/grappelli/jquery/ui/css/custom-theme/jquery-ui-1.10.3.custom.min.css" rel="stylesheet" type="text/css" media="screen" />
+        <link href="/static/grappelli/stylesheets/screen.css" rel="stylesheet" type="text/css" media="screen" />
+        
+            <link href="/static/grappelli/stylesheets/mueller/grid/output.css" rel="stylesheet" type="text/css" media="screen" />
+
+
+        <script src="/static/grappelli/jquery/jquery-1.9.1.min.js" type="text/javascript"></script>
+        <script src="/static/grappelli/jquery/ui/js/jquery-ui-1.10.3.custom.min.js" type="text/javascript"></script>
+        
+<link rel="stylesheet" type="text/css" href="/static/css/bootstrap.css">
+<link rel="stylesheet" type="text/css" href="/static/css/morris.css">
+            <!-- Grappelli Minified -->
+            <script src="/static/grappelli/js/grappelli.min.js" type="text/javascript"></script>
+</head>
+
+    <div id="grp-container">
+
+        <!-- HEADER -->
+        <header id="grp-header" class="">
+            
+
+                <!-- NAVIGATION -->
+                <div id="grp-navigation">
+                    <h1 id="grp-admin-title">Gargantext</h1>
+                    <!-- Nav-Global -->
+                </div>
+        </header>
+
+
+        <!-- CONTENT -->
+        <article id="grp-content" class="">
+            
+                
+            
+
+            <!-- CONTENT TITLE -->
+            <header id="grp-content-title">
+                <br><br><br>
+            </header>
+
+<div class="container theme-showcase" role="main">
+        <div class="jumbotron">
+            <!-- CONTENT-CONTAINER -->
+            <div id="grp-content-container">
+                
+                <div class="g-d-c g-d-10 g-centered">
+                    <form action="/auth/" method="post"><input type='hidden' name='csrfmiddlewaretoken' value='Yopwc8YatA3C68kSevZX9upoXdIcZzPn' />
+                        <fieldset class="grp-module grp-module-login">
+                            <h2>
+                                <span class="grp-current-page">Login</span>
+                            </h2a>
+                            <div class="grp-module">
+
+                                
+                                
+                                {% block main %}
+                                    <div id="login">
+                                        <form class="form-horizontal" name="LoginForm" action="/auth/" method="post">
+                                        {% csrf_token %}
+                                        {% if next %}
+                                            <input type="hidden" name="next" value="{{ next }}" />
+                                        {% endif %}
+                                        <div class="grp-row">
+                                            <label class="control-label" for="username">Username</label>
+                                            <input type="text" id="username" name="username"  placeholder="Username">
+                                        </div>
+                                        <div class="grp-row grp-connected">
+                                            <label class="control-label" for="password">Password</label>
+                                            <input type="password" name="password" id="password" placeholder="Password">
+                                        </div>
+                                
+                                
+                            </div>
+                        </fieldset>
+                        <div class="grp-module grp-submit-row">
+                            <ul>
+                                <li><input type="submit" class="grp-button grp-default" value="Log in" /></li>
+                            </ul>
+                        </div>
+                    </form>
+                    </div>
+                {% endblock %}
+                </div>
+            </div>
+
+        </article>
+
+        </div>
+</div>
+    </div>
+
+
+    
+    <script type="text/javascript">
+        (function($) {
+            $(window).load(function(){ $('#username').focus(); });
+        })(grp.jQuery);
+    </script>
+</body>
+</html>
\ No newline at end of file

templates/chart.html

@@ -13,8 +13,8 @@
 
     <div class="container theme-showcase" role="main">
         <div class="jumbotron">
-						<h1>{{ project.name }}</h1>
-            <p>Advanced charts</p>
+            <h1>Advanced chart</h1>
+            <p>Custom, test, interpret</p>
         </div>
     </div>
 
@@ -218,40 +218,39 @@
         </div>
     -->
 
-        <div ng-app="Gargantext" ng-controller="GraphController">        
+
+
+        <div ng-app="Gargantext" ng-controller="GraphController">
+
             
             <ul class="datasets">
                 <button class="add" ng-click="addDataset()">Add a dataset...</button>
                 <li class="dataset" ng-controller="DatasetController" ng-repeat="dataset in datasets">
-                    <hr/>                
+                    <hr/>
                     <div class="corpus">
                         <button ng-click="removeDataset($index)" title="remove this dataset">X</button>
                         <select ng-model="mesured" style="background-color:{{ getColor($index, datasets.length) }}" ng-options="value as key for (key, value) in {'Documents count': 'nodes.count', 'Ngrams count': 'ngrams.count'}" ng-change="updateQuery()"></select>
-                        in the corpus
-                        <select ng-model="corpusId" ng-change="updateEntities()">
-                            <option ng-repeat="corpus in corpora" value="{{corpus.id}}">{{corpus.name}}</option>
-                        </select>
+                        in the project
+                        <select ng-model="projectId" ng-change="updateCorpora()" ng-options="project.id as project.name for project in projects"></select>,
+                        corpus
+                        <select ng-model="corpusId" ng-change="updateEntities()" ng-options="corpus.id as corpus.name for corpus in corpora"></select>
                     </div>                    
                     <div class="filters" ng-if="entities">
                         <ul>
                             <li ng-repeat="filter in filters">
                                 <button ng-click="removeFilter($index)" title="remove this filter">x</button>
                                 <span>...where the </span>
-                                <select ng-model="filter.entity">
-                                    <option ng-repeat="(entityName, entityColumns) in entities" value="{{entityName}}">{{entityName}}</option>
-                                </select>
-                                <span ng-if="filter.entity">
-                                    <select ng-model="filter.column">
-                                        <option ng-repeat="column in entities[filter.entity] | orderBy:'key'" value="{{column.key}}">{{column.key}}</option>
-                                    </select>
-                                    <span ng-if="filter.column" ng-repeat="column in entities[filter.entity]">
-                                        <span ng-if="column.key == filter.column">
-                                            <select ng-model="filter.operator">
-                                                <option ng-repeat="operator in operators[column.type]" value="{{operator.key}}">{{operator.label}}</option>
-                                            </select>
-                                            <input type="text" ng-model="filter.value" ng-change="updateQuery()">
-                                        </span>
-                                    </span>
+                                <select ng-model="filter.entity" ng-options="entity as entity.key for entity in entities"></select>
+                                <span ng-if="filter.entity.key != 'ngrams'">
+                                    <select ng-if="filter.entity" ng-model="filter.column" ng-options="column as column.key for column in filter.entity.columns | orderBy:'key'"></select>
+                                    <select ng-if="filter.column" ng-model="filter.operator" ng-options="operator.key as operator.label for operator in operators[filter.column.type]"></select>
+                                    <input ng-if="filter.operator" type="text" ng-model="filter.value" ng-change="updateQuery()">
+                                </span>
+                                <span ng-if="filter.entity.key == 'ngrams'">
+                                    are in this list:
+                                    <tags-input ng-model="filter.value" display-property="terms" placeholder="Add an ngram" on-tag-added="updateQuery()" on-tag-removed="updateQuery()" add-from-autocomplete-only="true">
+                                        <auto-complete source="getNgrams($query)"></auto-complete>
+                                    </tags-input ng-model="tags">
                                 </span>
                             </li>
                         </ul>
@@ -273,7 +272,7 @@
                 <br/>
 
                 Y-axis: use a
-                <select ng-model="graph.options.axes.y.type" ng-options="type for type in ['log', 'linear']"></select>
+                <select ng-model="graph.options.axes.y.type" ng-options="type for type in ['linear', 'log']"></select>
                 scale
                 <br/>
                 
@@ -334,7 +333,9 @@
     <script type="text/javascript" src="{% static "js/angular-cookies.min.js" %}"></script>
     <script type="text/javascript" src="{% static "js/d3/d3.v2.min.js" %}"></script>
     <script type="text/javascript" src="{% static "js/d3/n3.line-chart.min.js" %}"></script>
-    <!-- <script type="text/javascript" src="{% static "js/d3/angular-charts.js" %}"></script> -->
+
+    <script type="text/javascript" src="{% static "js/ng-tags-input.min.js" %}"></script>
+    <link rel="stylesheet" href="{% static "css/ng-tags-input.min.css" %}">
 
     <script type="text/javascript" src="{% static "js/gargantext.angular.js" %}"></script>
 

templates/corpus.html

@@ -14,6 +14,15 @@
 <script type="text/javascript" src="{% static "js/charts/crossfilter.js"%}"></script>
 <script type="text/javascript" src="{% static "js/charts/dc.js"%}"></script>
 
+<style>
+.no-transition {
+  -webkit-transition: height 0.1s;
+  -moz-transition: height 0.1s;
+  -ms-transition: height 0.1s;
+  -o-transition: height 0.1s;
+  transition: height 0.1s;
+}
+</style>
 
 {% endblock %}
 
@@ -33,7 +42,7 @@
 							</p>
 							{% endif %}
 
-							<!--						<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.pk }}/">Add file</a> --!>
+							<!--						<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.pk }}/">Add file</a> -->
 						<a class="btn btn-primary btn-lg" role="button" href="/project/{{project.pk}}/corpus/{{ corpus.pk }}/corpus.csv">Save as</a>
 						<a class="btn btn-primary btn-lg" role="button" href="/project/{{project.pk}}/corpus/{{ corpus.pk }}/delete">Delete</a></p>
 
@@ -79,7 +88,7 @@
         </a>
       </h4>
     </div>
-    <div id="collapseOne" class="panel-collapse collapse" role="tabpanel">
+    <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
       <div class="panel-body">
 				<p align="right">
 				
@@ -110,7 +119,7 @@
 
 				<div class="col-md-4">
 						<div class="jumbotron">
-								<h3><a href="/corpus/{{corpus.id}}/matrix">Matrix</a></h3>
+								<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3>
 						<ol>
 								<li>Sort</li>
 								<li>Group</li>
@@ -122,7 +131,7 @@
 
 				<div class="col-md-4">
 						<div class="jumbotron">
-						<h3><a href="/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+						<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
 						<ol>
 								<li>Visualize</li>
 								<li>Explore</li>
@@ -144,6 +153,7 @@
 
 
 var datesbuffer = false;
+var latest,oldest;
 
 function pr(msg) {
     console.log(msg)
@@ -165,14 +175,30 @@ function updateDocuments(pagenumber,pagenav) {
     pagenav = (pagenav)?pagenav:true;
     pagenumber = (pagenumber)?pagenumber:1;
     pr("in here   pagenav:"+pagenav+" - pagenumber:"+pagenumber)
-    pr($( "#collapseOne" ).height())
+    pr("offset left: "+$( "#collapseOne" ).offset().left)
     // if "Read Documents" collapsible is close, then... show some me pubs!
-    if ( pagenav || $( "#collapseOne" ).height() < 50) {
+    if ( pagenav || $( "#collapseOne" ).offset().left == 0) {
 
         // Here u ask for the server some paginated results (pubs)
         // if u havent select a timerange from the blue chart, then show me all pubs
         if(!datesbuffer) {
-            console.log("nothing cause dont wanna")
+
+            var dataini = oldest.join("")
+            var datafin = latest.join("")
+            //http://localhost:8000/project/37525/corpus/37526/timerange/20040117/20040125?page=1
+
+            var base = window.location.href;
+            var theurl = base+"timerange/"+dataini+"/"+datafin+"?page="+pagenumber;
+            pr("theurl:   "+theurl)
+
+            $.ajax({
+              url: theurl,
+              success: function(data) {
+                // console.log(data)
+                $('#subcorpusdiv').html(data);
+                return true;
+              }
+            });
         } 
         // there's some timerange selected in the blue chart, so show me the pubs of that period
         else {
@@ -196,12 +222,26 @@ function updateDocuments(pagenumber,pagenav) {
 
     }
     //else: "Read Documents" collapsible is open!, so do nothing 
-    
-
+}
 
 
-}
 
+var current_docs = {}
+var BIS_dict = {}
+var corpusid = window.location.href.split("corpus")[1].replace(/\//g, '')//replace all the slashes
+var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
+$.ajax({
+  url: theurl,
+  success: function(data) {
+    bisarray = data.data
+    for(var i in bisarray) {
+        untitlebis = bisarray[i].values
+        BIS_dict[untitlebis[0]] = [bisarray[i].count , 0];// [ total amount , removed ]
+    }
+    pr(BIS_dict)
+    if(Object.keys(BIS_dict).length>0) $("#delAll").css("visibility", "visible"); $("#delAll").show();
+  }
+});
 
 // var gainOrLossChart = dc.pieChart("#gain-loss-chart");
 // var fluctuationChart = dc.barChart("#fluctuation-chart");
@@ -233,13 +273,8 @@ d3.csv("/chart/corpus/{{ corpus.id }}/data.csv", function (data) {
 
             var orderDates = Object.keys(justdates).reverse();
             
-            var latest = orderDates[0].split("/")
-            var oldest = orderDates[orderDates.length-1].split("/")
-
-            pr("latest:")
-            pr(latest)
-            pr("oldest:")
-            pr(oldest)
+            latest = orderDates[0].split("/")
+            oldest = orderDates[orderDates.length-1].split("/")
 
             var t0_year = oldest[0]
             var t0_month = oldest[1]
@@ -352,5 +387,6 @@ d3.csv("/chart/corpus/{{ corpus.id }}/data.csv", function (data) {
 
 </script>
 
+<script type="text/javascript" src="{% static "js/jquery/jquery.ba-dotimeout.min.js"%}"></script>
 
 {% endblock %}

templates/home.html

@@ -24,12 +24,16 @@
 				<div class="row">
 						<div class="content">
 						<center>
+						<img src="{% static "img/logo.png"%}" alt="Logo Gargantext" style="100px; height:150px">
+
+						<!--
 						<h2>Introduction Video</h2>
 						<video width="320" height="240" controls>
 						<source src="movie.mp4" type="video/mp4">
 						<source src="movie.ogg" type="video/ogg">
 						Your browser does not support this video sorry. Try Firefox or Chromium.
 						</video>
+						--!>
 						</center>
 				</div>
 				</div>
@@ -39,25 +43,62 @@
 
 
 <div class="container">
-    <div class="row">
-										<div class="col-md-4 content">
-												<h3>Presentation</h3>
+
+
+<div class="row">
+
+<div class="col-md-4 content">
+				<h3><a href="#">Historic</a></h3>
 												<p>
-												Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+
+    Chapter 1.VI. -- How Gargantua was born in a strange manner.
+    Chapter 2.XXIII. -- How Pantagruel departed from Paris, hearing
+    news that the Dipsodes had invaded the land of the Amaurots; and
+    the cause wherefore the leagues are so short in France. Chapter
+    3.XLVI. -- How Pantagruel and Panurge diversely interpret the
+    words of Triboulet. Chapter 4.LV. -- How Pantagruel, being at sea,
+    heard various unfrozen words. Chapter 5.IX. -- How we arrived at
+    the island of Tools.
+
 												</p>
 										</div>
-        
+
 <div class="col-md-4 content">
-												<h3>Historic</h3>
+<h3><a href="#">Presentation</a></h3>
 												<p>
-												Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+
+										Lorem ipsum dolor sit amet, consectetur adipiscing elit,
+				sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+				Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris
+				nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
+				reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
+				pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
+				culpa qui officia deserunt mollit anim id est laborum.
+
 												</p>
 										</div>
-        
+
+  
 <div class="col-md-4 content">
-												<h3>Tutorials</h3>
+<h3><a href="#">Tutoreil</a></h3>
 												<p>
-												Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+												<!-- Why not French ? -->
+												<!-- find Cambridge source which inspired this --!>
+
+        Il praaît que l'odrre des ltetres dnas un mot n'a pas
+        d'iprnorotncae. La pmeirère et la drenèire letrte diovent
+        êrte à la bnnoe pclae. Le rsete peut êrte dnas un dsérorde
+        ttoal et on puet tujoruos lrie snas poribême. On ne lit
+        donc pas chuaqe ltetre en elle-mmêe, mias le mot cmome un
+        tuot. Un chnagmnet de réfretniel et nuos tarnsposns ce
+        rselutat au txete lui-mmêe: l'odrre des mtos est faiblement
+        imoprtnat copmraé au cnotxete du txete qui, lui, est copmté:
+        comptexter avec Gargantext.
+
+												
+												
+												
+												
 												</p>
 										</div>
         

templates/matrix.html

@@ -61,7 +61,7 @@ text.inactive {
 
 var margin = {top: 80, right: 0, bottom: 10, left: 80},
     width = 720,
-    height = 720;
+    height = 2720;
 
 var x = d3.scale.ordinal().rangeBands([0, width]),
     z = d3.scale.linear().domain([0, 4]).clamp(true), //alpha for color-cells
@@ -69,7 +69,7 @@ var x = d3.scale.ordinal().rangeBands([0, width]),
 
 var svg = d3.select("body").append("svg")
     .attr("width", "100%")
-    .attr("height", "100%")
+    .attr("height", "115%")
     .style("margin-left", "15%")
   .append("g")
     .attr("transform", "translate(" + "300" + "," + "300" + ")");//challenge: to calculate both parameters! maybe do something with the labels length

templates/menu.html

@@ -22,9 +22,20 @@
         
         <div class="navbar-collapse collapse">
           <ul class="nav navbar-nav">
-            <li><a href="/admin/">Admin</a></li>
+							<!-- <li><a href="/admin/">Admin/</a></li> --!>
+            <li><a href="/about/">About</a>
+						</li>
+						{% if user.is_authenticated %}
             <li><a href="/projects/">Projects</a></li>
-            <li><a href="/projects/">Corpus</a></li>
+						{% endif %}
+						{% if project %}
+						<li><a href="/project/{{project.id}}">{{project.name}}</a></li>
+						{% endif %}
+						{% if corpus %}
+						<li><a href="/project/{{project.id}}/corpus/{{corpus.id}}">{{corpus.name}}</a></li>
+						{% endif %}
+
+
           </ul>
 
 				<ul class="nav pull-right">
@@ -33,10 +44,10 @@
 
 								</a>
 								<ul class="dropdown-menu">
-										<li><a tabindex="-1" href="/admin/logout/">Login</a></li>
+										<li><a tabindex="-1" href="/auth/">Login</a></li>
 										<li><a tabindex="-1" href="#">Profile</a></li>
 										<li class="divider"></li>
-										<li><a tabindex="-1" href="#">Help</a></li>
+										<li><a tabindex="-1" href="/auth/logout">Logout</a></li>
 								</ul>
 						</li>
 				</ul>
@@ -54,7 +65,7 @@
 <hr>
 
 <footer>
-        <p>Gargantext v.1.0 (Copyrights {{ date.year }})</p>
+        <p>Gargantext, version 1.0.5, Copyrights CNRS {{ date.year }}.</p>
 </footer>
 
 

templates/project.html

@@ -25,7 +25,7 @@
         <div class="col-md-6">
             {% if project %}
             <h1>{{ project.name }}</h1>
-            <h3> {{number}} corpora </h3>
+						<!--<h3> {{number}} corpora </h3>--!>
             {% endif %}
         </div>
 
@@ -51,8 +51,7 @@
 		
 		{{ formResource.non_field_errors }}
 		{{ formResource.as_p}}
-		<input type="submit" class="btn" value="Add this corpus" />
-</form>
+		<input onclick='$("#semLoader").css("visibility", "visible"); $("#semLoader").show();' type="submit" name="submit" id="submit" class="btn" value="Add this corpus" /><div>
     </center>
 				</p>
 
@@ -65,7 +64,11 @@
         </div>
         </div>
 </div>
-<!-- Add jumbotron container for each type of corpus (presse, science etc.) --!>
+<!-- Add jumbotron container for each type of corpus (presse, science etc.) -->
+
+          <div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
+							<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
+          </div>
 
 <div class="container">
 				{% if list_corpora  %}
@@ -77,16 +80,18 @@
 												<ul>
 														{% for corpus in corpora %}
 														<li> {% ifnotequal corpus.count 0 %}
-																		<a href="/project/{{project.id}}/corpus/{{corpus.id}}">{{corpus.name}}</a>
+																		<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> 
+																			{{corpus.name}}
+																		</a>
 																		, {{ corpus.count }} Documents 
 																 {% else %}
-																 {{corpus.name}} : Processing, drink a cup of tea, and refresh the page :)
+																 {{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
 																 {% endifnotequal %}
 																		<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" 
 																		data-content='
 																		<ul>
-																		<li> Add documents </li>
 																		<li> Rename </li>
+																		<li> Add new documents </li>
 																		<li><a href="/project/{{ project.id }}/corpus/{{ corpus.id}}/delete">Delete</a></li>
 																		</ul>
 																		'>Manage</button>

templates/projects.html

@@ -38,7 +38,16 @@
 										<!--<div class="col-md-offset-7 col-md-4 content" style="background-color:grey">!-->
 										<div class="col-md-3 content">
 												<h3><a href="/project/{{ project.id }}">{{ project.name }}</a>
-														<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" data-content='<a href="/project/{{ project.id }}/delete">Yes, I am sure!</a>'>Delete</button>
+														
+														<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" 
+																		data-content='
+																		<ul>
+																		<li> Rename </li>
+																		<li> Add new corpus </li>
+																		<li><a href="/project/{{ project.id }}/delete">Delete</a></li>
+																		</ul>
+																		'>Manage</button>
+
 												</h3>
 
 												<h4>{{ project.subtitle }}<h4>

templates/subcorpus.html

 
 
-{% if date %}
-<p>Today: {{date}}</p>
-{% endif %}
-
 <div class="pagination">
     <span class="step-links">
         {% if documents.has_previous %}
-            <a onclick="updateDocuments({{ documents.previous_page_number }},true);">previous</a>
+            <a style="cursor: pointer;" onclick="updateDocuments({{ documents.previous_page_number }},true);">previous</a>
         {% endif %}
 
+        <div id="currentpage" style="display: none;">{{ documents.number }}</div>
         <span class="current">
-            Page {{ documents.number }} of {{ documents.paginator.num_pages }}.
+            <strong>Page {{ documents.number }}</strong> of {{ documents.paginator.num_pages }}.
         </span>
 
         {% if documents.has_next %}
-            <a onclick="updateDocuments({{ documents.next_page_number }},true);">next</a>
+            <a style="cursor: pointer;" onclick="updateDocuments({{ documents.next_page_number }},true);">next</a>
         {% endif %}
     </span>
 </div>
 
 {% if documents %}
 
-<p>Paginator stuff</p>
+
+          <div id="delAll" style="visibility: hidden;">
+              <button onclick="deleteDuplicates(theurl);">Delete Duplicates</button>
+          </div>
+
+
 <ul>
 {% for doc in documents %}
     {% if doc.date %}
-    <li> <b>{{ doc.date }}</b>: <a target="_blank" href="/nodeinfo/{{doc.id}}">{{ doc.name}}</a> , @ {{ doc.metadata.source}}</li>
+    <li><div id="doc_{{doc.id}}"> <b>{{ doc.date }}</b>: <a target="_blank" href="/nodeinfo/{{doc.id}}">{{ doc.name}}</a> , @ {{ doc.metadata.source}}</div></li>
     {% endif %}
 {% endfor %}
 </ul>
 
-{% endif %}
+<script>
\ No newline at end of file
+
+function getCookie(name) {
+    var cookieValue = null;
+    if (document.cookie && document.cookie != '') {
+        var cookies = document.cookie.split(';');
+        for (var i = 0; i < cookies.length; i++) {
+            var cookie = jQuery.trim(cookies[i]);
+            // Does this cookie string begin with the name we want?
+            if (cookie.substring(0, name.length + 1) == (name + '=')) {
+                cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                break;
+            }
+        }
+    }
+    return cookieValue;
+}
+
+function deleteDuplicates(url) {
+    console.log("hello world")
+    console.log(url)
+    $.ajax({
+      url: url,
+      type: 'DELETE',
+      beforeSend: function(xhr) {
+        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+      },
+      success: function(data) {
+        console.log("in DeleteDuplicates")
+        console.log(data)
+        $("#delAll").remove();
+      },
+        error: function(result) {
+            console.log("Data not found");
+            console.log(result)
+        }
+    });
+}
+
+function deleteNode(node_id) {
+    $.ajax({
+        url: '/api/nodes/'+node_id,
+        type: 'DELETE',
+        beforeSend: function(xhr) {
+            xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+        },
+        success: function(result) {
+            console.log("deleting: ")
+            console.log(current_docs[node_id])
+            var title = current_docs[node_id]
+            console.log(result)
+            var currentpage = $("#currentpage").html()
+            console.log("vaciando div de subcorpus")
+            $("#divsubcorpus").html("")
+            console.log("before:")
+            console.log(BIS_dict[title])
+            BIS_dict[title][0]--;
+            BIS_dict[title][1]++;
+            console.log("after:")
+            console.log(BIS_dict[title])
+            // updateDocuments( currentpage , true );
+            console.log("after delete:     updateDocuments( "+currentpage+" , true )")
+            updateDocuments( currentpage , true )
+            // $.when(  updateDocuments( currentpage , true )  ).done(function(a1) {
+            //     console.log("inside the wheeeeen")
+            //     console.log(BIS_dict[title])
+            //     if(BIS_dict[title][0] == BIS_dict[title][1]) {
+
+            //         $.doTimeout( 1000, function(){
+            //             var elems = current_docs[title]
+            //             for (var i in elems ) {
+            //                 var id = elems[i]
+            //                 console.log("deleting icons for :"+"#delete_"+id)
+            //                 $("#delete_"+id).remove();  
+            //             }
+            //         });
+            //     }
+            //     // // // if(BIS_dict[title][0] == BIS_dict[title][1])
+            //     // // //     $("#delete_"+node_id).remove();         
+            //     // // current_docs = []
+            //     // {% for doc in documents %}
+            //     //     id = "doc_{{doc.id}}"
+            //     //     title = "{{doc.name}}"
+            //     //     console.log(BIS_dict[title])
+            //     //     if(BIS_dict[title] && BIS_dict[title][0] > BIS_dict[title][1]) {
+            //     //         var del_img = '<span id="delete_{{doc.id}}"><a title="Delete this document!" style="cursor: pointer;" onclick="deleteNode('+"{{doc.id}}"+',\''+title+'\')"><img width="20px" src="/static/img/delete-big.png"></img></a><span>'
+            //     //         $("#"+id).prepend( del_img )
+            //     //     }
+            //     // {% endfor %}
+            // });
+
+        },
+        error: function(result) {
+            console.log("Data not found");
+            console.log(result)
+        }
+    });
+}
+
+
+
+//'+"{{doc.id}}"+',\"'+title+'\"
+
+current_docs = {}
+if(Object.keys(BIS_dict).length>0) {
+    $("#delAll").css("visibility", "visible"); 
+    $("#delAll").show();
+} else $("#delAll").remove()
+
+{% for doc in documents %}
+    id = "doc_{{doc.id}}"
+    title = $("<div/>").html("{{doc.name}}").text()
+    current_docs["{{doc.id}}"] = title;
+    if(BIS_dict[title] && BIS_dict[title][0] > BIS_dict[title][1]) {
+        jspart = 'onclick= "deleteNode( {{doc.id}} );"'
+        var del_img = '<a title="Delete this document!" style="cursor: pointer;" '+jspart+'><img width="20px" src="/static/img/delete-big.png"></img></a>'
+        $("#"+id).prepend( del_img )
+    }
+{% endfor %}
+
+</script>
+{% endif %}

templates/tests/formexample.html

+<html>
+<body>
+
+<form action="/formexample/" enctype="multipart/form-data" method="post">
+	{% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="OK">
+</form>
+
+{% if msg  %}
+<p>Response: {{msg}}</p>
+{% endif %}
+
+</body>
+</html>
\ No newline at end of file

templates/tests/mvc.html

@@ -294,7 +294,7 @@
                     <br/>
                     Interpolation:
                     <select ng-model="graph.options.lineMode">
-                        <option ng-repeat="mode in ['bundle', 'linear']" value="{{ mode }}">{{ mode }}</option>
+                        <option ng-repeat="mode in ['linear', 'bundle']" value="{{ mode }}">{{ mode }}</option>
                     </select>
                     <span ng-if="graph.options.lineMode != 'linear'">
                         with a tension of