Merge branch 'unstable' into testing

analysis/diachronic_specificity.py

@@ -10,6 +10,8 @@ NodeType = models.NodeType.sa
 NodeNgram = models.Node_Ngram.sa
 NodeNodeNgram = models.NodeNgramNgram.sa
 Ngram = models.Ngram.sa
+Node_Metadata = models.Node_Metadata.sa
+Metadata = models.Metadata.sa
 Node = models.Node.sa
 Corpus = models.Corpus.sa
 
@@ -36,7 +38,7 @@ def result2dict(query):
     return(results)
 
 
-def diachronic_specificity(corpus_id, string, order=True):
+def diachronic_specificity(corpus_id, terms, order=True):
     ''' 
     Take as parameter Corpus primary key and text of ngrams.
     Result is a dictionnary.
@@ -44,33 +46,44 @@ def diachronic_specificity(corpus_id, string, order=True):
     Values are measure to indicate diachronic specificity.
     Nowadays, the measure is rather simple: distance of frequency of period from mean of frequency of all corpus.
     '''
-    corpus = session.query(Node).get(int(corpus_id))
-    ngram = session.query(Ngram).filter(Ngram.terms == string).first()
-    
-    ngram_frequency_query = session.query(Node.metadata['publication_year'], func.count('*'))                        .join(NodeNgram, Node.id == NodeNgram.node_id)                        .filter( NodeNgram.ngram == ngram)                        .filter(Node.parent_id == corpus.id)                        .group_by(Node.metadata['publication_year']).all()
-
-    document_year_sum_query = session.query(Node.metadata['publication_year'], func.count('*'))            .filter(Node.parent_id == corpus.id)            .group_by(Node.metadata['publication_year']).all()
-            
-            
-    document_filterByngram_year = result2dict(ngram_frequency_query)
-    document_all_year = result2dict(document_year_sum_query)
+    ngram_frequency_query = (session
+        .query(Node.metadata['publication_year'], func.count('*'))
+        .join(NodeNgram, Node.id == NodeNgram.node_id)
+        .join(Ngram, Ngram.id == NodeNgram.ngram_id)
+        .filter(Ngram.terms == terms)
+        .filter(Node.parent_id == corpus_id)
+        .group_by(Node.metadata['publication_year'])
+    )
+
+    document_year_sum_query = (session
+        .query(Node.metadata['publication_year'], func.count('*'))
+        .filter(Node.parent_id == corpus_id)
+        .group_by(Node.metadata['publication_year'])
+    )
+            
+            
+    document_filterByngram_year = dict(ngram_frequency_query.all())
+    document_all_year = dict(document_year_sum_query.all())
     #print(document_all_year)
-    data = dict()
     
-    for year in document_all_year.keys():
-       data[year] = document_filterByngram_year.get(year, 0) / document_all_year[year]
+    relative_terms_count = dict()
+    for year, total in document_all_year.items():
+        terms_count = document_filterByngram_year.get(year, 0)
+        relative_terms_count[year] = terms_count / total
     
-    mean = np.mean(list(data.values()))
+    mean = np.mean(list(relative_terms_count.values()))
     
-    data_dict = dict(zip(data.keys(), list(map(lambda x: x - mean, data.values()))))
+    relative_terms_count = {
+        key: (value - mean)
+        for key, value in relative_terms_count.items()
+    }
     
     if order == True:
-        return collections.OrderedDict(sorted(data_dict.items()))
+        return collections.OrderedDict(sorted(relative_terms_count.items()))
     else:
-        return data_dict
+        return relative_terms_count
 
 
 # For tests
-#diachronic_specificity(102750, "bayer", order=True)
-
-
+# diachronic_specificity(102750, "bayer", order=True)
+# diachronic_specificity(26128, "bee", order=True)

gargantext_web/views.py

@@ -496,11 +496,17 @@ def corpus(request, project_id, corpus_id):
     except Exception as error:
         print(error)
     
+    try:
+        processing = corpus.metadata['Processing']
+    except:
+        processing = 0
+
     html = t.render(Context({\
             'user': user,\
             'date': date,\
             'project': project,\
             'corpus' : corpus,\
+            'processing' : processing,\
 #            'documents': documents,\
             'number' : number,\
             'dates' : chart,\

init/README.rst

@@ -153,4 +153,10 @@ $ sudo aptitude install tmux
 $ tmux -c ./manage.py celery worker --loglevel=info
 $ python manage.py runserver
 
+Versions on git
+---------------
+
+stable branch    : current version  for production server with nginx config (and tina branch for tina/apache server)
+testing branch   : current version for users' tests
+unstable branch  : current version for developers
 

init_cooc.py

+# Without this, we couldn't use the Django environment
+
+import os
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext_web.settings")
+os.environ.setdefault("DJANGO_HSTORE_GLOBAL_REGISTER", "False")
+
+
+# We're gonna use all the models!
+
+from node.models import User, NodeType, Node
+
+
+user = User.objects.get(username = 'contro2015.lait')
+
+# Reset: all data
+
+try:
+    typeDoc     = NodeType.objects.get(name='Cooccurrence')
+except Exception as error:
+    print(error)
+
+Node.objects.filter(user=user, type=typeDoc).all().delete()
+
+exit()

node/models.py

@@ -237,12 +237,16 @@ class Node(CTENode):
     @current_app.task(filter=task_method)
     def workflow(self, keys=None, ngramsextractorscache=None, ngramscaches=None, verbose=False):
         print("In workflow() START")
+        self.metadata['Processing'] = 1
+        self.save()
         self.parse_resources()
         type_document   = NodeType.objects.get(name='Document')
         self.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
         from analysis.functions import do_tfidf
         do_tfidf(self)
         print("In workflow() END")
+        self.metadata['Processing'] = 0
+        self.save()
 
 class Node_Metadata(models.Model):
     node        = models.ForeignKey(Node, on_delete=models.CASCADE)

static/js/gargantext.angular.js

@@ -381,7 +381,7 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
                 y: {key: 'y', type: 'linear', type: 'numeric'},
             },
             tension: 1.0,
-            lineMode: 'bundle',
+            lineMode: 'linear',
             tooltip: {mode: 'scrubber', formatter: function(x, y, series) {
                 var grouping = groupings.datetime[$scope.groupingKey];
                 return grouping.representation(x) + ' → ' + y;
@@ -442,6 +442,13 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
             angular.forEach(results, function(result, r){
                 var x = grouping.truncate(result[0]);
                 var y = parseFloat(result[1]);
+                if (dataObject[x] === undefined) {
+                    var row = [];
+                    angular.forEach($scope.datasets, function(){
+                        row.push(0);
+                    });
+                    dataObject[x] = row;
+                }
                 dataObject[x][datasetIndex] += y;
             });
         });

templates/about.html

@@ -35,15 +35,20 @@
 										<ul>
 												<li>Version 1.0</li>
 														<ul>
-																<li>Beta Version </li>
-																<li>Licence of Gargantext is GPL v3+ </li>
+																<li>[Start] Beta Version </li>
+																<li>[Law] Licence of Gargantext is GPL v3+ </li>
 														</ul>
 												<li>Version 1.0.5</li>
 														<ul>
-																<li>Bug resolution: xml zipped from Mac</li>
-																<li>Bug resolution: french accents in filenames</li>
+																<li>Bug resolution: [Import] xml zipped from Mac</li>
+																<li>Bug resolution: [Import] french accents in filenames</li>
 																<li>New features: [Advanced chart] ngrams completion</li>
-																<li>New features: button to delete all duplicates</li>
+																<li>New features: [Duplicates management] button to delete all duplicates</li>
+														</ul>
+												<li>Version 1.0.6</li>
+														<ul>
+																<li>Bug resolution: [Advanced chart] one can make comparisons with different corpora at different scales</li>
+																<li>Bug resolution: [Graph] Graph link can not be executed until workflow is finished.</li>
 														</ul>
 										</ul>
 								</div>

templates/corpus.html

@@ -131,7 +131,11 @@
 
 				<div class="col-md-4">
 						<div class="jumbotron">
+                {% if processing == "0" %}
 								<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% else %}
+                <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
+                {% endif %}
 						<ol>
 								<li>Visualize</li>
 								<li>Explore</li>

templates/menu.html

@@ -65,7 +65,7 @@
 <hr>
 
 <footer>
-        <p>Gargantext, version 1.0.5, Copyrights CNRS {{ date.year }}.</p>
+        <p>Gargantext, version 1.0.6, Copyrights CNRS {{ date.year }}.</p>
 </footer>
 
 

templates/tests/mvc.html

@@ -294,7 +294,7 @@
                     <br/>
                     Interpolation:
                     <select ng-model="graph.options.lineMode">
-                        <option ng-repeat="mode in ['bundle', 'linear']" value="{{ mode }}">{{ mode }}</option>
+                        <option ng-repeat="mode in ['linear', 'bundle']" value="{{ mode }}">{{ mode }}</option>
                     </select>
                     <span ng-if="graph.options.lineMode != 'linear'">
                         with a tension of