Merge branch 'master' of ssh://delanoe.org:1979/gargantext into mat

gargantext_web/api.py

@@ -66,7 +66,7 @@ class CorpusController:
         corpus = corpusQuery.first()
         if corpus.type.name != 'Corpus':
             raise Http404("No such corpus: %d" % (corpus_id, ))
-        # if corpus.user != {currentUser}:
+        # if corpus.user != request.user:
         #     raise Http403("Unauthorized access.")
         return corpus
 
@@ -177,10 +177,13 @@ class CorpusController:
             else:
                 raise ValidationError('Unrecognized "filter[]=%s"' % (filter, ))
         # query building: initializing SQL
-        sql_0 = _sql_cte
+        #sql_0 = _sql_cte
+        sql_0 = ''
         sql_1 = '\nSELECT '
-        sql_2 = '\nFROM %s\nINNER JOIN cte ON cte."id" = %s.id' % (Node._meta.db_table, Node._meta.db_table, )
-        sql_3 = '\nWHERE ((NOT cte.id = \'%d\') AND (\'%d\' = ANY(cte."path")))' % (corpus.id, corpus.id, )
+        #sql_2 = '\nFROM %s\nINNER JOIN cte ON cte."id" = %s.id' % (Node._meta.db_table, Node._meta.db_table, )
+        sql_2 = '\nFROM %s' % (Node._meta.db_table, )
+        #sql_3 = '\nWHERE ((NOT cte.id = \'%d\') AND (\'%d\' = ANY(cte."path")))' % (corpus.id, corpus.id, )
+        sql_3 = '\nWHERE (%s.parent_id = %d)' % (Node._meta.db_table, corpus.id, )
         # query building: assembling SQL
         sql_1 += ", ".join(columns)
         sql_2 += "\nINNER JOIN %s ON %s.id = %s.type_id" % (NodeType._meta.db_table, NodeType._meta.db_table, Node._meta.db_table, )

gargantext_web/settings.py

@@ -63,7 +63,6 @@ INSTALLED_APPS = (
     'django.contrib.staticfiles',
     'django_extensions',
     #'south',
-    #'documents',
     'cte_tree',
     'node',
     'ngram',

gargantext_web/urls.py

@@ -35,13 +35,13 @@ urlpatterns = patterns('',
     url(r'^exploration$', views.exploration),
 
     url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
-    url(r'^graph.json$', views.send_graph),
+    url(r'^graph.json$', views.json_node_link),
 
     url(r'^api/corpus/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
     url(r'^api/corpus/(\d+)/metadata$', gargantext_web.api.CorpusController.metadata),
     url(r'^api/corpus/(\d+)/data$', gargantext_web.api.CorpusController.data),
 
-    # url(r'^graph-it$', gargantext_web.api.CorpusController.data),
+    url(r'^graph-it$', views.graph_it),
 )
 
 from django.conf import settings

gargantext_web/views.py

@@ -8,7 +8,7 @@ from django.template import Context
 #from documents.models import Project, Corpus, Document
 
 from node.models import Language, ResourceType, Resource
-from node.models import Node, NodeType, Node_Resource, Project, Corpus
+from node.models import Node, NodeType, Node_Resource, Project, Corpus, NodeNgramNgram
 from node.admin import CorpusForm, ProjectForm, ResourceForm
 
 from django.contrib.auth.models import User
@@ -140,9 +140,11 @@ def project(request, project_id):
     user = request.user
     date = datetime.datetime.now()
     
+    type_corpus = NodeType.objects.get(name='Corpus')
+
     project = Node.objects.get(id=project_id)
-    corpora = project.children.all()
-    number  = project.children.count()
+    corpora = project.children.filter(type=type_corpus)
+    number  = len(corpora)
 
 
     # DONUT corpora representation
@@ -228,7 +230,8 @@ def project(request, project_id):
                     )
 
             try:
-                corpus.parse_resources.apply_async((), countdown=1)
+                #corpus.parse_resources.apply_async((), countdown=1)
+                corpus.parse_resources()
             except Exception as error:
                 print(error)
 
@@ -413,18 +416,78 @@ def send_csv(request, corpus_id):
 
     return response
 
-def send_graph(request):
+def json_node_link(request):
     '''
     Create the HttpResponse object with the graph dataset.
     '''
-    response = HttpResponse(content_type='text/json')
-    response['Content-Disposition'] = 'attachment; filename="graph.json"'
 
-    writer = csv.writer(response)
+    import pandas as pd
+    from copy import copy
+    import numpy as np
+    import networkx as nx
+    from networkx.readwrite import json_graph
+    from gargantext_web.api import JsonHttpResponse
+    #from analysis.louvain import *
 
-    file = open('/srv/gargantext/tests/graphsam/randomgraphgen.json', 'r')
-    for line in file.readlines():
-        writer.writerow(line)
+    matrix = defaultdict(lambda : defaultdict(float))
+    labels = dict()
+    cooc = Node.objects.get(id=61314)
 
-    return response
+    for cooccurrence in NodeNgramNgram.objects.filter(node=cooc):
+        labels[cooccurrence.ngramx.id] = cooccurrence.ngramx.terms
+        labels[cooccurrence.ngramy.id] = cooccurrence.ngramy.terms
+        
+        matrix[cooccurrence.ngramx.id][cooccurrence.ngramy.id] = cooccurrence.score
+        matrix[cooccurrence.ngramy.id][cooccurrence.ngramx.id] = cooccurrence.score
+
+
+    df = pd.DataFrame(matrix).T.fillna(0)
+    x = copy(df.values)
+    x = x / x.sum(axis=1)
 
+
+    # Removing unconnected nodes
+    threshold = min(x.max(axis=1))
+    matrix_filtered = np.where(x > threshold, 1, 0)
+    #matrix_filtered = np.where(x > threshold, x, 0)
+    
+    G = nx.from_numpy_matrix(matrix_filtered)
+    G = nx.relabel_nodes(G, dict(enumerate([ labels[x] for x in list(df.columns)])))
+    #G = nx.relabel_nodes(G, dict(enumerate(df.columns)))
+    
+    # Removing too connected nodes (find automatic way to do it)
+    outdeg = G.degree()
+    to_remove = [n for n in outdeg if outdeg[n] >= 10]
+    G.remove_nodes_from(to_remove)
+    
+    for node in G.nodes():
+        try:
+            #node,type(labels[node])
+            G.node[node]['label'] = node
+#            G.node[node]['color'] = '19,180,300'
+        except Exception as error:
+            print(error)
+    
+    data = json_graph.node_link_data(G)
+#    data = json_graph.node_link_data(G, attrs={\
+#            'source':'source',\
+#            'target':'target',\
+#            'weight':'weight',\
+#            #'label':'label',\
+#            #'color':'color',\
+#            'id':'id',})
+    #print(data)
+    return JsonHttpResponse(data)
+
+
+
+def graph_it(request):
+    '''The new multimodal graph.'''
+    t = get_template('graph-it.html')
+    user = request.user
+    date = datetime.datetime.now()
+    html = t.render(Context({
+        'user': user,
+        'date': date,
+    }))    
+    return HttpResponse(html)

init/cte_tree.models.diff

+--- models.old.py	2014-11-18 12:12:12.289169335 +0100
++++ models.py	2014-11-18 12:11:31.157171030 +0100
+@@ -118,7 +118,7 @@
+                             **kwargs)
+                     
+                     def maybe_alias(table):
+-                        if self.query.table_map.has_key(table):
++                        if table in self.query.table_map:
+                             return self.query.table_map[table][0]
+                         return table
+                     
+@@ -153,11 +153,7 @@
+                                 name = field[0]
+                             else:
+                                 name = field
+-                            if self.query.model._meta.get_field_by_name(name)[
+-                                0].db_type().startswith('varchar'):
+-                                return 'CAST (T."%s" AS TEXT)' % name
+-                            else:
+-                                return 'T."%s"' % name
++                            return 'T."%s"' % name
+ 
+                     # The primary key is used in the path; in case it is of a
+                     # custom type, ensure appropriate casting is performed. This

init/graph.sh

 #Usage : ./manage.ph > graph.pdf
 
-./manage.py graph_models documents | dot -Tpdf 
+./manage.py graph_models node | dot -Tpdf 

node/admin.py

@@ -2,12 +2,19 @@ from django.contrib import admin
 from django.forms import ModelForm, ModelChoiceField
 from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
 
-from node.models import NodeType, Language, Node, Project, Corpus, Document, ResourceType, Resource, Node_Ngram, Node_Resource
+from node.models import NodeType, Language, Node, \
+        Project, Corpus, Document, \
+        ResourceType, Resource, \
+        Ngram, Node_Ngram, Node_Resource
 
 class ResourceInLine(admin.TabularInline):
     model = Resource
     extra = 0
 
+class NgramAdmin(admin.ModelAdmin):
+    list_display = ('N', 'Terms')
+    search_fields = ('Terms',)
+
 class NodeAdmin(admin.ModelAdmin):
     exclude = ('user', 'path', 'depth', 'numchild', 'ngrams')
     list_display = ('name', 'date')
@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin)
 admin.site.register(Corpus, CorpusAdmin)
 admin.site.register(Document, DocumentAdmin)
 
+
 admin.site.register(Node_Resource)
+admin.site.register(Ngram)
 admin.site.register(Node_Ngram)
 

node/models.py

@@ -46,6 +46,9 @@ class Ngram(models.Model):
     n           = models.IntegerField()
     terms       = models.CharField(max_length=255)
     nodes       = models.ManyToManyField(through='Node_Ngram', to='Node')
+    def __str__(self):
+        return self.terms
+
 
 class Resource(models.Model):
     user        = models.ForeignKey(User)
@@ -207,8 +210,6 @@ class Node_Ngram(models.Model):
     def __str__(self):
         return "%s: %s" % (self.node.name, self.ngram.terms)
 
-
-
 class Project(Node):
     class Meta:
         proxy=True

templates/explorer.html

@@ -21,16 +21,6 @@
 {% block content %}
 
 
-
-    <link rel="stylesheet" href="{% static "js/libs/css2/freshslider.css" %} media="screen">
-    <link rel="stylesheet" href="{% static "js/libs/css2/custom.css" %} media="screen">
-    <link rel="stylesheet" href="{% static "js/libs/css2/sidebar.css" %} media="screen">
-
-
-
-
-
-
     <!-- this is the tweakbar -->
     <div id="defaultop" class="navbar navbar-default">
         <div class="navbar-collapse collapse navbar-responsive-collapse">
@@ -68,6 +58,12 @@
                          </a>
                        </li>
 
+                        <li>
+                          <a>
+                            <div id="graphid" style="visibility: hidden;">/graph.json</div>
+                          </a>
+                        </li>
+
                       <!--
                       <li>
                          <a>
@@ -123,6 +119,8 @@
                      </a></li>
                   </ul>
 
+
+                  <!--
                   <ul id="colorGraph" class="nav navbar-nav navbar-right">
                      <li class="dropdown">
                       <a href="#" class="dropdown-toggle" data-toggle="dropdown">
@@ -137,6 +135,7 @@
                       </ul>
                      </li>
                   </ul>
+                  -->
 
 
 
@@ -173,7 +172,9 @@
 
           <div style="visibility: hidden;" id="sigma-othergraph"></div>
 
-          <img id="semLoader" style="position:absolute; top:50%; left:40%; width:80px;" src="{% static "js/libs/img2/loading-bar.gif" %}" ></img>
+          <div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
+							<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
+          </div>
 
           <ul id="ctlzoom">
 
@@ -185,12 +186,13 @@
                           <a href="#" id="aUnfold" class="rightarrow"></a>
                       </div>
                   </li> 
-
+                  <!--
                    <li>
                       <a href="#" id="geomapicon" onclick="$('#geomapmodal').modal('show'); callGeomap();">
                         <img title="World Map Distribution" width="34px" src="{% static "js/libs/img2/world.png" %}" ></img>
                       </a>
                   </li>
+                  --> 
 
 
                    <li>
@@ -232,7 +234,27 @@
       <div id="leftcolumn">      
         <div id="tips"></div>
         <div id="names"></div>
+
+
+        <br>
+        <div id="tab-container" class='tab-container' style="display: none;">
+
+           <ul class='etabs'>
+             <li id="taboppos" class='tab'><a href="#tabs1">Opposite-Neighbors</a></li>
+             <li id="tabneigh" class='tab'><a href="#tabs2">Neighbors</a></li>
+           </ul>
+
+           <div class='panel-container'>
+              <div id="tabs1">
                 <div id="opossiteNodes"></div>
+              </div>
+              <div id="tabs2">
+                <div id="sameNodes"></div>
+              </div>
+           </div>
+
+        </div>
+
         <!-- <div id="topPapers"></div> -->
         <div id="information"></div>
       </div>
@@ -331,13 +353,14 @@
 
 
 
-
     <script src="{% static "js/libs/jquery/jquery-1.10.2.js" %}" type="text/javascript"></script>
     <script src="{% static "js/libs/jquery/jquery-ui.js" %}" type="text/javascript"></script>
     <script src="{% static "js/libs/jquery/jquery.ba-dotimeout.min.js" %}" type="text/javascript"></script>
     <script src="{% static "js/libs/jquery/jquery.mousewheel.min.js" %}" type="text/javascript"></script>
 
     <script type="text/javascript" src="{% static "js/libs/freshslider.1.0.js" %}"></script>
+    <script type="text/javascript" src="{% static "js/libs/readmore.js" %}"></script>
+    <script type="text/javascript" src="{% static "js/libs/jquery/jquery.easytabs.min.js" %}"></script>
     
     <script src="{% static "js/libs/bootstrap/js/bootstrap.min.js" %}"></script>
     <script src="{% static "js/libs/bootstrap/js/bootstrap-modal.js" %}" type="text/javascript"></script>
@@ -346,7 +369,6 @@
 
     <script src="{% static "js/tinawebJS/globalUtils.js" %}" type="text/javascript"></script>
 
-    
     <script src="{% static "js/tinawebJS/sigma.min.js" %}" type="text/javascript" language="javascript"></script>
     <script src="{% static "js/tinawebJS/sigma.forceatlas2.js" %}" type="text/javascript" language="javascript"></script>
     <script src="{% static "js/settings_explorerjs.js" %}" type="text/javascript" language="javascript"></script>
@@ -360,5 +382,14 @@
     <script src="{% static "js/tinawebJS/DataScanner.js" %}" type="text/javascript" language="javascript"></script>
     <script src="{% static "js/tinawebJS/main.js" %}" type="text/javascript" language="javascript"></script>
 
+    <script type="text/javascript">
+        $('#tab-container').easytabs({updateHash:false});
+        // Popup window code
+        function newPopup(url) {
+          popupWindow = window.open(
+            url,'popUpWindow','height=700,width=800,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=no,menubar=no,location=no,directories=no,status=no')
+        }
+    </script>
+
 
 {% endblock %}

templates/menu.html

@@ -60,7 +60,8 @@
 {% load staticfiles %}
 
 
-<script src="{% static "js/jquery/jquery-1.9.1.min.js"%}"></script>
+<!-- <script src="{% static "js/jquery/jquery-1.9.1.min.js"%}"></script> -->
+
 <script src="{% static "js/easy/jquery.easypiechart.js"%}"></script>
 
 <script type="text/javascript">