Merge branch 'refactoring-rom' into refactoring-alex

gargantext/util/json.py

@@ -27,7 +27,8 @@ class JSONEncoder(json.JSONEncoder):
         else:
             return super(self.__class__, self).default(obj)
 
-json_encoder = JSONEncoder(indent=4)
+# json_encoder = JSONEncoder(indent=4)
+json_encoder = JSONEncoder()   # compact json
 
 def json_dumps(obj):
     return json.dumps(obj, cls=JSONEncoder)

gargantext/views/api/ngramlists.py

+from gargantext.util.http     import APIView, get_parameters, JsonHttpResponse,\
+                                     ValidationException
+from gargantext.util.db       import session, aliased
+from gargantext.util.db_cache import cache
+from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram
+from gargantext.util.lists    import Translations
+from sqlalchemy               import desc
+# from gargantext.constants import *
+# from gargantext.util.validation import validate
+# from collections import defaultdict
+
+
+def _query_list(list_id,
+                pagination_limit=None, pagination_offset=None,
+                details=False, scoring_metric_id=None
+                ):
+    """
+    Paginated listing of ngram_ids in a NodeNgram lists.
+
+    Works for a mainlist or stoplist or maplist (not grouplists!)
+
+    Parameter:
+      - pagination_limit, pagination_offset
+      - details: if False, send just the array of ngram_ids
+                 if True, send triples with (ngram_id, term, scoring)
+                                                             ^^^^^^^
+      - scoring_metric_id: id of a scoring metric node   (TFIDF or OCCS)
+                           (for details and sorting)
+    """
+    if not details:
+        # simple contents
+        query = session.query(NodeNgram.ngram_id)
+    else:
+        # detailed contents (terms and some NodeNodeNgram for score)
+        query = (session
+                    .query(
+                        NodeNgram.ngram_id,
+                        Ngram.terms,
+                        NodeNodeNgram.score
+                     )
+                    .join(Ngram, NodeNgram.ngram_id == Ngram.id)
+                    .join(NodeNodeNgram, NodeNgram.ngram_id == NodeNodeNgram.ngram_id)
+                    .filter(NodeNodeNgram.node1_id == scoring_metric_id)
+                    .order_by(desc(NodeNodeNgram.score))
+                )
+
+    # main filter
+    # -----------
+    query = query.filter(NodeNgram.node_id == list_id)
+
+    if pagination_limit:
+        query = query.limit(pagination_limit)
+
+    if pagination_offset:
+        query = query.offset(pagination_offsets)
+
+    return query
+
+
+
+class List(APIView):
+    """
+    see already available API query api/nodes/<list_id>?fields[]=ngrams
+    """
+    pass
+
+class ListFamily(APIView):
+    """
+    Compact combination of *multiple* list info
+        custom made for the "terms" view
+    ---
+    Sends all JSON info of a collection of the 4 list types of a corpus
+    (or for any combination of lists that go together):
+      - a mainlist
+      - an optional stoplist
+      - an optional maplist
+      - an optional grouplist
+
+    USAGE EXEMPLES
+    HOST/api/ngramlists/family?corpus=2
+    HOST/api/ngramlists/family?corpus=2&glance=10
+    HOST/api/ngramlists/family?mainlist=91&scoring=94
+    HOST/api/ngramlists/family?mainlist=91&scoring=94&glance=10
+    HOST/api/ngramlists/family?mainlist=91&stoplist=90&scoring=94
+     etc.
+
+    REST Parameters:
+    "glance=20"
+        use pagination to only load the k top ngrams of the mainlist
+        (useful for fast loading of terms view)
+    "corpus=ID"
+        the corpus id to retrieve all 4 lists
+    "scoring=ID"
+        the scoring node (defaults to the OCCURRENCES child of the corpus)
+    "mainlist=ID&scoring=ID[&stoplist=ID&groups=ID&maplist=ID]"
+        alternative call syntax without specifying a corpus
+        (uses all explicit IDs of the lists => gives the possibility for custom term views)
+    """
+    def get(self, request):
+
+        parameters = get_parameters(request)
+        glance_limit = None
+        mainlist = None
+        scores_id = None
+        groups_id = None
+        secondary_lists = {'maplist':None, 'stoplist':None}
+
+        # 1) retrieve a mainlist_id and other lists
+        ##########################################
+
+        # simple request: just refers to the parent corpus
+        # ------------------------------------------------
+        if "corpus" in parameters:
+            corpus_id = parameters['corpus']
+            corpus = cache.Node[corpus_id]
+            # with a corpus_id, the explicit scoring pointer is optional
+            if "scoring" in parameters:
+                scores_id = parameters['scoring']
+            else:
+                scores_id = corpus.children('OCCURRENCES').first().id
+            # retrieve the family of lists that have corpus as parent
+            mainlist = corpus.children('MAINLIST').first().id,
+            groups_id = corpus.children('GROUPLIST').first().id
+            secondary_lists['stoplist'] = corpus.children('STOPLIST').first().id
+            secondary_lists['maplist']  = corpus.children('MAPLIST').first().id,
+
+        # custom request: refers to each list individually
+        # -------------------------------------------------
+        elif "mainlist" in parameters and "scoring" in parameters:
+            mainlist = parameters['mainlist']
+            scores_id = parameters['scoring']
+            groups_id = None
+            if 'groups' in parameters:
+                groups_id = parameters['scoring']
+            for k in ['stoplist', 'maplist']:
+                if k in parameters:
+                    secondary_lists[k] = parameters[k]
+
+        # or request has an error
+        # -----------------------
+        else:
+            raise ValidationException(
+                "Either a 'corpus' parameter or 'mainlist' & 'scoring' params are required"
+                )
+
+
+        # 2) get the infos for each list
+        ################################
+        ngraminfo = {}           # ngram details sorted per ngram id
+        linkinfo  = {}           # ngram groups sorted per ngram id
+        listmembers = {}         # ngram ids sorted per list name
+        if "glance" in parameters:
+            # glance <=> only mainlist AND only k top ngrams
+            glance_limit = int(parameters['glance'])
+            mainlist_query = _query_list(mainlist, details=True,
+                                          pagination_limit = glance_limit,
+                                          scoring_metric_id= scores_id)
+        else:
+            # infos for all ngrams
+            mainlist_query = _query_list(mainlist, details=True,
+                                          scoring_metric_id= scores_id)
+            # and for the other lists (stop and map)
+            for li in secondary_lists:
+                li_elts = _query_list(secondary_lists[li], details=False
+                                      ).all()
+                listmembers[li] = {ng[0]:True for ng in li_elts}
+
+            # and the groupings
+            if groups_id:
+                links = Translations(groups_id)
+                linkinfo = links.groups
+
+        # the output form
+        ngraminfo = {}
+        for ng in mainlist_query.all():
+            ng_id   = ng[0]
+            # id => [term, weight]
+            ngraminfo[ng_id] = ng[1:]
+
+        return JsonHttpResponse({
+            'ngraminfos' : ngraminfo,
+            'listmembers' : listmembers,
+            'links' : linkinfo
+        })

gargantext/views/api/urls.py

 from django.conf.urls import url
 
 from . import nodes
+from . import ngramlists
 
 
 urlpatterns = [
@@ -8,4 +9,17 @@ urlpatterns = [
     url(r'^nodes/(\d+)$', nodes.NodeResource.as_view()),
 
     url(r'^nodes/(\d+)/facets$', nodes.CorpusFacet.as_view()),
+
+    # get a list of ngram_ids or ngram_infos by list_id
+    #
+    # url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
+
+    # entire combination of lists from a corpus
+    # (or any combination of lists that go together :
+    #   - a mainlist
+    #   - an optional stoplist
+    #   - an optional maplist
+    #   - an optional grouplist
+    # aka lexical model
+    url(r'^ngramlists/family$', ngramlists.ListFamily.as_view()),
 ]

gargantext/views/pages/terms.py

+from gargantext.util.http     import requires_auth, render, settings
+from gargantext.util.db       import session
+from gargantext.util.db_cache import cache
+from gargantext.models        import Node
+from datetime                 import datetime
+
+@requires_auth
+def ngramtable(request, project_id, corpus_id):
+    '''
+    Browse and modify all lists together.
+       => maplist and mainlist terms in a table
+          with groupings as '+' nodes
+       => uses API GET batch of lists
+       => uses API PUT/DEL for list modifications (TODO)
+       => uses frontend AJAX through Ngrams_dyna_charts_and_table.js
+    # TODO refactor Ngrams_dyna_charts_and_table.js
+    '''
+    # corpus still necessary to find all lists
+    corpus = cache.Node[corpus_id]
+
+    # and the project just for project.id in corpusBannerTop
+    project = cache.Node[project_id]
+
+    # rendered page : journals.html
+    return render(
+        template_name = 'pages/corpora/terms.html',
+        request = request,
+        context = {
+            'debug': settings.DEBUG,
+            'date': datetime.now(),
+            'project': project,
+            'corpus' : corpus,
+            'view': 'terms'
+        },
+    )

gargantext/views/pages/urls.py

 from django.conf.urls import url
 
 from . import main, auth
-from . import projects, corpora
+from . import projects, corpora, terms
 
 
 urlpatterns = [
@@ -25,4 +25,7 @@ urlpatterns = [
 
     # corpus by journals
     url(r'^projects/(\d+)/corpora/(\d+)/journals/?$', corpora.docs_by_journals),
+
+    # terms table for the corpus
+    url(r'^projects/(\d+)/corpora/(\d+)/terms/?$', terms.ngramtable),
 ]

static/css/gargantext/tables.css

-// styles for dynatables
+/* styles for dynatables */
 
 .no-transition {
   -webkit-transition: height 0.1s;
@@ -27,3 +27,105 @@ th a {
     margin:0 auto;
     display:block;
 }
+
+
+/* notes under table titles */
+th p.note {
+  color: #ccc;
+  font-size: 0.6em;
+  margin: 1em 0 0 0 ;
+}
+
+th p.note > input {
+  float: left;
+  margin: 0 .2em 0 0 ;
+}
+
+th p.note > label {
+  float: left;
+}
+
+tr:hover {
+  cursor: pointer;
+  font-weight: bold;
+}
+
+/* ngram states */
+.normal {
+  color: black;
+}
+
+.delete {
+  color:red;
+  opacity: 0.8;
+}
+.keep {
+  color:green;
+}
+
+.group {
+  color:white;
+  pointer-events: none;
+  cursor: default;
+}
+
+/* group box with + */
+.group_box {
+  font-size: 90%;
+  border: 1px solid blue;
+}
+.group_box .header {
+  font-size: 120%;
+}
+.group_box .content {
+  border: 1px solid yellow;
+}
+
+#group_flag {
+}
+
+.dynatable-record-count {
+  font-size: 0.7em;
+}
+.dynatable-pagination-links {
+  font-size: 0.7em;
+}
+
+input[type=radio] + label {
+    display:inline-block;
+    margin:-2px;
+    padding: 4px 12px;
+    margin-bottom: 0;
+    font-size: 14px;
+    line-height: 20px;
+    color: #333;
+    text-align: center;
+    text-shadow: 0 1px 1px rgba(255,255,255,0.75);
+    vertical-align: middle;
+    cursor: pointer;
+    background-color: #f5f5f5;
+    background-image: -moz-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));
+    background-image: -webkit-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -o-linear-gradient(top,#fff,#e6e6e6);
+    background-image: linear-gradient(to bottom,#fff,#e6e6e6);
+    background-repeat: repeat-x;
+    border: 1px solid #ccc;
+    border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+    border-color: rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);
+    border-bottom-color: #b3b3b3;
+    filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);
+    filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
+    -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+}
+
+input[type=radio]:checked + label {
+    background-image: none;
+    outline: 0;
+    -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+        background-color:#e0e0e0;
+}

templates/pages/corpora/terms.html

+{% extends "pages/menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+
+<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/d3/dc.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.dynatable.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/gargantext/tables.css"%}"/>
+
+<script type="text/javascript" src="{% static "js/d3/d3.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/crossfilter.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/dc.js"%}"></script>
+
+{% endblock %}
+
+
+{% block content %}
+
+<div class="container">
+    <div class="jumbotron">
+        <div class="row">
+            <div id="monthly-move-chart">
+                <center>
+                        Select a time range in the chart with blue bars to zoom in
+                <p align="center">
+                <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
+                <a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
+                <div class="clearfix"></div>
+                </center>
+            </div>
+        </div>
+        <div class="row">
+            <div id="monthly-volume-chart"></div>
+        </div>
+        <div id="content_loader">
+          <br>
+          <center>
+            <img width="10%" src="{% static "img/ajax-loader.gif"%}"></img>
+          </center>
+          <br>
+        </div>
+
+        <input type="hidden" id="list_id" value="{{ list_id }}"></input>
+
+        <div class="row">
+            <div class="panel panel-default">
+
+                <div class="panel-heading">
+                  <h4 class="panel-title">
+                    <a data-toggle="collapse" data-target="#terms_table" href="#">
+                        <!-- Final_UpdateTable redraws the dynatable if necessary -->
+                        <p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg">
+                          Close term list
+                        </p>
+                    </a>
+                  </h4>
+                </div>
+
+                <div id="terms_table" class="panel-collapse collapse in no-transition" role="tabpanel">
+                    <div class="panel-body">
+                      <div id="div-table">
+                          <!-- (table id="my-ajax-table") dynamically set by Ngrams_dyna_chart_and_table -->
+                      </div>
+
+                      <!-- under the table -->
+                      <p align="right">
+                          <button id="Save_All" class="btn btn-primary">Save changes permanently</button>
+                      </p>
+                    </div>
+                </div> <!-- /div panel-collapse -->
+
+            </div> <!-- /div panel -->
+        </div> <!-- /row with the dynatable panels -->
+
+    </div> <!-- /jumbotron -->
+
+    <button id="ImportList" onclick="GetUserPortfolio();" class="btn btn-warning">
+        Import a Corpus-List
+    </button>
+
+</div> <!-- /container -->
+
+
+
+
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/bootstrap/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
+
+<!-- custom-lib for dynatable.js and dc.js -->
+<script type="text/javascript" src="{% static "js/gargantext/NGrams_dyna_chart_and_table.js" %}"></script>
+
+{% endblock %}

templates/pages/menu.html

@@ -68,6 +68,7 @@
                             <center>
                                 <a type="button" class="btn btn-default {% if view == "documents" %}active{%endif%}" href="/projects/{{project.id}}/corpora/{{ corpus.id }}/">Documents</a>
                                 <a type="button" class="btn btn-default {% if view == "journals" %}active{%endif%}" href="/projects/{{project.id}}/corpora/{{ corpus.id }}/journals">Journals</a>
+                                <a type="button" class="btn btn-default {% if view == "terms" %}active{%endif%}" href="/projects/{{project.id}}/corpora/{{ corpus.id }}/terms">Terms</a>
                             </center>
                         </div>
                     </div>