Update/delete possible for terms table (TODO: groupings)

gargantext/util/db.py

@@ -35,8 +35,7 @@ from sqlalchemy.ext.mutable import MutableDict, MutableList
 # useful for queries
 
 from sqlalchemy.orm import aliased
-from sqlalchemy import func
-
+from sqlalchemy import func, desc
 
 # bulk insertions
 

gargantext/views/api/ngramlists.py

+"""
+API views for advanced operations on ngrams and ngramlists
+-----------------------------------------------------------
+    - retrieve several lists together ("family")
+    - retrieve detailed list infos (ngram_id, term strings, scores...)
+    - modify NodeNgram lists (PUT/DEL an ngram to a MAINLIST OR MAPLIST...)
+"""
+
 from gargantext.util.http     import APIView, get_parameters, JsonHttpResponse,\
-                                     ValidationException
-from gargantext.util.db       import session, aliased
+                                     ValidationException, Http404
+from gargantext.util.db       import session, aliased, desc
 from gargantext.util.db_cache import cache
 from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram
-from gargantext.util.lists    import Translations
-from sqlalchemy               import desc
-# from gargantext.constants import *
-# from gargantext.util.validation import validate
-# from collections import defaultdict
+from gargantext.util.lists    import UnweightedList, Translations
 
 
 def _query_list(list_id,
@@ -64,6 +68,116 @@ class List(APIView):
     """
     pass
 
+
+class ListChange(APIView):
+    """
+    Any ngram action on standard NodeNgram lists (MAIN, MAP, STOP)
+
+    USAGE EXEMPLE:
+    HOST/api/ngramlists/change?list=42&ngrams=1,2,3,4,5
+                                vvvvvv          ||||||
+                               old list         vvvvvv
+                               to modify     new list items
+                                  |                |
+                                  v                v
+     2 x UnweightedLists:   self.base_list   self.change_list
+
+    We use DEL/PUT HTTP methods to differentiate the 2 basic rm/add actions
+    They rely only on inline parameters (no need for payload data)
+
+    No chained effects: eg removing from MAPLIST will not remove
+                          automatically from associated MAINLIST
+
+    NB: request.user is also checked for current authentication status
+    """
+
+    def initial(self, request):
+        """
+        Before dispatching to put(), delete()...
+
+        1) Checks current user authentication to prevent remote DB manipulation
+        2) Prepares self.list_objects from params
+        """
+        if not request.user.is_authenticated():
+            raise Http404()
+            # can't use return in initial() (although 401 maybe better than 404)
+            # can't use @requires_auth because of positional 'self' within class
+
+        # get validated params
+        self.params = get_parameters(request)
+        (self.base_list, self.change_list) = ListChange._validate(self.params)
+
+    def put(self, request):
+        """
+        Adds one or more ngrams to a list.
+        """
+        # union of items ----------------------------
+        new_list = self.base_list + self.change_list
+        # -------------------------------------------
+
+        # save
+        new_list.save(self.base_list.id)
+
+        return JsonHttpResponse({
+            'parameters': self.params,
+            'count_added': len(new_list.items) - len(self.base_list.items),
+        }, 201)
+
+    def delete(self, request):
+        """
+        Removes one or more ngrams from a list.
+        """
+        # removal (set difference) ------------------
+        new_list = self.base_list - self.change_list
+        # -------------------------------------------
+
+        # save
+        new_list.save(self.base_list.id)
+
+        return JsonHttpResponse({
+            'parameters': self.params,
+            'count_removed': len(self.base_list.items) - len(new_list.items),
+        }, 200)
+
+    @staticmethod
+    def _validate(params):
+        """
+        Checks "list" and "ngrams" parameters for their:
+          - presence
+          - type
+
+        These two parameters are mandatory for any ListChange methods.
+
+        ngrams are also converted to an UnweightedList object for easy add/remove
+        """
+        if 'list' not in params:
+            raise ValidationException('The route /api/ngramlists/change requires a "list" \
+                                       parameter, for instance /api/ngramlists/change?list_id=42')
+        if 'ngrams' not in params:
+            raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
+                                       parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
+
+        # 2 x retrieval => 2 x UnweightedLists
+        # ------------------------------------
+        base_list_id = None
+        try:
+            base_list_id = int(params['list'])
+            # UnweightedList retrieved by id
+            base_list = UnweightedList(base_list_id)
+        except:
+            raise ValidationException('The "list" parameter requires an existing list id.')
+
+        change_ngram_ids = []
+        try:
+            change_ngram_ids = [int(n) for n in params['ngrams'].split(',')]
+            # UnweightedList created from items
+            change_list = UnweightedList(change_ngram_ids)
+        except:
+            raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
+
+        return(base_list, change_list)
+
+
 class ListFamily(APIView):
     """
     Compact combination of *multiple* list info
@@ -78,14 +192,14 @@ class ListFamily(APIView):
 
     USAGE EXEMPLES
     HOST/api/ngramlists/family?corpus=2
-    HOST/api/ngramlists/family?corpus=2&glance=10
+    HOST/api/ngramlists/family?corpus=2&head=10
     HOST/api/ngramlists/family?mainlist=91&scoring=94
-    HOST/api/ngramlists/family?mainlist=91&scoring=94&glance=10
+    HOST/api/ngramlists/family?mainlist=91&scoring=94&head=10
     HOST/api/ngramlists/family?mainlist=91&stoplist=90&scoring=94
      etc.
 
     REST Parameters:
-    "glance=20"
+    "head=20"
         use pagination to only load the k top ngrams of the mainlist
         (useful for fast loading of terms view)
     "corpus=ID"
@@ -100,10 +214,10 @@ class ListFamily(APIView):
 
         parameters = get_parameters(request)
         glance_limit = None
-        mainlist = None
+        mainlist_id = None
         scores_id = None
         groups_id = None
-        secondary_lists = {'maplist':None, 'stoplist':None}
+        other_list_ids = {'maplist':None, 'stoplist':None}
 
         # 1) retrieve a mainlist_id and other lists
         ##########################################
@@ -119,22 +233,22 @@ class ListFamily(APIView):
             else:
                 scores_id = corpus.children('OCCURRENCES').first().id
             # retrieve the family of lists that have corpus as parent
-            mainlist = corpus.children('MAINLIST').first().id,
+            mainlist_id = corpus.children('MAINLIST').first().id
             groups_id = corpus.children('GROUPLIST').first().id
-            secondary_lists['stoplist'] = corpus.children('STOPLIST').first().id
-            secondary_lists['maplist']  = corpus.children('MAPLIST').first().id,
+            other_list_ids['stoplist'] = corpus.children('STOPLIST').first().id
+            other_list_ids['maplist']  = corpus.children('MAPLIST').first().id
 
         # custom request: refers to each list individually
         # -------------------------------------------------
         elif "mainlist" in parameters and "scoring" in parameters:
-            mainlist = parameters['mainlist']
+            mainlist_id = parameters['mainlist']
             scores_id = parameters['scoring']
             groups_id = None
             if 'groups' in parameters:
                 groups_id = parameters['scoring']
             for k in ['stoplist', 'maplist']:
                 if k in parameters:
-                    secondary_lists[k] = parameters[k]
+                    other_list_ids[k] = parameters[k]
 
         # or request has an error
         # -----------------------
@@ -149,21 +263,22 @@ class ListFamily(APIView):
         ngraminfo = {}           # ngram details sorted per ngram id
         linkinfo  = {}           # ngram groups sorted per ngram id
         listmembers = {}         # ngram ids sorted per list name
-        if "glance" in parameters:
-            # glance <=> only mainlist AND only k top ngrams
-            glance_limit = int(parameters['glance'])
-            mainlist_query = _query_list(mainlist, details=True,
+        if "head" in parameters:
+            # head <=> only mainlist AND only k top ngrams
+            glance_limit = int(parameters['head'])
+            mainlist_query = _query_list(mainlist_id, details=True,
                                           pagination_limit = glance_limit,
                                           scoring_metric_id= scores_id)
         else:
             # infos for all ngrams
-            mainlist_query = _query_list(mainlist, details=True,
+            mainlist_query = _query_list(mainlist_id, details=True,
                                           scoring_metric_id= scores_id)
             # and for the other lists (stop and map)
-            for li in secondary_lists:
-                li_elts = _query_list(secondary_lists[li], details=False
+            for li in other_list_ids:
+                li_elts = _query_list(other_list_ids[li], details=False
                                       ).all()
-                listmembers[li] = {ng[0]:True for ng in li_elts}
+                # simple array of ngram_ids
+                listmembers[li] = [ng[0] for ng in li_elts]
 
             # and the groupings
             if groups_id:
@@ -171,7 +286,6 @@ class ListFamily(APIView):
                 linkinfo = links.groups
 
         # the output form
-        ngraminfo = {}
         for ng in mainlist_query.all():
             ng_id   = ng[0]
             # id => [term, weight]
@@ -180,5 +294,12 @@ class ListFamily(APIView):
         return JsonHttpResponse({
             'ngraminfos' : ngraminfo,
             'listmembers' : listmembers,
-            'links' : linkinfo
+            'links' : linkinfo,
+            'nodeids' : {
+                'mainlist':  mainlist_id,
+                'maplist' :  other_list_ids['maplist'],
+                'stoplist':  other_list_ids['stoplist'],
+                'groups':  groups_id,
+                'scores':  scores_id,
+            }
         })

gargantext/views/api/urls.py

@@ -10,16 +10,18 @@ urlpatterns = [
 
     url(r'^nodes/(\d+)/facets$', nodes.CorpusFacet.as_view()),
 
-    # get a list of ngram_ids or ngram_infos by list_id
-    #
-    # url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
+    # add or remove ngram from a list
+    #  ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
+    #       rm <=> DEL ngramlists/change?list=42&ngrams=1,2
+    url(r'^ngramlists/change$', ngramlists.ListChange.as_view()),
 
-    # entire combination of lists from a corpus
+    # get entire combination of lists from a corpus
     # (or any combination of lists that go together :
     #   - a mainlist
     #   - an optional stoplist
     #   - an optional maplist
-    #   - an optional grouplist
-    # aka lexical model
+    #   - an optional grouplist)
     url(r'^ngramlists/family$', ngramlists.ListFamily.as_view()),
+
+
 ]

static/js/gargantext/NGrams_dyna_chart_and_table.js

@@ -318,8 +318,8 @@ function Final_UpdateTable( action ) {
     // debug
     console.log("\nFUN Final_UpdateTable()")
 
-    console.log("AjaxRecords")
-    console.log(AjaxRecords)
+    // console.log("AjaxRecords")
+    // console.log(AjaxRecords)
 
     // (1) Identifying if the button is collapsed:
     var isCollapsed=false;
@@ -346,9 +346,8 @@ function Final_UpdateTable( action ) {
 
     var dataini = (TheBuffer[0])?TheBuffer[0]:oldest;
     var datafin = (TheBuffer[1])?TheBuffer[1]:latest;
-    pr("show me the pubs of the selected period")
+    pr("show me the pubs of the selected score range")
     pr("\tfrom ["+dataini+"] to ["+datafin+"]")
-    pr("\tfrom ["+oldest+"] to ["+latest+"]")
 
     TimeRange = []
     for (var i in AjaxRecords) {
@@ -557,7 +556,7 @@ function transformContent(rec_id) {
     result["ngramId"] = ngram_info["id"] ;
 
     // uncomment if column state (here and in Main_test)
-    result["state"] = AjaxRecords[rec_id].state
+    // result["state"] = AjaxRecords[rec_id].state
 
     // -------------------------------------------
     // check box state columns 'will_be_map' and 'will_be_stop'
@@ -870,16 +869,16 @@ function SaveLocalChanges() {
   }
   // [ = = = = / For deleting subforms = = = = ]
 
-  console.log(" = = = = = = = = = == ")
-  console.log("FlagsBuffer:")
-  console.log(JSON.stringify(FlagsBuffer))
+  // console.log(" = = = = = = = = = == ")
+  // console.log("FlagsBuffer:")
+  // console.log(JSON.stringify(FlagsBuffer))
 
 
-  var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)
-  var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)
+  var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)  // main => stop
+  var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)   // ??? stop => main ???
   var nodes_2group = $.extend({}, FlagsBuffer["group"])
-  var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])
-  var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])
+  var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])     //  add to map
+  var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])   //  remove from map
 
    // console.log("")
    // console.log("")
@@ -896,8 +895,12 @@ function SaveLocalChanges() {
    // console.log("")
    // console.log("")
 
-  var list_id = $("#list_id").val()
-  var corpus_id = getIDFromURL( "corpora" )
+  // retrieve node_ids from hidden input
+  var mainlist_id = $("#mainlist_id").val()
+  var maplist_id  = $("#maplist_id" ).val()
+  var stoplist_id = $("#stoplist_id" ).val()
+
+  // var corpus_id = getIDFromURL( "corpora" )
 
   $("#stoplist_content").html()
 
@@ -906,15 +909,59 @@ function SaveLocalChanges() {
   // });
 
   $("#Save_All").append('<img width="8%" src="/static/img/ajax-loader.gif"></img>')
-  CRUD( corpus_id , "/group" , [] , nodes_2group , "PUT" , function(result) {
-   CRUD( corpus_id , "/keep" , [] , nodes_2inmap , "PUT" , function(result) {
-     CRUD( corpus_id , "/keep" , [] , nodes_2outmap , "DELETE" , function(result) {
-         CRUD( list_id , "" , nodes_2del , [] , "DELETE", function(result) {
-          window.location.reload()
-         });
-     });
-   });
-  });
+
+  // chained CRUD calls
+  CRUD_1_AddMap()
+
+  // add some ngrams to maplist
+  function CRUD_1_AddMap() {
+    CRUD( maplist_id , Object.keys(nodes_2inmap), "PUT" , function(success) {
+      if (success) {
+        CRUD_2_RmMap()             // chained AJAX  1 -> 2
+      }
+      else {
+        console.warn('CRUD error on ngrams add to maplist ('+maplist_id+')')
+      }
+    });
+  }
+  // remove some ngrams from maplist
+  function CRUD_2_RmMap() {
+    CRUD( maplist_id , Object.keys(nodes_2outmap), "DELETE" , function(success) {
+      if (success) {
+        CRUD_3_AddStopRmMain()    // chained AJAX  2 -> 3
+      }
+      else {
+        console.warn('CRUD error on ngrams remove from maplist ('+maplist_id+')')
+      }
+    });
+  }
+
+  // 2 operations going together: add ngrams to stoplist and remove them from mainlist
+  function CRUD_3_AddStopRmMain() {
+    CRUD( stoplist_id , nodes_2del, "PUT" , function(success) {
+      if (success) {
+        // console.log("OK CRUD 3a add stop")
+        CRUD( mainlist_id , nodes_2del, "DELETE" , function(success) {
+          if (success) {
+            // console.log("OK CRUD 3b rm main")
+            CRUD_4()              // chained AJAX  3 -> 4
+          }
+          else {
+            console.warn('CRUD error on ngrams remove from mainlist ('+mainlist_id+')')
+          }
+        });
+      }
+      else {
+        console.warn('CRUD error on ngrams add to stoplist ('+stoplist_id+')')
+      }
+    });
+  }
+
+
+  // TODO add to groups
+  function CRUD_4() {
+      window.location.reload() // refresh whole page if all OK
+  }
 }
 
 
@@ -938,37 +985,40 @@ $("#Save_All").click(function(){
   SaveLocalChanges()
 });
 
-// For lists, all http-requests
-function CRUD( parent_id , action , nodes , args , http_method , callback) {
-	var the_url = window.location.origin+"/api/node/"+parent_id+"/ngrams"+action+"/"+nodes.join("+");
-	the_url = the_url.replace(/\/$/, ""); //remove trailing slash
+// For list modifications (add/delete), all http-requests
+function CRUD( list_id , ngram_ids , http_method , callback) {
+    // ngramlists/change?node_id=42&ngram_ids=1,2
+    var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
 
     // debug
-    // console.log("CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
-
-    if(nodes.length>0 || Object.keys(args).length>0) {
-		$.ajax({
-		  method: http_method,
-		  url: the_url,
-		  data: args,
-		  beforeSend: function(xhr) {
-		    xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-		  },
-		  success: function(data){
-		  		console.log(http_method + " ok!!")
-		        console.log(nodes)
-		        console.log(data)
-		        callback(true);
-		  },
-		  error: function(result) {
-		      console.log("Data not found in #Save_All");
-		      console.log(result)
-		      callback(false);
-		  }
-		});
-
-	} else callback(false);
-}
+    // console.log("starting CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
+
+    if(ngram_ids.length>0) {
+        $.ajax({
+          method: http_method,
+          url: the_url,
+            //    data: args,   // currently all data explicitly in the url (like a GET)
+          beforeSend: function(xhr) {
+            xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+          },
+          success: function(data){
+                console.log("-- CRUD ----------")
+                console.log(http_method + " ok!!")
+                console.log(JSON.stringify(data))
+                console.log("------------------")
+                callback(true);
+          },
+          error: function(result) {
+              console.log("-- CRUD ----------")
+              console.log("Data not found in #Save_All");
+              console.log(result)
+              console.log("------------------")
+              callback(false);
+          }
+        });
+
+    } else callback(true);
+    }
 
 
 
@@ -1003,6 +1053,13 @@ function Main_test( data , initial , search_filter) {
 	console.log(" = = = = / MAIN_TEST: = = = = ")
 	console.log("")
 
+    // Expected infos in "data.ngrams" should have the form:
+    // { "1": { id: "1", name: "réalité",        score: 36  },
+    //   "9": { id: "9", name: "pdg",            score: 116 },
+    //  "10": { id:"10", name: "infrastructure", score:  12 }  etc. }
+
+    // (see filling of rec_info below)
+    // console.log(data.ngrams)
 
     var DistributionDict = {}
     for(var i in DistributionDict)
@@ -1032,7 +1089,7 @@ function Main_test( data , initial , search_filter) {
       div_table += "\t"+"\t"+'<th data-dynatable-column="ngramId" style="background-color:grey">ngramId</th>'+"\n";
 
       // uncomment for column stateId (here and in transformContent)
-      div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
+      // div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
 
       div_table += "\t"+"\t"+'<th data-dynatable-column="name">Terms</th>'+"\n";
       div_table += "\t"+"\t"+'<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>'+"\n";
@@ -1089,15 +1146,14 @@ function Main_test( data , initial , search_filter) {
     $("#stats").html(div_stats)
 
     AjaxRecords = {}
-    console.log("data.ngrams")
-    console.log(data.ngrams)
+
     for(var id in data.ngrams) {
 
       // console.log(i)
       // console.log(data.ngrams[i])
       var le_ngram = data.ngrams[id] ;
 
-      var node_info = {
+      var rec_info = {
         "id" : le_ngram.id,
         "name": le_ngram.name,
         "score": le_ngram.score,
@@ -1112,14 +1168,14 @@ function Main_test( data , initial , search_filter) {
         "will_be_map": null,
         "will_be_stop": null
       }
-      // AjaxRecords.push(node_info)
-      AjaxRecords[id] = node_info
+      // AjaxRecords.push(rec_info)
+      AjaxRecords[id] = rec_info
 
-      if ( ! DistributionDict[node_info.score] ) DistributionDict[node_info.score] = 0;
-      DistributionDict[node_info.score]++;
+      if ( ! DistributionDict[rec_info.score] ) DistributionDict[rec_info.score] = 0;
+      DistributionDict[rec_info.score]++;
     }
 
-    console.log(FirstScore)
+    // console.log(FirstScore)
 
     // console.log("The Distribution!:")
     // console.log(Distribution)
@@ -1418,8 +1474,17 @@ GET_(new_url, function(res) {
     	        "nb_ngrams":Object.keys(main_ngrams_objects).length,
     	    }
         } ;
-        NGrams["map"] = res.listmembers.maplist ;
-        NGrams["stop"] = res.listmembers.stoplist ;  // TODO use simpler struct
+        // map & stop: 2x(array of ids) ==> 2x(lookup hash)
+        NGrams["map"] = {} ;
+        for (var i in res.listmembers.maplist) {
+            var map_ng_id = res.listmembers.maplist[i] ;
+            NGrams["map"][map_ng_id] = true ;
+        }
+        NGrams["stop"] = {} ;
+        for (var i in res.listmembers.stoplist) {
+            var stop_ng_id = res.listmembers.stoplist[i] ;
+            NGrams["stop"][stop_ng_id] = true ;
+        }
         NGrams["group"] = {"links" : res.links , "nodes" : {}};
         for (var parent_ng_id in res.links) {
             NGrams["group"]["nodes"][parent_ng_id] = false ;
@@ -1429,8 +1494,15 @@ GET_(new_url, function(res) {
             }
         }
     }
-    console.log('after init NGrams["main"].ngrams')
-    console.log(NGrams["main"].ngrams)
+    // console.log('after init NGrams["main"].ngrams')
+    // console.log(NGrams["main"].ngrams)
+
+    // cache all DB node_ids
+    $("input#mainlist_id").val(res.nodeids['mainlist'])
+    $("input#maplist_id" ).val(res.nodeids['maplist'])
+    $("input#stoplist_id").val(res.nodeids['stoplist'])
+    $("input#groups_id").val(res.nodeids['groups'])
+    $("input#scores_id").val(res.nodeids['scores'])
     AfterAjax() ;
 });
 
@@ -1537,8 +1609,8 @@ function AfterAjax() {
     // console.log('NGrams["group"]["nodes"]')
     // console.log( NGrams["group"]["nodes"] )
 
-    console.log('after subforms deletion NGrams["main"].ngrams')
-    console.log(NGrams["main"].ngrams)
+    // console.log('after subforms deletion NGrams["main"].ngrams')
+    // console.log(NGrams["main"].ngrams)
 
     // initialize state of maplist items
     if( Object.keys(NGrams["map"]).length>0 ) {

templates/pages/corpora/terms.html

@@ -22,7 +22,7 @@
         <div class="row">
             <div id="monthly-move-chart">
                 <center>
-                        Select a time range in the chart with blue bars to zoom in
+                        Select a score/frequency range in the chart with blue bars to zoom in
                 <p align="center">
                 <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
                 <a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
@@ -41,7 +41,12 @@
           <br>
         </div>
 
-        <input type="hidden" id="list_id" value="{{ list_id }}"></input>
+        <!-- (values set by js) caching our DB ids (handy for list update commands) -->
+        <input type="hidden" id="mainlist_id" value=""></input>
+        <input type="hidden" id="maplist_id" value=""></input>
+        <input type="hidden" id="stoplist_id" value=""></input>
+        <input type="hidden" id="groups_id" value=""></input>
+        <input type="hidden" id="scores_id" value=""></input>
 
         <div class="row">
             <div class="panel panel-default">