[FIX] merge url conflicts.

doc/schemas/ngram_parsing_flow.dot

+// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
+digraph ngramflow {
+
+    edge [fontsize=10] ;
+    label=<<B><U>gargantext.util.toolchain</U></B><BR/>(ngram extraction flow)>;
+    labelloc="t" ;
+
+    "extracted_ngrams" -> "grouplist" ;
+    "extracted_ngrams" -> "occs+tfidfs" ;
+    "main_user_stoplist" -> "stoplist" ;
+    "stoplist" -> "mainlist" ;
+    "occs+tfidfs" -> "mainlist" [label="  TFIDF_LIMIT"];
+    "mainlist" -> "coocs" [label="  COOCS_THRESHOLD"] ;
+    "coocs" -> "specificity" ;
+
+    "specificity" -> "maplist" [label="MAPLIST_LIMIT\nMONOGRAM_PART"];
+
+    "maplist" -> "explore" ;
+    "grouplist" -> "maplist" ;
+}

gargantext/constants.py

@@ -156,10 +156,10 @@ RESOURCETYPES = [
         'parser': RISParser,
         'default_language': 'en',
     },
-    # {   'name': 'CSV',
-    #     # 'parser': CSVParser,
-    #     'default_language': 'en',
-    # },
+    {   'name': 'CSV',
+        'parser': CSVParser,
+        'default_language': 'en',
+    },
     # {   'name': 'ISTex',
     #     # 'parser': ISTexParser,
     #     'default_language': 'en',

gargantext/util/db.py

@@ -35,8 +35,7 @@ Double = DOUBLE_PRECISION
 # useful for queries
 
 from sqlalchemy.orm import aliased
-from sqlalchemy import func
-
+from sqlalchemy import func, desc
 
 # bulk insertions
 

gargantext/util/parsers/CSVParser.py → gargantext/util/parsers/CSV.py

@@ -8,34 +8,32 @@ import os
 
 class CSVParser(Parser):
 
-    def CSVsample( self, filename , delim) :
-        ifile  = open( filename, "r" )
-        reader = csv.reader(ifile, delimiter=delim)
+    def CSVsample( self, small_contents , delim) :
+        reader = csv.reader(small_contents, delimiter=delim)
 
         Freqs = []
         for row in reader:
             Freqs.append(len(row))
 
-
-        ifile.close()
         return Freqs
 
-    
+
     def parse(self, filename):
 
+        print("CSV: parsing (assuming UTF-8 and LF line endings)")
+
+        contents = filename.read().decode("UTF-8").split("\n")
+
         sample_size = 10
-        sample_file = filename.replace(".csv","_sample.csv")
+        sample_contents = contents[0:sample_size]
 
         hyperdata_list = []
 
-        command_for_sample = "cat '"+filename+"' | head -n "+str(sample_size)+" > '"+sample_file+"'"
-        os.system(command_for_sample) # you just created a  *_sample.csv
-
         # # = = = = [ Getting delimiters frequency ] = = = = #
         PossibleDelimiters = [ ',',' ','\t', ';', '|', ':' ]
         AllDelimiters = {}
         for delim in PossibleDelimiters:
-            AllDelimiters[delim] = self.CSVsample( sample_file , delim ) 
+            AllDelimiters[delim] = self.CSVsample( sample_contents , delim )
         # # = = = = [ / Getting delimiters frequency ] = = = = #
         # # OUTPUT example:
         # #  AllDelimiters = {
@@ -59,8 +57,8 @@ class CSVParser(Parser):
         # # = = = = [ / Stand.Dev=0 & Sum of delimiters ] = = = = #
         # # OUTPUT example:
         # #  Delimiters = [
-        # #     ['\t', 5, 5, 0.0], 
-        # #     [',', 75, 5, 0.0], 
+        # #     ['\t', 5, 5, 0.0],
+        # #     [',', 75, 5, 0.0],
         # #     ['|', 5, 5, 0.0]
         # #  ]
 
@@ -68,23 +66,22 @@ class CSVParser(Parser):
         # # = = = = [ Delimiter selection ] = = = = #
         Sorted_Delims = sorted(Delimiters, key=lambda x: x[1], reverse=True)
         HighestDelim = Sorted_Delims[0][0]
-        # print("selected delimiter:",[HighestDelim]
-        # print
+        # HighestDelim = ","
+        print("CSV selected delimiter:",[HighestDelim])
         # # = = = = [ / Delimiter selection ] = = = = #
 
 
-
-
         # # = = = = [ First data coordinate ] = = = = #
         Coords = {
             "row": -1,
             "column": -1
         }
 
-        ifile  = open( sample_file, "r" )
-        reader = csv.reader(ifile, delimiter=HighestDelim)
+        reader = csv.reader(contents, delimiter=HighestDelim)
 
         for rownum, tokens in enumerate(reader):
+            if rownum % 250 == 0:
+                print("CSV row: ", rownum)
             joined_tokens = "".join (tokens)
             if Coords["row"]<0 and len( joined_tokens )>0 :
                 Coords["row"] = rownum
@@ -93,22 +90,21 @@ class CSVParser(Parser):
                     if len(t)>0:
                         Coords["column"] = columnum
                         break
-        ifile.close()
         # # = = = = [ / First data coordinate ] = = = = #
 
 
 
         # # = = = = [ Setting Headers ] = = = = #
         Headers_Int2Str = {}
-        ifile  = open( sample_file, "r" )
-        reader = csv.reader(ifile, delimiter=HighestDelim)
+        reader = csv.reader(contents, delimiter=HighestDelim)
         for rownum, tokens in enumerate(reader):
             if rownum>=Coords["row"]:
                 for columnum in range( Coords["column"],len(tokens) ):
                     t = tokens[columnum]
                     Headers_Int2Str[columnum] = t
                 break
-        ifile.close()
+        # print("Headers_Int2Str")
+        # print(Headers_Int2Str)
         # # = = = = [ / Setting Headers ] = = = = #
         # # OUTPUT example:
         # #  Headers_Int2Str = {
@@ -119,11 +115,9 @@ class CSVParser(Parser):
         # #  }
 
 
-
         # # = = = = [ Reading the whole CSV and saving ] = = = = #
         hyperdata_list = []
-        ifile  = open( filename, "r" )
-        reader = csv.reader(ifile, delimiter=HighestDelim)
+        reader = csv.reader(contents, delimiter=HighestDelim)
         for rownum, tokens in enumerate(reader):
             if rownum>Coords["row"]:
                 RecordDict = {}
@@ -131,7 +125,6 @@ class CSVParser(Parser):
                     data = tokens[columnum]
                     RecordDict[ Headers_Int2Str[columnum] ] = data
                 hyperdata_list.append( RecordDict )
-        ifile.close()
         # # = = = = [ / Reading the whole CSV and saving ] = = = = #
 
         return hyperdata_list

gargantext/util/parsers/__init__.py

@@ -8,4 +8,4 @@ from .Pubmed import PubmedParser
 from .Europress import EuropressParser
 
 # from .ISTex import ISTexParser
-# from .CSV import CSVParser
+from .CSV import CSVParser

gargantext/views/api/ngramlists.py

+"""
+API views for advanced operations on ngrams and ngramlists
+-----------------------------------------------------------
+    - retrieve several lists together ("family")
+    - retrieve detailed list infos (ngram_id, term strings, scores...)
+    - modify NodeNgram lists (PUT/DEL an ngram to a MAINLIST OR MAPLIST...)
+"""
+
 from gargantext.util.http     import APIView, get_parameters, JsonHttpResponse,\
-                                     ValidationException
-from gargantext.util.db       import session, aliased
+                                     ValidationException, Http404
+from gargantext.util.db       import session, aliased, desc
 from gargantext.util.db_cache import cache
 from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram
-from gargantext.util.lists    import Translations
-from sqlalchemy               import desc
-# from gargantext.constants import *
-# from gargantext.util.validation import validate
-# from collections import defaultdict
+from gargantext.util.lists    import UnweightedList, Translations
 
 
 def _query_list(list_id,
@@ -64,6 +68,116 @@ class List(APIView):
     """
     pass
 
+
+class ListChange(APIView):
+    """
+    Any ngram action on standard NodeNgram lists (MAIN, MAP, STOP)
+
+    USAGE EXEMPLE:
+    HOST/api/ngramlists/change?list=42&ngrams=1,2,3,4,5
+                                vvvvvv          ||||||
+                               old list         vvvvvv
+                               to modify     new list items
+                                  |                |
+                                  v                v
+     2 x UnweightedLists:   self.base_list   self.change_list
+
+    We use DEL/PUT HTTP methods to differentiate the 2 basic rm/add actions
+    They rely only on inline parameters (no need for payload data)
+
+    No chained effects: eg removing from MAPLIST will not remove
+                          automatically from associated MAINLIST
+
+    NB: request.user is also checked for current authentication status
+    """
+
+    def initial(self, request):
+        """
+        Before dispatching to put(), delete()...
+
+        1) Checks current user authentication to prevent remote DB manipulation
+        2) Prepares self.list_objects from params
+        """
+        if not request.user.is_authenticated():
+            raise Http404()
+            # can't use return in initial() (although 401 maybe better than 404)
+            # can't use @requires_auth because of positional 'self' within class
+
+        # get validated params
+        self.params = get_parameters(request)
+        (self.base_list, self.change_list) = ListChange._validate(self.params)
+
+    def put(self, request):
+        """
+        Adds one or more ngrams to a list.
+        """
+        # union of items ----------------------------
+        new_list = self.base_list + self.change_list
+        # -------------------------------------------
+
+        # save
+        new_list.save(self.base_list.id)
+
+        return JsonHttpResponse({
+            'parameters': self.params,
+            'count_added': len(new_list.items) - len(self.base_list.items),
+        }, 201)
+
+    def delete(self, request):
+        """
+        Removes one or more ngrams from a list.
+        """
+        # removal (set difference) ------------------
+        new_list = self.base_list - self.change_list
+        # -------------------------------------------
+
+        # save
+        new_list.save(self.base_list.id)
+
+        return JsonHttpResponse({
+            'parameters': self.params,
+            'count_removed': len(self.base_list.items) - len(new_list.items),
+        }, 200)
+
+    @staticmethod
+    def _validate(params):
+        """
+        Checks "list" and "ngrams" parameters for their:
+          - presence
+          - type
+
+        These two parameters are mandatory for any ListChange methods.
+
+        ngrams are also converted to an UnweightedList object for easy add/remove
+        """
+        if 'list' not in params:
+            raise ValidationException('The route /api/ngramlists/change requires a "list" \
+                                       parameter, for instance /api/ngramlists/change?list_id=42')
+        if 'ngrams' not in params:
+            raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
+                                       parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
+
+        # 2 x retrieval => 2 x UnweightedLists
+        # ------------------------------------
+        base_list_id = None
+        try:
+            base_list_id = int(params['list'])
+            # UnweightedList retrieved by id
+            base_list = UnweightedList(base_list_id)
+        except:
+            raise ValidationException('The "list" parameter requires an existing list id.')
+
+        change_ngram_ids = []
+        try:
+            change_ngram_ids = [int(n) for n in params['ngrams'].split(',')]
+            # UnweightedList created from items
+            change_list = UnweightedList(change_ngram_ids)
+        except:
+            raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
+
+        return(base_list, change_list)
+
+
 class ListFamily(APIView):
     """
     Compact combination of *multiple* list info
@@ -78,14 +192,14 @@ class ListFamily(APIView):
 
     USAGE EXEMPLES
     HOST/api/ngramlists/family?corpus=2
-    HOST/api/ngramlists/family?corpus=2&glance=10
+    HOST/api/ngramlists/family?corpus=2&head=10
     HOST/api/ngramlists/family?mainlist=91&scoring=94
-    HOST/api/ngramlists/family?mainlist=91&scoring=94&glance=10
+    HOST/api/ngramlists/family?mainlist=91&scoring=94&head=10
     HOST/api/ngramlists/family?mainlist=91&stoplist=90&scoring=94
      etc.
 
     REST Parameters:
-    "glance=20"
+    "head=20"
         use pagination to only load the k top ngrams of the mainlist
         (useful for fast loading of terms view)
     "corpus=ID"
@@ -100,10 +214,10 @@ class ListFamily(APIView):
 
         parameters = get_parameters(request)
         glance_limit = None
-        mainlist = None
+        mainlist_id = None
         scores_id = None
         groups_id = None
-        secondary_lists = {'maplist':None, 'stoplist':None}
+        other_list_ids = {'maplist':None, 'stoplist':None}
 
         # 1) retrieve a mainlist_id and other lists
         ##########################################
@@ -119,22 +233,22 @@ class ListFamily(APIView):
             else:
                 scores_id = corpus.children('OCCURRENCES').first().id
             # retrieve the family of lists that have corpus as parent
-            mainlist = corpus.children('MAINLIST').first().id,
+            mainlist_id = corpus.children('MAINLIST').first().id
             groups_id = corpus.children('GROUPLIST').first().id
-            secondary_lists['stoplist'] = corpus.children('STOPLIST').first().id
-            secondary_lists['maplist']  = corpus.children('MAPLIST').first().id,
+            other_list_ids['stoplist'] = corpus.children('STOPLIST').first().id
+            other_list_ids['maplist']  = corpus.children('MAPLIST').first().id
 
         # custom request: refers to each list individually
         # -------------------------------------------------
         elif "mainlist" in parameters and "scoring" in parameters:
-            mainlist = parameters['mainlist']
+            mainlist_id = parameters['mainlist']
             scores_id = parameters['scoring']
             groups_id = None
             if 'groups' in parameters:
                 groups_id = parameters['scoring']
             for k in ['stoplist', 'maplist']:
                 if k in parameters:
-                    secondary_lists[k] = parameters[k]
+                    other_list_ids[k] = parameters[k]
 
         # or request has an error
         # -----------------------
@@ -149,21 +263,22 @@ class ListFamily(APIView):
         ngraminfo = {}           # ngram details sorted per ngram id
         linkinfo  = {}           # ngram groups sorted per ngram id
         listmembers = {}         # ngram ids sorted per list name
-        if "glance" in parameters:
-            # glance <=> only mainlist AND only k top ngrams
-            glance_limit = int(parameters['glance'])
-            mainlist_query = _query_list(mainlist, details=True,
+        if "head" in parameters:
+            # head <=> only mainlist AND only k top ngrams
+            glance_limit = int(parameters['head'])
+            mainlist_query = _query_list(mainlist_id, details=True,
                                           pagination_limit = glance_limit,
                                           scoring_metric_id= scores_id)
         else:
             # infos for all ngrams
-            mainlist_query = _query_list(mainlist, details=True,
+            mainlist_query = _query_list(mainlist_id, details=True,
                                           scoring_metric_id= scores_id)
             # and for the other lists (stop and map)
-            for li in secondary_lists:
-                li_elts = _query_list(secondary_lists[li], details=False
+            for li in other_list_ids:
+                li_elts = _query_list(other_list_ids[li], details=False
                                       ).all()
-                listmembers[li] = {ng[0]:True for ng in li_elts}
+                # simple array of ngram_ids
+                listmembers[li] = [ng[0] for ng in li_elts]
 
             # and the groupings
             if groups_id:
@@ -171,7 +286,6 @@ class ListFamily(APIView):
                 linkinfo = links.groups
 
         # the output form
-        ngraminfo = {}
         for ng in mainlist_query.all():
             ng_id   = ng[0]
             # id => [term, weight]
@@ -180,5 +294,12 @@ class ListFamily(APIView):
         return JsonHttpResponse({
             'ngraminfos' : ngraminfo,
             'listmembers' : listmembers,
-            'links' : linkinfo
+            'links' : linkinfo,
+            'nodeids' : {
+                'mainlist':  mainlist_id,
+                'maplist' :  other_list_ids['maplist'],
+                'stoplist':  other_list_ids['stoplist'],
+                'groups':  groups_id,
+                'scores':  scores_id,
+            }
         })

gargantext/views/api/urls.py

@@ -11,17 +11,18 @@ urlpatterns = [
     url(r'^nodes/(\d+)/facets$' , nodes.CorpusFacet.as_view()),
     url(r'^nodes/(\d+)/having$' , nodes.NodeListHaving.as_view()),
 
-    # get a list of ngram_ids or ngram_infos by list_id
-    #
-    # url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
+    # add or remove ngram from a list
+    #  ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
+    #       rm <=> DEL ngramlists/change?list=42&ngrams=1,2
+    url(r'^ngramlists/change$', ngramlists.ListChange.as_view()),
 
-    # entire combination of lists from a corpus
+    # get entire combination of lists from a corpus
     # (or any combination of lists that go together :
     #   - a mainlist
     #   - an optional stoplist
     #   - an optional maplist
-    #   - an optional grouplist
-    # aka lexical model
-    url(r'^ngramlists/family$'  , ngramlists.ListFamily.as_view()),
+    #   - an optional grouplist)
+    url(r'^ngramlists/family$', ngramlists.ListFamily.as_view()),
+
 
 ]

static/js/gargantext/NGrams_dyna_chart_and_table.js

@@ -318,8 +318,8 @@ function Final_UpdateTable( action ) {
     // debug
     console.log("\nFUN Final_UpdateTable()")
 
-    console.log("AjaxRecords")
-    console.log(AjaxRecords)
+    // console.log("AjaxRecords")
+    // console.log(AjaxRecords)
 
     // (1) Identifying if the button is collapsed:
     var isCollapsed=false;
@@ -346,9 +346,8 @@ function Final_UpdateTable( action ) {
 
     var dataini = (TheBuffer[0])?TheBuffer[0]:oldest;
     var datafin = (TheBuffer[1])?TheBuffer[1]:latest;
-    pr("show me the pubs of the selected period")
+    pr("show me the pubs of the selected score range")
     pr("\tfrom ["+dataini+"] to ["+datafin+"]")
-    pr("\tfrom ["+oldest+"] to ["+latest+"]")
 
     TimeRange = []
     for (var i in AjaxRecords) {
@@ -557,7 +556,7 @@ function transformContent(rec_id) {
     result["ngramId"] = ngram_info["id"] ;
 
     // uncomment if column state (here and in Main_test)
-    result["state"] = AjaxRecords[rec_id].state
+    // result["state"] = AjaxRecords[rec_id].state
 
     // -------------------------------------------
     // check box state columns 'will_be_map' and 'will_be_stop'
@@ -870,16 +869,16 @@ function SaveLocalChanges() {
   }
   // [ = = = = / For deleting subforms = = = = ]
 
-  console.log(" = = = = = = = = = == ")
-  console.log("FlagsBuffer:")
-  console.log(JSON.stringify(FlagsBuffer))
+  // console.log(" = = = = = = = = = == ")
+  // console.log("FlagsBuffer:")
+  // console.log(JSON.stringify(FlagsBuffer))
 
 
-  var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)
-  var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)
+  var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)  // main => stop
+  var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)   // ??? stop => main ???
   var nodes_2group = $.extend({}, FlagsBuffer["group"])
-  var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])
-  var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])
+  var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])     //  add to map
+  var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])   //  remove from map
 
    // console.log("")
    // console.log("")
@@ -896,8 +895,12 @@ function SaveLocalChanges() {
    // console.log("")
    // console.log("")
 
-  var list_id = $("#list_id").val()
-  var corpus_id = getIDFromURL( "corpora" )
+  // retrieve node_ids from hidden input
+  var mainlist_id = $("#mainlist_id").val()
+  var maplist_id  = $("#maplist_id" ).val()
+  var stoplist_id = $("#stoplist_id" ).val()
+
+  // var corpus_id = getIDFromURL( "corpora" )
 
   $("#stoplist_content").html()
 
@@ -906,15 +909,59 @@ function SaveLocalChanges() {
   // });
 
   $("#Save_All").append('<img width="8%" src="/static/img/ajax-loader.gif"></img>')
-  CRUD( corpus_id , "/group" , [] , nodes_2group , "PUT" , function(result) {
-   CRUD( corpus_id , "/keep" , [] , nodes_2inmap , "PUT" , function(result) {
-     CRUD( corpus_id , "/keep" , [] , nodes_2outmap , "DELETE" , function(result) {
-         CRUD( list_id , "" , nodes_2del , [] , "DELETE", function(result) {
-          window.location.reload()
-         });
-     });
-   });
-  });
+
+  // chained CRUD calls
+  CRUD_1_AddMap()
+
+  // add some ngrams to maplist
+  function CRUD_1_AddMap() {
+    CRUD( maplist_id , Object.keys(nodes_2inmap), "PUT" , function(success) {
+      if (success) {
+        CRUD_2_RmMap()             // chained AJAX  1 -> 2
+      }
+      else {
+        console.warn('CRUD error on ngrams add to maplist ('+maplist_id+')')
+      }
+    });
+  }
+  // remove some ngrams from maplist
+  function CRUD_2_RmMap() {
+    CRUD( maplist_id , Object.keys(nodes_2outmap), "DELETE" , function(success) {
+      if (success) {
+        CRUD_3_AddStopRmMain()    // chained AJAX  2 -> 3
+      }
+      else {
+        console.warn('CRUD error on ngrams remove from maplist ('+maplist_id+')')
+      }
+    });
+  }
+
+  // 2 operations going together: add ngrams to stoplist and remove them from mainlist
+  function CRUD_3_AddStopRmMain() {
+    CRUD( stoplist_id , nodes_2del, "PUT" , function(success) {
+      if (success) {
+        // console.log("OK CRUD 3a add stop")
+        CRUD( mainlist_id , nodes_2del, "DELETE" , function(success) {
+          if (success) {
+            // console.log("OK CRUD 3b rm main")
+            CRUD_4()              // chained AJAX  3 -> 4
+          }
+          else {
+            console.warn('CRUD error on ngrams remove from mainlist ('+mainlist_id+')')
+          }
+        });
+      }
+      else {
+        console.warn('CRUD error on ngrams add to stoplist ('+stoplist_id+')')
+      }
+    });
+  }
+
+
+  // TODO add to groups
+  function CRUD_4() {
+      window.location.reload() // refresh whole page if all OK
+  }
 }
 
 
@@ -938,37 +985,40 @@ $("#Save_All").click(function(){
   SaveLocalChanges()
 });
 
-// For lists, all http-requests
-function CRUD( parent_id , action , nodes , args , http_method , callback) {
-	var the_url = window.location.origin+"/api/node/"+parent_id+"/ngrams"+action+"/"+nodes.join("+");
-	the_url = the_url.replace(/\/$/, ""); //remove trailing slash
+// For list modifications (add/delete), all http-requests
+function CRUD( list_id , ngram_ids , http_method , callback) {
+    // ngramlists/change?node_id=42&ngram_ids=1,2
+    var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
 
     // debug
-    // console.log("CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
-
-    if(nodes.length>0 || Object.keys(args).length>0) {
-		$.ajax({
-		  method: http_method,
-		  url: the_url,
-		  data: args,
-		  beforeSend: function(xhr) {
-		    xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-		  },
-		  success: function(data){
-		  		console.log(http_method + " ok!!")
-		        console.log(nodes)
-		        console.log(data)
-		        callback(true);
-		  },
-		  error: function(result) {
-		      console.log("Data not found in #Save_All");
-		      console.log(result)
-		      callback(false);
-		  }
-		});
-
-	} else callback(false);
-}
+    // console.log("starting CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
+
+    if(ngram_ids.length>0) {
+        $.ajax({
+          method: http_method,
+          url: the_url,
+            //    data: args,   // currently all data explicitly in the url (like a GET)
+          beforeSend: function(xhr) {
+            xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+          },
+          success: function(data){
+                console.log("-- CRUD ----------")
+                console.log(http_method + " ok!!")
+                console.log(JSON.stringify(data))
+                console.log("------------------")
+                callback(true);
+          },
+          error: function(result) {
+              console.log("-- CRUD ----------")
+              console.log("Data not found in #Save_All");
+              console.log(result)
+              console.log("------------------")
+              callback(false);
+          }
+        });
+
+    } else callback(true);
+    }
 
 
 
@@ -1003,6 +1053,13 @@ function Main_test( data , initial , search_filter) {
 	console.log(" = = = = / MAIN_TEST: = = = = ")
 	console.log("")
 
+    // Expected infos in "data.ngrams" should have the form:
+    // { "1": { id: "1", name: "réalité",        score: 36  },
+    //   "9": { id: "9", name: "pdg",            score: 116 },
+    //  "10": { id:"10", name: "infrastructure", score:  12 }  etc. }
+
+    // (see filling of rec_info below)
+    // console.log(data.ngrams)
 
     var DistributionDict = {}
     for(var i in DistributionDict)
@@ -1032,7 +1089,7 @@ function Main_test( data , initial , search_filter) {
       div_table += "\t"+"\t"+'<th data-dynatable-column="ngramId" style="background-color:grey">ngramId</th>'+"\n";
 
       // uncomment for column stateId (here and in transformContent)
-      div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
+      // div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
 
       div_table += "\t"+"\t"+'<th data-dynatable-column="name">Terms</th>'+"\n";
       div_table += "\t"+"\t"+'<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>'+"\n";
@@ -1089,15 +1146,14 @@ function Main_test( data , initial , search_filter) {
     $("#stats").html(div_stats)
 
     AjaxRecords = {}
-    console.log("data.ngrams")
-    console.log(data.ngrams)
+
     for(var id in data.ngrams) {
 
       // console.log(i)
       // console.log(data.ngrams[i])
       var le_ngram = data.ngrams[id] ;
 
-      var node_info = {
+      var rec_info = {
         "id" : le_ngram.id,
         "name": le_ngram.name,
         "score": le_ngram.score,
@@ -1112,14 +1168,14 @@ function Main_test( data , initial , search_filter) {
         "will_be_map": null,
         "will_be_stop": null
       }
-      // AjaxRecords.push(node_info)
-      AjaxRecords[id] = node_info
+      // AjaxRecords.push(rec_info)
+      AjaxRecords[id] = rec_info
 
-      if ( ! DistributionDict[node_info.score] ) DistributionDict[node_info.score] = 0;
-      DistributionDict[node_info.score]++;
+      if ( ! DistributionDict[rec_info.score] ) DistributionDict[rec_info.score] = 0;
+      DistributionDict[rec_info.score]++;
     }
 
-    console.log(FirstScore)
+    // console.log(FirstScore)
 
     // console.log("The Distribution!:")
     // console.log(Distribution)
@@ -1418,8 +1474,17 @@ GET_(new_url, function(res) {
     	        "nb_ngrams":Object.keys(main_ngrams_objects).length,
     	    }
         } ;
-        NGrams["map"] = res.listmembers.maplist ;
-        NGrams["stop"] = res.listmembers.stoplist ;  // TODO use simpler struct
+        // map & stop: 2x(array of ids) ==> 2x(lookup hash)
+        NGrams["map"] = {} ;
+        for (var i in res.listmembers.maplist) {
+            var map_ng_id = res.listmembers.maplist[i] ;
+            NGrams["map"][map_ng_id] = true ;
+        }
+        NGrams["stop"] = {} ;
+        for (var i in res.listmembers.stoplist) {
+            var stop_ng_id = res.listmembers.stoplist[i] ;
+            NGrams["stop"][stop_ng_id] = true ;
+        }
         NGrams["group"] = {"links" : res.links , "nodes" : {}};
         for (var parent_ng_id in res.links) {
             NGrams["group"]["nodes"][parent_ng_id] = false ;
@@ -1429,8 +1494,15 @@ GET_(new_url, function(res) {
             }
         }
     }
-    console.log('after init NGrams["main"].ngrams')
-    console.log(NGrams["main"].ngrams)
+    // console.log('after init NGrams["main"].ngrams')
+    // console.log(NGrams["main"].ngrams)
+
+    // cache all DB node_ids
+    $("input#mainlist_id").val(res.nodeids['mainlist'])
+    $("input#maplist_id" ).val(res.nodeids['maplist'])
+    $("input#stoplist_id").val(res.nodeids['stoplist'])
+    $("input#groups_id").val(res.nodeids['groups'])
+    $("input#scores_id").val(res.nodeids['scores'])
     AfterAjax() ;
 });
 
@@ -1537,8 +1609,8 @@ function AfterAjax() {
     // console.log('NGrams["group"]["nodes"]')
     // console.log( NGrams["group"]["nodes"] )
 
-    console.log('after subforms deletion NGrams["main"].ngrams')
-    console.log(NGrams["main"].ngrams)
+    // console.log('after subforms deletion NGrams["main"].ngrams')
+    // console.log(NGrams["main"].ngrams)
 
     // initialize state of maplist items
     if( Object.keys(NGrams["map"]).length>0 ) {

templates/pages/corpora/terms.html

@@ -22,7 +22,7 @@
         <div class="row">
             <div id="monthly-move-chart">
                 <center>
-                        Select a time range in the chart with blue bars to zoom in
+                        Select a score/frequency range in the chart with blue bars to zoom in
                 <p align="center">
                 <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
                 <a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
@@ -41,7 +41,12 @@
           <br>
         </div>
 
-        <input type="hidden" id="list_id" value="{{ list_id }}"></input>
+        <!-- (values set by js) caching our DB ids (handy for list update commands) -->
+        <input type="hidden" id="mainlist_id" value=""></input>
+        <input type="hidden" id="maplist_id" value=""></input>
+        <input type="hidden" id="stoplist_id" value=""></input>
+        <input type="hidden" id="groups_id" value=""></input>
+        <input type="hidden" id="scores_id" value=""></input>
 
         <div class="row">
             <div class="panel panel-default">