Merge branch 'romain-refactoring' into merge

gargantext/constants.py

@@ -194,9 +194,9 @@ RESOURCETYPES = [
 ]
 
 # linguistic extraction parameters ---------------------------------------------
-DEFAULT_TFIDF_CUTOFF_RATIO      = .75        # MAINLIST maximum terms in %
+DEFAULT_RANK_CUTOFF_RATIO      = .75         # MAINLIST maximum terms in %
 
-DEFAULT_TFIDF_HARD_LIMIT        = 5000       # MAINLIST maximum terms abs
+DEFAULT_RANK_HARD_LIMIT        = 5000        # MAINLIST maximum terms abs
                                              # (makes COOCS larger ~ O(N²) /!\)
 
 DEFAULT_COOC_THRESHOLD          = 2          # inclusive minimum for COOCS coefs

gargantext/util/toolchain/__init__.py

@@ -6,7 +6,7 @@ from .hyperdata_indexing  import index_hyperdata
 
 # in usual run order
 from .list_stop           import do_stoplist
-from .metric_tfidf        import compute_occs, compute_tfidf_local, compute_cumulated_tfidf
+from .metric_tfidf        import compute_occs, compute_tfidf_local, compute_ti_ranking
 from .list_main           import do_mainlist
 from .ngram_coocs         import compute_coocs
 from .metric_specificity  import compute_specificity
@@ -116,13 +116,15 @@ def parse_extract_indexhyperdata(corpus):
     ltfidf_id = compute_tfidf_local(corpus)
     print('CORPUS #%d: [%s] new localtfidf node #%i' % (corpus.id, t(), ltfidf_id))
 
-    # -> write global and cumulated tfidf to Node and NodeNodeNgram
-    gtfidf_id = compute_cumulated_tfidf(corpus, scope="global")
-    print('CORPUS #%d: [%s] new globaltfidf node #%i' % (corpus.id, t(), gtfidf_id))
+    # -> write global and cumulated ti_ranking (tfidf ranking vector) to Node and NodeNodeNgram
+    tirank_id = compute_ti_ranking(corpus,
+                                   count_scope="global",
+                                   termset_scope="local")
+    print('CORPUS #%d: [%s] new tfidf ranking node #%i' % (corpus.id, t(), tirank_id))
 
     # -> mainlist: filter + write (to Node and NodeNgram)
     mainlist_id = do_mainlist(corpus,
-                              tfidf_id = gtfidf_id,
+                              ranking_scores_id = tirank_id,
                               stoplist_id = stop_id)
     print('CORPUS #%d: [%s] new mainlist node #%i' % (corpus.id, t(), mainlist_id))
 
@@ -143,7 +145,7 @@ def parse_extract_indexhyperdata(corpus):
     print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id))
 
     print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t()))
-    
+
     corpus.status('Lists', progress=0, complete=True)
     corpus.save_hyperdata()
     session.commit()

gargantext/util/toolchain/list_main.py

@@ -2,14 +2,14 @@ from gargantext.models     import Node, NodeNgram, NodeNodeNgram
 from gargantext.util.db    import session
 from gargantext.util.lists import UnweightedList
 from sqlalchemy            import desc
-from gargantext.constants  import DEFAULT_TFIDF_CUTOFF_RATIO, \
-                                  DEFAULT_TFIDF_HARD_LIMIT
+from gargantext.constants  import DEFAULT_RANK_CUTOFF_RATIO, \
+                                  DEFAULT_RANK_HARD_LIMIT
 
 def do_mainlist(corpus,
                     overwrite_id  = None,
-                    tfidf_id=None, stoplist_id=None,
-                    hard_limit=DEFAULT_TFIDF_HARD_LIMIT,
-                    ratio_limit=DEFAULT_TFIDF_CUTOFF_RATIO
+                    ranking_scores_id=None, stoplist_id=None,
+                    hard_limit=DEFAULT_RANK_HARD_LIMIT,
+                    ratio_limit=DEFAULT_RANK_CUTOFF_RATIO
                     ):
     """
     Select top n terms according to a global tfidf ranking and stoplist filter.
@@ -18,7 +18,7 @@ def do_mainlist(corpus,
         min(hard_limit, number_of_terms * ratio_limit)
 
     NB : We use a global tfidf node where the values are global but the ngrams
-         are already selected (== only within this corpus documents).
+         are already selected (termset_scope == only within this corpus docs).
          TO DISCUSS: allow influence of the local tfidf scores too
 
     Parameters:
@@ -37,12 +37,12 @@ def do_mainlist(corpus,
     """
 
     # retrieve helper nodes if not provided
-    if not tfidf_id:
-        tfidf_id  = session.query(Node.id).filter(
+    if not ranking_scores_id:
+        ranking_scores_id  = session.query(Node.id).filter(
                                 Node.typename  == "TFIDF-GLOBAL",
                                 Node.parent_id == corpus.id
                     ).first()
-        if not tfidf_id:
+        if not ranking_scores_id:
             raise ValueError("MAINLIST: TFIDF node needed for mainlist creation")
 
     if not stoplist_id:
@@ -64,7 +64,7 @@ def do_mainlist(corpus,
     # tfidf-ranked query
     ordered_filtered_tfidf = (session
         .query(NodeNodeNgram.ngram_id)
-        .filter(NodeNodeNgram.node1_id == tfidf_id)
+        .filter(NodeNodeNgram.node1_id == ranking_scores_id)
         .filter(~ NodeNodeNgram.ngram_id.in_(stopterms_subquery))
         .order_by(desc(NodeNodeNgram.score))
         )

gargantext/util/toolchain/metric_specificity.py

@@ -44,11 +44,11 @@ def compute_specificity(corpus, cooc_id=None, overwrite_id = None):
     # v = d.sum(axis=1) (- lui-même)
     xs = x.sum(axis=1) - x
     ys = x.sum(axis=0) - x
-    
+
 
     # top inclus ou exclus
     #n = ( xs + ys) / (2 * (x.shape[0] - 1))
-    
+
     # top generic or specific (asc is spec, desc is generic)
     v = ( xs - ys) / ( 2 * (x.shape[0] - 1))
 
@@ -105,11 +105,14 @@ def compute_specificity(corpus, cooc_id=None, overwrite_id = None):
     # print(v)
     pd.options.display.float_format = '${:,.2f}'.format
 
-    data = WeightedList(
-            zip(  v.index.tolist()
-                , v.values.tolist()[0]
-             )
-           )
-    data.save(the_id)
+    if not v.empty:
+        data = WeightedList(
+                zip(  v.index.tolist()
+                    , v.values.tolist()[0]
+                 )
+               )
+        data.save(the_id)
+    else:
+        print("WARNING: had no terms in COOCS => empty SPECIFICITY node")
 
     return(the_id)

gargantext/util/toolchain/metric_tfidf.py

@@ -88,7 +88,7 @@ def compute_occs(corpus, overwrite_id = None):
     return the_id
 
 
-def compute_cumulated_tfidf(corpus, scope="local", overwrite_id=None):
+def compute_ti_ranking(corpus, count_scope="local", termset_scope="local", overwrite_id=None):
     """
     # TODO check if cumulated tfs correspond to app's use cases and intention
 
@@ -96,55 +96,93 @@ def compute_cumulated_tfidf(corpus, scope="local", overwrite_id=None):
 
     Parameters:
       - the corpus itself
-      - scope: {"local" or "global"}
+      - count_scope: {"local" or "global"}
+         - local  <=> frequencies counted in the current corpus
+         - global <=> frequencies counted in all corpora of this type
+
+
+        when the count_scope is global, there is another parameter:
+          - termset_scope: {"local" or "global"}
+             - local <=> output list of terms limited to the current corpus
+               (SELECT ngram_id FROM nodes_ngrams WHERE node_id IN <docs>)
+             - global <=> output list of terms from all corpora of this type
+                                                    !!!! (more terms)
+
       - overwrite_id: optional id of a pre-existing TFIDF-XXXX node for this corpus
                    (the Node and its previous NodeNodeNgram rows will be replaced)
     """
 
+    corpus_docids_subquery = (session
+                    .query(Node.id)
+                    .filter(Node.parent_id == corpus.id)
+                    .filter(Node.typename == "DOCUMENT")
+                    .subquery()
+                   )
+
     # local <=> within this corpus
-    if scope == "local":
+    if count_scope == "local":
         # All docs of this corpus
-        docids_subquery = (session
-                            .query(Node.id)
-                            .filter(Node.parent_id == corpus.id)
-                            .filter(Node.typename == "DOCUMENT")
-                            .subquery()
-                           )
+        count_scope_subquery = corpus_docids_subquery
+
+        termset_scope_subquery = (session
+                        .query(NodeNgram.ngram_id)
+                        .filter(NodeNgram.node_id.in_(corpus_docids_subquery))
+                        .subquery()
+                       )
+
     # global <=> within all corpora of this source
-    elif scope == "global":
+    elif count_scope == "global":
         this_source_type = corpus.resources()[0]['type']
 
         # all corpora with the same source type
         # (we need raw SQL query for postgres JSON operators) (TODO test speed)
         same_source_corpora_query = (session
-                            .query(Node.id)
-                            .from_statement(text(
-                                """
-                                SELECT id FROM nodes
-                                WHERE hyperdata->'resources' @> '[{\"type\"\:%s}]'
-                                """ % this_source_type
-                                ))
-                            )
+                        .query(Node.id)
+                        .from_statement(text(
+                            """
+                            SELECT id FROM nodes
+                            WHERE hyperdata->'resources' @> '[{\"type\"\:%s}]'
+                            """ % this_source_type
+                            ))
+                        )
 
         # All docs **in all corpora of the same source**
-        docids_subquery = (session
-                            .query(Node.id)
-                            .filter(Node.parent_id.in_(same_source_corpora_query))
-                            .filter(Node.typename == "DOCUMENT")
+        ressource_docids_subquery = (session
+                        .query(Node.id)
+                        .filter(Node.parent_id.in_(same_source_corpora_query))
+                        .filter(Node.typename == "DOCUMENT")
+                        .subquery()
+                       )
+
+
+        count_scope_subquery = ressource_docids_subquery
+
+        if termset_scope == "global":
+            termset_scope_subquery = (session
+                            .query(NodeNgram.ngram_id)
+                            .filter(NodeNgram.node_id.in_(ressource_docids_subquery))
+                            .subquery()
+                           )
+        else:
+            termset_scope_subquery = (session
+                            .query(NodeNgram.ngram_id)
+                            .filter(NodeNgram.node_id.in_(corpus_docids_subquery))
                             .subquery()
                            )
 
+
     # N
-    total_docs = session.query(docids_subquery).count()
+    total_docs = session.query(ressource_docids_subquery).count()
 
-    # or perhaps at least do the occurrences right now at the same time
+    # nb: possible to do the occurrences right now at the same time
     tf_nd = (session
                     .query(
                         NodeNgram.ngram_id,
                         func.sum(NodeNgram.weight),    # tf: same as occnode
                         func.count(NodeNgram.node_id)  # nd: n docs with term
                      )
-                    .filter(NodeNgram.node_id.in_(docids_subquery))
+                    .filter(NodeNgram.node_id.in_(count_scope_subquery))
+                    .filter(NodeNgram.ngram_id.in_(termset_scope_subquery))
                     .group_by(NodeNgram.ngram_id)
                     .all()
                    )
@@ -162,10 +200,10 @@ def compute_cumulated_tfidf(corpus, scope="local", overwrite_id=None):
     else:
         # create the new TFIDF-XXXX node
         tfidf_nd = corpus.add_child()
-        if scope == "local":            # TODO discuss use and find new typename
+        if count_scope == "local":            # TODO discuss use and find new typename
             tfidf_nd.typename  = "TFIDF-CORPUS"
             tfidf_nd.name      = "tfidf-cumul-corpus (in:%s)" % corpus.id
-        elif scope == "global":
+        elif count_scope == "global":
             tfidf_nd.typename  = "TFIDF-GLOBAL"
             tfidf_nd.name      = "tfidf-cumul-global (in type:%s)" % this_source_type
         session.add(tfidf_nd)

static/lib/gargantext/NGrams_dyna_chart_and_table.js

@@ -29,8 +29,10 @@
  *           - unify table ids with ngram ids
  *           - new api routes + prefetch maplist terms
  *           - simplify UpdateTable
+ *           - clarify cruds
+ *           - better "created groups" handling
  *
- * @version 1.1
+ * @version 1.2
  *
  * @requires jquery.dynatable
  * @requires d3
@@ -48,7 +50,6 @@
 // with some expanding in AfterAjax
 var AjaxRecords = [] ;
 
-
 // table element (+config +events)
 // -------------------------------
 var MyTable ;
@@ -105,10 +106,6 @@ for(var i in System[GState]["states"] ) {
 // DICT BUFFERS FOR MAP/MAIN//STOP SAVING LOGIC
 // ----------------------------------------------
 var FlagsBuffer = {}
-// 3 main buffers per state (and later additional per target)
-for(var i in System[GState]["states"]) {
-  FlagsBuffer[System[GState]["states"][i]] = {}
-}
 
 // + 1 for groups
 GroupsBuffer = {}
@@ -238,7 +235,7 @@ function printCorpuses() {
       console.log( url )
 
 
-      GET_( url , function(results) {
+      GET_( url , function(results, url) {
           if(Object.keys( results ).length>0) {
          var sub_ngrams_data = {
            "ngrams":[],
@@ -369,7 +366,7 @@ function saveActiveGroup() {
     GroupsBuffer[mainform] = activeGroup.now_links
     // ---------------------------------------------------
 
-    console.log(AjaxRecords[mainform])
+    // console.log(AjaxRecords[mainform])
 
     // also we prefix "*" to the name if not already there
     if (AjaxRecords[mainform].name[0] != '*') {
@@ -461,7 +458,7 @@ function seeGroup ( ngramId , allowChangeFlag) {
     // 2/7 attach flag open to global state register
     vizopenGroup[ngramId] = true ;
 
-    // 3/7   retrieve names of the original (from DB) grouped ngrams
+    // 3/7   retrieve names of the untouched (from DB) grouped ngrams (aka "old")
     var oldlinksNames = [] ;
     if( ngramId in NGrams.group.links ) {
         for (var i in NGrams.group.links[ngramId]) {
@@ -470,7 +467,7 @@ function seeGroup ( ngramId , allowChangeFlag) {
         }
     }
 
-    // 4/7   retrieve names of the newly created grouped ngrams
+    // 4/7   retrieve names of the newly created grouped ngrams (aka "new" + "oldnew")
     var newlinksNames = [] ;
     if( ngramId in GroupsBuffer ) {
         for(var i in GroupsBuffer[ ngramId ] ) {
@@ -1180,87 +1177,151 @@ $("#Save_All").click(function(){
 });
 
 
+// find all the consequences of changes from MAP => MAIN
+// -----------------------------------------------------
+// (see forge.iscpif.fr/projects/garg/wiki/Ngram_Lists)
+
+// NB: (MAP => MAIN) also has the consequence that (MAIN || DEL)
+// in other words:
+//   inmain <==> outdel
+//   indel  <==> outmain
+// (but we'll keep them distinct in FlagsBuffer for coherence with
+//  the distinct API CRUD commands that will be entailed)
+function InferCRUDFlags(id, oldState, desiredState, registry) {
+
+    state_skip = -1                              // -1
+    state_main = System[0]["statesD"]["normal"]  //  0
+    state_map  = System[0]["statesD"]["keep"]    //  1
+    state_stop = System[0]["statesD"]["delete"]  //  2
+
+
+    // thus skips newly grouped items and returns unmodified registry
+    if (desiredState != state_skip) {
+        // (if was previously in MAP)
+        if (oldState === state_map) {
+            if (desiredState === state_main || desiredState === state_stop) {
+                registry["outmap"][ id ] = true
+                // (... and some more actions only if is now desired to be in STOP)
+                if(desiredState === state_stop) {
+                    registry["indel"][id] = true
+                    registry["outmain"][id] = true
+                }
+            }
+        }
+        // (if previously was in STOP)
+        else if (oldState === state_stop) {
+            if (desiredState === state_main || desiredState === state_map) {
+                registry["outdel"][id] = true
+                registry["inmain"][id] = true
+                // (... and one more action only if is now desired to be in MAP)
+                if(desiredState === state_map) {
+                  registry["inmap"][ id ] = true
+                }
+            }
+        }
+        // (if previously was in MAIN)
+        else  {
+            if(desiredState === state_map) {
+                registry["inmap"][ id ] = true
+            }
+            else if(desiredState === state_stop) {
+                registry["indel"][id] = true
+                registry["outmain"][id] = true
+            }
+        }
+    }
+    return registry
+}
+
+
+
 // MAIN SAVE + MAIN CREATE TABLE
 // -----------------------------
 
 // Save changes to all corpusA-lists
 function SaveLocalChanges() {
-  console.log("\nFUN SaveLocalChanges()")
   // console.clear()
-  console.log("In SaveChanges()")
-  var sum__selected_elems = 0;
+  console.log("In SaveLocalChanges()")
+
+  // registry with summary of the requested changes with consequences
+  // ------------------------------------------------------------------
+  // (see InferCRUDFlags)
+
+  FlagsBuffer["outmain"] = {}         // remove from MAINLIST
+  FlagsBuffer["inmain"] = {}          //      add to MAINLIST
 
-  FlagsBuffer["delete"] = {}
-  FlagsBuffer["keep"] = {}
-  FlagsBuffer["outmap"] = {}
-  FlagsBuffer["inmap"] = {}
+  FlagsBuffer["outmap"] = {}          // remove from MAPLIST
+  FlagsBuffer["inmap"] = {}           //      add to MAPLIST
 
+  FlagsBuffer["outdel"] = {}          // remove from STOPLIST
+  FlagsBuffer["indel"] = {}           //      add to STOPLIST
+
+
+  // LOOP on all mainforms + subforms
+  // --------------------------------
+  // we use 2 globals to evaluate change-of-state
+  //   => NGrams for old states (as in DB)
+  //   => AjaxRecords for current (desired) states
   for(var id in AjaxRecords) {
-    if( NGrams["map"][ id ] ) {
-      if(AjaxRecords[id]["state"]==System[0]["statesD"]["normal"] || AjaxRecords[id]["state"]==System[0]["statesD"]["delete"]) {
-        FlagsBuffer["outmap"][ id ] = true
-        if(AjaxRecords[id]["state"]==System[0]["statesD"]["delete"]) {
-          FlagsBuffer["delete"][id] = true
-        }
-      }
-      if(GroupsBuffer[id] && AjaxRecords[id]["state"]==System[0]["statesD"]["keep"])  {
-        FlagsBuffer["inmap"][ id ] = true
-      }
-    } else {
-      if(AjaxRecords[id]["state"]==System[0]["statesD"]["keep"]) {
-        FlagsBuffer["inmap"][ id ] = true
-      }
-      if(AjaxRecords[id]["state"]==System[0]["statesD"]["delete"]) {
-        FlagsBuffer["delete"][id] = true
-      }
-    }
-  }
-  // [ = = = = For deleting subforms = = = = ]
 
-  for(var i in NGrams["group"].links) {
-    // i is ngram_id of a group mainNode
-    if(FlagsBuffer["delete"][i]) {
-      for(var j in NGrams["group"].links[i] ) {
-        FlagsBuffer["delete"][NGrams["group"].links[i][j]] = true
-      }
-      for(var j in FlagsBuffer["delete"][i] ) {
-        FlagsBuffer["delete"][FlagsBuffer["delete"][i][j]] = true
-      }
+    var oldState = 0 ;
+    if      (NGrams["map"][ id ] ) oldState = 1
+    else if (NGrams["stop"][ id ]) oldState = 2
+
+    var mainNewState = AjaxRecords[id]["state"] ;
+
+    // update the crud flags buffer according to old/new states and what they entail
+    if(oldState != mainNewState) {
+        FlagsBuffer = InferCRUDFlags(id, oldState, mainNewState, FlagsBuffer)
     }
-    if(FlagsBuffer["inmap"][i]) {
-      for(var j in GroupsBuffer[i] ) {
-        FlagsBuffer["outmap"][GroupsBuffer[i][j]] = true
-      }
+
+    // [ = = = = propagating to subforms = = = = ]
+
+    // if change in mainform list or change in groups
+    if(oldState != mainNewState || GroupsBuffer[id]) {
+        // linked nodes
+        var linkedNodes ;
+
+        // a) retrieve the untouched (from DB) grouped ngrams (aka "old")
+        if(NGrams.group.links[id]) linkedNodes = NGrams.group.links[id]
+
+        // b) or retrieve the new linked nodes (aka "new" + "oldnew")
+        else if( GroupsBuffer[id] )     linkedNodes = GroupsBuffer[id]
+
+        for (var i in linkedNodes) {
+            var subNgramId = linkedNodes[i] ;
+
+            // todo check (if undefined old state, should add to main too...)
+            var subOldState = undefined ;
+            if      (NGrams["map"][ subNgramId ] ) subOldState = System[0]["statesD"]["keep"]
+            else if (NGrams["stop"][ subNgramId ]) subOldState = System[0]["statesD"]["delete"]
+            else {
+                subOldState = System[0]["statesD"]["normal"] ;
+                // (special legacy case: subforms can have oldStates == undefined,
+                //  then iff target state is != delete, we should add to main too)
+                if (mainNewState == System[0]["statesD"]["normal"]
+                    || mainNewState == System[0]["statesD"]["map"]) {
+                    FlagsBuffer['inmain'][subNgramId] = true
+                }
+            }
+
+            // update the crud flags buffer with mainNewState (goes to same target state as their mainform)
+            FlagsBuffer = InferCRUDFlags(subNgramId, subOldState, mainNewState, FlagsBuffer)
+        }
     }
+    // [ = = = = / propagating to subforms = = = = ]
   }
-  // [ = = = = / For deleting subforms = = = = ]
 
   // console.log(" = = = = = = = = = == ")
-  // console.log("FlagsBuffer:")
+  // ("FlagsBuffer:")
   // console.log(JSON.stringify(FlagsBuffer))
+  // console.warn("GroupsBuffer:")
+  // console.log(JSON.stringify(GroupsBuffer))
 
 
-  var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)  // main => stop
-  var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)   // ??? stop => main ???
-  var nodes_2group = $.extend({}, GroupsBuffer)
-  var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])     //  add to map
-  var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])   //  remove from map
-
-   // console.log("")
-   // console.log("")
-   // console.log(" nodes_2del: ")
-   // console.log(nodes_2del)
-   // console.log(" nodes_2keep: ")
-   // console.log(nodes_2keep)
-   // console.log(" nodes_2group: ")
-   // console.log(nodes_2group)
-   // console.log(" nodes_2inmap: ")
-   // console.log(nodes_2inmap)
-   // console.log(" nodes_2outmap: ")
-   // console.log(nodes_2outmap)
-   // console.log("")
-   // console.log("")
 
+  // transmit the requested changes to server
+  // ----------------------------------------
   // retrieve node_ids from hidden input
   var mainlist_id = $("#mainlist_id").val()
   var maplist_id  = $("#maplist_id" ).val()
@@ -1273,17 +1334,18 @@ function SaveLocalChanges() {
 
   // The AJAX CRUDs in cascade:
 
+  // £TODO reactivate here and AddMap
   $("#Save_All").append('<img width="8%" src="/static/img/ajax-loader.gif"></img>')
 
-  // chained CRUD calls
+  // trigger chained CRUD calls
   CRUD_1_AddMap()
 
   // add some ngrams to maplist
   function CRUD_1_AddMap() {
     console.log("===> AJAX CRUD1 AddMap <===\n") ;
-    CRUD( maplist_id , Object.keys(nodes_2inmap), "PUT" , function(success) {
+    CRUD( maplist_id , Object.keys(FlagsBuffer["inmap"]), "PUT" , function(success) {
       if (success) {
-        CRUD_2_RmMap()             // chained AJAX  1 -> 2
+        CRUD_2_RmMap()      // trigger chained AJAX  1 -> 2
       }
       else {
         console.warn('CRUD error on ngrams add to maplist ('+maplist_id+')')
@@ -1293,44 +1355,69 @@ function SaveLocalChanges() {
   // remove some ngrams from maplist
   function CRUD_2_RmMap() {
     console.log("===> AJAX CRUD2 RmMap <===\n") ;
-    CRUD( maplist_id , Object.keys(nodes_2outmap), "DELETE" , function(success) {
+    CRUD( maplist_id , Object.keys(FlagsBuffer["outmap"]), "DELETE" , function(success) {
       if (success) {
-        CRUD_3_AddStopRmMain()    // chained AJAX  2 -> 3
+        CRUD_3_AddMain()    // chained AJAX  2 -> 3
       }
       else {
         console.warn('CRUD error on ngrams remove from maplist ('+maplist_id+')')
       }
     });
   }
-
-  // 2 operations going together: add ngrams to stoplist and remove them from mainlist
-  function CRUD_3_AddStopRmMain() {
-    console.log("===> AJAX CRUD3a+b AddStopRmMain <===\n") ;
-    CRUD( stoplist_id , nodes_2del, "PUT" , function(success) {
+  // add some ngrams to mainlist
+  function CRUD_3_AddMain() {
+    console.log("===> AJAX CRUD3 AddMain <===\n") ;
+    CRUD( mainlist_id , Object.keys(FlagsBuffer["inmain"]), "PUT" , function(success) {
       if (success) {
-        // console.log("OK CRUD 3a add stop")
-        CRUD( mainlist_id , nodes_2del, "DELETE" , function(success) {
-          if (success) {
-            // console.log("OK CRUD 3b rm main")
-            CRUD_4()              // chained AJAX  3 -> 4
-          }
-          else {
-            console.warn('CRUD error on ngrams remove from mainlist ('+mainlist_id+')')
-          }
-        });
+        CRUD_4_RmMain()    // chained AJAX  3 -> 4
+      }
+      else {
+        console.warn('CRUD error on ngrams add to mainlist ('+mainlist_id+')')
+      }
+    });
+  }
+  // remove some ngrams from mainlist
+  function CRUD_4_RmMain() {
+    console.log("===> AJAX CRUD4 RmMain <===\n") ;
+    CRUD( mainlist_id , Object.keys(FlagsBuffer["outmain"]), "DELETE" , function(success) {
+      if (success) {
+        CRUD_5_AddStop()    // chained AJAX  4 -> 5
+      }
+      else {
+        console.warn('CRUD error on ngrams remove from mainlist ('+mainlist_id+')')
+      }
+    });
+  }
+  // add some ngrams to stoplist
+  function CRUD_5_AddStop() {
+    console.log("===> AJAX CRUD5 AddStop <===\n") ;
+    CRUD( stoplist_id , Object.keys(FlagsBuffer["indel"]), "PUT" , function(success) {
+      if (success) {
+        CRUD_6_RmStop()    // chained AJAX  5 -> 6
       }
       else {
         console.warn('CRUD error on ngrams add to stoplist ('+stoplist_id+')')
       }
     });
   }
-
+  // remove some ngrams from stoplist
+  function CRUD_6_RmStop() {
+    console.log("===> AJAX CRUD6 RmStop <===\n") ;
+    CRUD( stoplist_id , Object.keys(FlagsBuffer["outdel"]), "DELETE" , function(success) {
+      if (success) {
+        CRUD_7_groups()    // chained AJAX  6 -> 7
+      }
+      else {
+        console.warn('CRUD error on ngrams remove from stoplist ('+stoplist_id+')')
+      }
+    });
+  }
   // add to groups reading data from GroupsBuffer
-  function CRUD_4() {
-      console.log("===> AJAX CRUD4 RewriteGroups <===\n") ;
+  function CRUD_7_groups() {
+      console.log("===> AJAX CRUD7 RewriteGroups <===\n") ;
       GROUPCRUD(groupnode_id, GroupsBuffer, function(success) {
           if (success) {
-              window.location.reload() // all 4 CRUDs OK => refresh whole page
+              window.location.reload() // all 7 CRUDs OK => refresh whole page
           }
           else {
               console.warn('CRUD error on ngrams add to group node ('+groupings_id+')')
@@ -1424,7 +1511,7 @@ function GROUPCRUD( groupnode_id , post_data , callback) {
  *
  * @param ngdata: a response from the api/node/CID/ngrams/list/ routes
  * @param initial: initial score type "occs" or "tfidf"
- * @param search_filter: eg 'filter_all' (see SearchFilters.MODE)
+ * @param search_filter: value among {0,1,2,'reset'} (see #picklistmenu options)
  */
 function MainTableAndCharts( ngdata , initial , search_filter) {
 
@@ -1682,14 +1769,11 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
     volumeChart.filterAll();
     dc.redrawAll();
 
-    // test if array is enough to restore proper page range
+    // AjaxRecords per ngramid => dense array to maintain proper page range
+    // see MyTable.data('dynatable').settings.dataset.originalRecords
     var ArrayAjaxRecords = [] ;
-    var i = 0 ;
-    var idmap = {}
     for (ngid in AjaxRecords) {
         ArrayAjaxRecords.push(AjaxRecords[ngid]) ;
-        i ++ ;
-        idmap[ngid] = i
     }
 
     MyTable = []
@@ -1737,7 +1821,8 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
         // nb: possible value are in {0,1,2} (see terms.html > #picklistmenu)
         .functions['my_state_filter'] = function(record,selectedValue) {
             if (selectedValue == 'reset') {
-                return (AjaxRecords[record.id].state >= 0)
+                // return (AjaxRecords[record.id].state >= 0)
+                return true
             }
             else {
                 // return true or false
@@ -1746,7 +1831,8 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
         }
 
     // and set this filter's initial status to 'maplist' (aka state == 1)
-    MyTable.data('dynatable').settings.dataset.queries['my_state_filter'] = 1 ;
+    // MyTable.data('dynatable').settings.dataset.queries['my_state_filter'] = 1 ;
+    MyTable.data('dynatable').settings.dataset.queries['my_state_filter'] = search_filter ;
     MyTable.data('dynatable').process();
 
     // moves pagination over table
@@ -1860,10 +1946,10 @@ function GET_( url , callback ) {
         url: url,
         dataType: "json",
         success : function(data, textStatus, jqXHR) {
-            callback(data);
+            callback(data, url);
         },
         error: function(exception) {
-            callback(false);
+            callback(false, url);
         }
     })
 }
@@ -1883,18 +1969,19 @@ var NGrams = {
 }
 
 
-// NEW AJAX x 2
+// MAIN AJAX
 
 var prefetch_url = window.location.origin+"/api/ngramlists/maplist?corpus="+corpus_id ;
-var new_url = window.location.origin+"/api/ngramlists/family?corpus="+corpus_id ;
+var final_url = window.location.origin+"/api/ngramlists/family?corpus="+corpus_id ;
 
 // faster call: just the maplist, will return first
-GET_(prefetch_url, HandleAjax);
+// 2016-05-13: deactivated because it causes a lag before the table is updated
+// GET_(prefetch_url, HandleAjax);
 
 // longer call (full list of terms) to return when ready and refresh all data
-GET_(new_url, HandleAjax)
+GET_(final_url, HandleAjax)
 
-function HandleAjax(res) {
+function HandleAjax(res, sourceUrl) {
     if (res && res.ngraminfos) {
 
         main_ngrams_objects = {}
@@ -1907,8 +1994,6 @@ function HandleAjax(res) {
             }
         }
 
-        console.log("===> AJAX INIT <===\n" + "source: " + new_url)
-
         // = = = = MIAM = = = = //
         NGrams["main"] = {
             "ngrams": main_ngrams_objects,
@@ -1949,10 +2034,10 @@ function HandleAjax(res) {
     $("input#stoplist_id").val(res.nodeids['stoplist'])
     $("input#groups_id").val(res.nodeids['groups'])
     $("input#scores_id").val(res.nodeids['scores'])
-    AfterAjax() ;
+    AfterAjax(sourceUrl) ;
 }
 
-function AfterAjax() {
+function AfterAjax(sourceUrl) {
   // -------------------------------------------------------------------
   // console.log(JSON.stringify(NGrams))
   // -------------------------------------------------------------------
@@ -2044,8 +2129,12 @@ function AfterAjax() {
     //   }
     // }
 
+    // show only map (option = 1) or all terms (option = "reset")
+    termsfilter = (sourceUrl == final_url) ? "reset" : "1"
+
     // Initializing the Charts and Table ---------------------------------------
-    var result = MainTableAndCharts( NGrams["main"] , FirstScore , "filter_all")
+    var result = MainTableAndCharts( NGrams["main"] , FirstScore , termsfilter) ;
+
     console.log( result ) // OK
     // -------------------------------------------------------------------------