groups in table: more detailed DELETE (subform params) and convert old POST...

groups in table: more detailed DELETE (subform params) and convert old POST method to PUT (for symmetry) (all params are in url)

gargantext/views/api/ngramlists.py

@@ -4,13 +4,14 @@ API views for advanced operations on ngrams and ngramlists
     - retrieve several lists together ("family")
     - retrieve detailed list infos (ngram_id, term strings, scores...)
     - modify NodeNgram lists (PUT/DEL an ngram to a MAINLIST OR MAPLIST...)
-    - modify NodeNgramNgram groups (POST a list of groupings like {"767":[209,640],"779":[436,265,385]}")
+    - modify NodeNgramNgram groups (PUT/DEL a list of groupings like {"767[]":[209,640],"779[]":[436,265,385]}")
 """
 
 from gargantext.util.http     import APIView, get_parameters, JsonHttpResponse,\
                                      ValidationException, Http404
 from gargantext.util.db       import session, aliased, desc, bulk_insert
-from gargantext.util.db_cache import cache, or_
+from gargantext.util.db_cache import cache
+from sqlalchemy               import tuple_
 from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram, NodeNgramNgram
 from gargantext.util.lists    import UnweightedList, Translations
 
@@ -132,19 +133,11 @@ class GroupChange(APIView):
                                group node
                                to modify
 
-    We use POST HTTP method to send group data with structure like:
-     {
-        mainform_A: [subform_A1],
-        mainformB:  [subform_B1,subform_B2,subform_B3]
-        ...
-     }
+    We use PUT HTTP method to send group data to DB and DELETE to remove them.
 
-    Chained effect:
-        any previous group under mainformA or B will be overwritten
+    They both use same data format in the url (see links_to_couples).
 
-
-    The DELETE HTTP method also works, with same url
-                 (and simple array in the data)
+    No chained effects : simply adds or deletes rows of couples
 
     NB: request.user is also checked for current authentication status
     """
@@ -160,80 +153,120 @@ class GroupChange(APIView):
             # can't use return in initial() (although 401 maybe better than 404)
             # can't use @requires_auth because of positional 'self' within class
 
-    def post(self, request):
+    def links_to_couples(self,params):
+        """
+        IN (dict from url params)
+        ---
+        params = {
+                       "mainform_A": ["subform_A1"]
+                       "mainform_B": ["subform_B1,subform_B2,subform_B3"]
+                       ...
+                     }
+
+        OUT (for DB rows)
+        ----
+        couples = [
+                    (mainform_A , subform_A1),
+                    (mainform_B , subform_B1),
+                    (mainform_B , subform_B2),
+                    (mainform_B , subform_B3),
+                    ...
+                  ]
         """
-        Rewrites the group node **selectively**
+        couples = []
+        for (mainform_id, subforms_ids) in params.items():
+            for subform_id in subforms_ids[0].split(','):
+                # append the couple
+                couples.append((int(mainform_id),int(subform_id)))
+        return couples
 
-          => removes couples where newly reconnected ngrams where involved
-          => adds new couples from GroupsBuffer of terms view
+    def put(self, request):
+        """
+        Add some group elements to a group node
+          => adds new couples from GroupsBuffer._to_add of terms view
 
         TODO see use of util.lists.Translations
 
-        POST data:
-            <QueryDict: {'1228[]': ['891', '1639']}> => creates 1228 - 891
-                                                            and 1228 - 1639
-        request.POST.lists() iterator where each elt is like :('1228[]',['891','1639'])
-        """
-        group_node = get_parameters(request)['node']
-        all_mainforms = []
-        links = []
+        Parameters are all in the url (for symmetry with DELETE method)
+           api/ngramlists/groups?node=783&1228[]=891,1639
+                                     => creates 1228 - 891
+                                            and 1228 - 1639
 
-        for (mainform_key, subforms_ids) in request.POST.lists():
-            mainform_id = mainform_key[:-2]   # remove brackets '543[]' -> '543'
-            all_mainforms.append(mainform_id)
-            for subform_id in subforms_ids:
-                links.append((mainform_id,subform_id))
+        general format is:   mainform_id[]=subform_id1,subform_id2 etc
+                                     => creates mainform_id - subform_id1
+                                            and mainform_id - subform_id2
 
-        # remove selectively all groupings with these mainforms
-        # using IN is correct in this case: list of ids is short and external
-        # see stackoverflow.com/questions/444475/
-        old_links = (session.query(NodeNgramNgram)
-                    .filter(NodeNgramNgram.node_id == group_node)
-                    .filter(NodeNgramNgram.ngram1_id.in_(all_mainforms))
+        NB: also checks if the couples exist before because the ngram table
+            will send the entire group (old existing links + new links)
+        """
+        # from the url
+        params = get_parameters(request)
+        # the node param is unique
+        group_node = params.pop('node')
+        # the others params are links to change
+        couples = self.links_to_couples(params)
+
+        # local version of "insert if not exists" -------------------->8--------
+        # (1) check already existing elements
+        check_query = (session.query(NodeNgramNgram)
+                        .filter(NodeNgramNgram.node_id == group_node)
+                        .filter(
+                            tuple_(NodeNgramNgram.ngram1_id, NodeNgramNgram.ngram2_id)
+                            .in_(couples)
+                    )
                 )
-        n_removed = old_links.delete(synchronize_session=False)
-        session.commit()
 
-        # add new groupings
+        existing = {}
+        for synonyms in check_query.all():
+            existing[(synonyms.ngram1_id,synonyms.ngram2_id)] = True
+
+        # (2) compute difference locally
+        couples_to_add = [(mform,sform) for (mform,sform)
+                                        in couples
+                                        if (mform,sform) not in existing]
+
+        # (3) add new groupings
         bulk_insert(
             NodeNgramNgram,
             ('node_id', 'ngram1_id', 'ngram2_id', 'weight'),
-            ((group_node, mainform, subform, 1.0) for (mainform,subform) in links)
+            ((group_node, mainform, subform, 1.0) for (mainform,subform)
+                                                  in couples_to_add)
         )
 
+        # ------------------------------------------------------------>8--------
+
         return JsonHttpResponse({
-            'count_removed': n_removed,
-            'count_added': len(links),
+            'count_added': len(couples_to_add),
             }, 200)
 
 
+
     def delete(self, request):
         """
-        Deletes some groups from the group node
-
-        Send in data format is simply a json { 'keys':'["11492","16438"]' }
-
-        ==> it means removing any synonym groups having these 2 as mainform
-            (within the url's groupnode_id)
-
-        NB: At reception here it becomes like:
-                <QueryDict: {'keys[]': ['11492', '16438']}>
+        Within a groupnode, deletes some group elements from some groups
 
+        Data format just like in POST, everything in the url
         """
 
         # from the url
-        group_node = get_parameters(request)['node']
-
-        print(request.POST)
+        params = get_parameters(request)
+        # the node param is unique
+        group_node = params.pop('node')
+        # the others params are links to change
+        couples_to_remove = self.links_to_couples(params)
 
-        # from the data in body
-        all_mainforms = request.POST.getlist('keys[]')
-
-        links_to_remove = (session.query(NodeNgramNgram)
+        # remove selectively group_couples
+        # using IN is correct in this case: list of ids is short and external
+        # see stackoverflow.com/questions/444475/
+        db_rows = (session.query(NodeNgramNgram)
                     .filter(NodeNgramNgram.node_id == group_node)
-                    .filter(NodeNgramNgram.ngram1_id.in_(all_mainforms))
+                    .filter(
+                      tuple_(NodeNgramNgram.ngram1_id, NodeNgramNgram.ngram2_id)
+                      .in_(couples_to_remove)
+                    )
                 )
-        n_removed = links_to_remove.delete(synchronize_session=False)
+
+        n_removed = db_rows.delete(synchronize_session=False)
         session.commit()
 
         return JsonHttpResponse({
@@ -428,8 +461,6 @@ class MapListGlance(APIView):
 
 
 
-
-
 class ListFamily(APIView):
     """
     Compact combination of *multiple* list info

gargantext/views/api/urls.py

@@ -35,8 +35,8 @@ urlpatterns = [ url(r'^nodes$'                , nodes.NodeListResource.as_view()
 
               ,  url(r'^ngramlists/groups$', ngramlists.GroupChange.as_view()        )
                 # modify grouping couples of a group node
-                #  ex: POST ngramlists/groups?node=43
-                # post data looks like : {"767":[209,640],"779":[436,265,385]}"
+                #  ex: PUT/DEL ngramlists/groups?node=43
+                #      & group data also in url: 767[]=209,640 & 779[]=436,265,385
 
               , url(r'^ngramlists/family$'     , ngramlists.ListFamily.as_view()     )
                 # entire combination of lists from a corpus, dedicated to termtable

static/lib/gargantext/NGrams_dyna_chart_and_table.js

@@ -54,9 +54,9 @@
  *           - new api routes + prefetch maplist terms
  *           - simplify UpdateTable
  *           - clarify cruds
- *           - better "created groups" handling
+ *           - fine-grained "created groups" handling
  *
- * @version 1.2
+ * @version 1.3
  *
  * @requires jquery.dynatable
  * @requires d3
@@ -144,9 +144,10 @@ var GroupsBuffer = {'_to_add':{}, '_to_del':{}}
 // to_add will have structure like this  { mainformid1 : [array1 of subformids],
 //                                         mainformid2 : [array2 of subformids]}
 
-// to_del is simple array of mainforms : [mainform3, mainform4]
-// because deleting "the group of mainform3" is deleting all DB rows of
-// couples having mainform3 on the left-hand-side ( <=> no need to know the subform)
+// to_del has same format
+// (will be interpreted as couples to remove from DB rows)
+//  for instance: _to_del  = {      A1 : [A2, A5],             B1 : [B2] }
+//  means to remove 3 DB rows   A1 -- A2   and   A1 -- A5   and B1 -- B2
 
 
 
@@ -423,7 +424,13 @@ function saveActiveGroup() {
     for (downgradedNgramId in activeGroup.were_mainforms) {
         // (except the one we just saved)
         if (downgradedNgramId != mainform) {
-            GroupsBuffer._to_del[downgradedNgramId] = true
+            // it deletes the current groups with these ex-mainforms
+            // TODO check
+
+            // for DB
+            GroupsBuffer._to_del[downgradedNgramId] = CurrentGroups["links"][downgradedNgramId]
+
+            // locally
             deleteInCurrentGroups(downgradedNgramId, false)     // "false" <=> no need to change subform states
         }
     }
@@ -674,9 +681,12 @@ function removeSubform(ngramId) {
         var oldMainFormId = CurrentGroups["subs"][ngramId]
 
         // it must break the old group, mark for deletion
-        GroupsBuffer._to_del[oldMainFormId] = true
+        // TODO make temporary
+
+        // for DB
+        GroupsBuffer._to_del[oldMainFormId] = CurrentGroups["links"][ngramId]
 
-        // consequences:
+        // local consequences:
         deleteInCurrentGroups(oldMainFormId, true)
         //     => they are all removed from CurrentGroups
         //     => the removed subform and all others from the old group
@@ -1506,7 +1516,7 @@ function SaveLocalChanges() {
   // (also removes previous groups with same mainforms!)
   function CRUD_7_AddGroups() {
       console.log("===> AJAX CRUD7 AddGroups <===\n") ;
-      GROUPCRUDS(groupnode_id, GroupsBuffer._to_add, "POST" , function(success) {
+      GROUPCRUDS(groupnode_id, GroupsBuffer._to_add, "PUT" , function(success) {
           if (success) {
               CRUD_8_RmGroups()    // chained AJAX  7 -> 8
           }
@@ -1567,35 +1577,46 @@ function CRUD( list_id , ngram_ids , http_method , callback) {
 }
 
 
-// For group modifications (POST: {mainformA: [subformsA1,A2,A3], mainformB:..})
-//                                (adds new groups for mainformA and mainformB)
-//                                (also deletes any old group under A or B)
+// For group modifications
+//  @param groupnode_id: the node with the groupings to change
+//  @param send_data: {mainformA: [subformsA1,A2,A3], mainformB:..})
+//
+//  @param http_method:
+//         PUT: adds new group rows : groupnode_id -- mainformA -- subformsA1
+//                                    groupnode_id -- mainformA -- subformsA2
+//                                    groupnode_id -- mainformA -- subformsA3
+//                                    groupnode_id -- mainformB -- ....
+//
+//         DEL: idem but removes the group rows
+//
+// ex: /api/ngramlists/groups?node=783&3409[]=4745,14691,3730
 //
-//                         (DEL: {"keys": [mainformX, mainformY]})
-//                               (just deletes the old groups of X and Y)
-function GROUPCRUDS( groupnode_id , send_object, http_method , callback) {
+// NB no chained effects
+
+function GROUPCRUDS( groupnode_id , send_data, http_method , callback) {
 
     // ngramlists/groups?node=9
     var the_url = window.location.origin+"/api/ngramlists/groups?node="+groupnode_id;
 
-    // array of the keys
-    var mainformIds = Object.keys(send_object)
-
-    var send_data ;
-
-    // if DELETE we need only them keys
-    if (http_method == "DELETE") {
-        send_data = {"keys": mainformIds} ;
-    }
-    else {
-        send_data = send_object ;
-    }
+    if(Object.keys(send_data).length > 0) {
+        // group details go also in the url as additional params
+        for (var mainformId in send_data) {
+            subformIds = send_data[mainformId]
+            if (typeof subformIds != "undefined" && subformIds.constructor == Array) {
+                console.log("(ok doing: "+http_method+") for mainform "+mainformId+" with subforms", subformIds)
+                the_url = the_url + '&' + mainformId + '[]=' + subformIds.join()
+            }
+            else {
+                console.error("(skipping: "+http_method+") for mainform "+mainformId+" with subforms", subformIds)
+            }
+        }
 
-    if(mainformIds.length > 0) {
         $.ajax({
           method: http_method,
           url: the_url,
-          data: send_data,
+          // data: send_data // all data explicitly in the url (like a GET)
+                             // because DEL can't consistently support form data
+
           beforeSend: function(xhr) {
             xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
           },
@@ -2155,6 +2176,11 @@ function HandleAjax(res, sourceUrl) {
     AfterAjax(sourceUrl) ;
 }
 
+
+
+// throws errors when ngram in list has no infos
+// (can't assign state and copy to AjaxRecords)
+
 function AfterAjax(sourceUrl) {
   // -------------------------------------------------------------------
   // console.log(JSON.stringify(OriginalNG))
@@ -2171,7 +2197,7 @@ function AfterAjax(sourceUrl) {
       for(var ngram_id in OriginalNG["map"]) {
           myNgramInfo = OriginalNG["records"][ngram_id]
           if (typeof myNgramInfo == "undefined") {
-              console.error("record of ngram " + ngram_id + " is undefined")
+              console.error("record of ngram " + ngram_id + " was undefined")
           }
           else {
               // initialize state of maplist items