Merge remote-tracking branch 'origin/romain-list' into stable-merge

gargantext/util/ngramlists_tools.py

@@ -11,7 +11,7 @@ from gargantext.util.group_tools import query_groups, group_union
 from gargantext.util.db          import session, desc, func, \
                                         bulk_insert_ifnotexists
 from gargantext.models           import Ngram, NodeNgram, NodeNodeNgram, \
-                                        NodeNgramNgram
+                                        NodeNgramNgram, Node
 
 from gargantext.util.lists       import UnweightedList, Translations
 
@@ -327,7 +327,7 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
 
 
 
-def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
+def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
                              group_delimiter=DEFAULT_CSV_DELIM_GROUP):
     '''
     This function reads a CSV of an ngrams table for a Corpus,
@@ -385,7 +385,7 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
     -------
         3 x UnweightedList + 1 x Translations
 
-    @param fname            a local filename or a filehandle-like
+    @param the_file         a local filename or file contents or a filehandle-like
     @param delimiter        a character used as separator in the CSV
     @param group_delimiter  a character used as grouped subforms separator
                             (in the last column)
@@ -418,21 +418,27 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
 
     # =============== READ CSV ===============
 
-    if isinstance(fname, str):
-        fh = open(fname, "r")
-    elif callable(getattr(fname, "read", None)):
-        fh = fname
+    if isinstance(the_file, list):
+        fname = 'imported_file'
+        contents = the_file
     else:
-        raise TypeError("IMPORT: fname argument has unknown type %s" % type(fh))
+        if isinstance(the_file, str):
+            fh = open(the_file, "r")
+            fname = the_file
+        elif callable(getattr(the_file, "read", None)):
+            fh = the_file
+            fname = the_file
+        else:
+            raise TypeError("IMPORT: the_file argument has unknown type %s" % type(the_file))
 
 
-    # reading all directly b/c csv.reader takes only lines or a real fh in bytes
-    # and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
-    # (but we checked its size before!)
-    contents = fh.read().decode("UTF-8").split("\n")
+        # reading all directly b/c csv.reader takes only lines or a real fh in bytes
+        # and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
+        # (but we checked its size before!)
+        contents = fh.read().decode("UTF-8").split("\n")
 
-    # end of CSV read
-    fh.close()
+        # end of CSV read
+        fh.close()
 
     # <class 'django.core.files.uploadedfile.InMemoryUploadedFile'>
 
@@ -609,7 +615,6 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
     return result
 
 
-
 def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
     """
     Integrates an external terms table to the current one:
@@ -832,3 +837,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
 
     # return a log
     return("\n".join(my_log))
+
+def import_and_merge_ngramlists(file_contents, onto_corpus_id):
+    """
+    A single function to run import_ngramlists and merge_ngramlists together
+    """
+    new_lists = import_ngramlists(file_contents)
+
+    corpus_node = session.query(Node).filter(Node.id == onto_corpus_id).first()
+
+    # merge the new_lists onto those of the target corpus
+    log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node)
+
+    return log_msg

gargantext/views/api/ngramlists.py

@@ -12,12 +12,13 @@ from gargantext.util.http     import APIView, get_parameters, JsonHttpResponse,\
 from gargantext.util.db       import session, aliased, bulk_insert
 from gargantext.util.db_cache import cache
 from sqlalchemy               import tuple_
-from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram, NodeNgramNgram
+from gargantext.models        import Ngram, NodeNgram, NodeNodeNgram, NodeNgramNgram, Node
 from gargantext.util.lists    import UnweightedList, Translations
 
 # useful subroutines
 from gargantext.util.ngramlists_tools import query_list, export_ngramlists, \
-                                             import_ngramlists, merge_ngramlists
+                                             import_ngramlists, merge_ngramlists, \
+                                             import_and_merge_ngramlists
 from gargantext.util.group_tools      import query_grouped_ngrams
 
 
@@ -82,23 +83,22 @@ class CSVLists(APIView):
         #                                                 ----------------------
         csv_file = request.data['csvfile']
 
-        # import the csv
-        try:
-            new_lists = import_ngramlists(csv_file)
-            print("======new_lists=========================!!!")
-            # print(new_lists) # very long
-            del csv_file
+        csv_contents = csv_file.read().decode("UTF-8").split("\n")
+        csv_file.close()
+        del csv_file
 
-            # merge the new_lists onto those of the target corpus
-            log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node)
-            return JsonHttpResponse({
-                'log': log_msg,
-                }, 200)
+        # import the csv
+        # try:
+        log_msg = import_and_merge_ngramlists(csv_contents,
+                                              onto_corpus_id = corpus_node.id)
+        return JsonHttpResponse({
+            'log': log_msg,
+            }, 200)
 
-        except Exception as e:
-            return JsonHttpResponse({
-                'err': str(e),
-                }, 400)
+        # except Exception as e:
+        #     return JsonHttpResponse({
+        #         'err': str(e),
+        #         }, 400)
 
     def patch(self,request):
         """
@@ -361,6 +361,7 @@ class ListChange(APIView):
         1) Checks current user authentication to prevent remote DB manipulation
         2) Prepares self.list_objects from params
         """
+
         if not request.user.is_authenticated():
             raise Http404()
             # can't use return in initial() (although 401 maybe better than 404)
@@ -368,8 +369,20 @@ class ListChange(APIView):
 
         # get validated params
         self.params = get_parameters(request)
+
         (self.base_list, self.change_list) = ListChange._validate(self.params)
 
+        if not len(self.change_list.items):
+            payload_ngrams = request.data['ngrams']
+            # print("no change_list in params but we got:", payload_ngrams)
+            # change_list can be in payload too
+            change_ngram_ids = [int(n) for n in payload_ngrams.split(',')]
+            if (not len(change_ngram_ids)):
+                raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
+            else:
+                self.change_list = UnweightedList(change_ngram_ids)
+
+
     def put(self, request):
         """
         Adds one or more ngrams to a list.
@@ -406,6 +419,7 @@ class ListChange(APIView):
             'count_removed': len(self.base_list.items) - len(new_list.items),
         }, 200)
 
+
     @staticmethod
     def _validate(params):
         """
@@ -420,9 +434,9 @@ class ListChange(APIView):
         if 'list' not in params:
             raise ValidationException('The route /api/ngramlists/change requires a "list" \
                                        parameter, for instance /api/ngramlists/change?list_id=42')
-        if 'ngrams' not in params:
-            raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
-                                       parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
+        # if 'ngrams' not in params:
+        #     raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
+        #                                parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
 
         # 2 x retrieval => 2 x UnweightedLists
         # ------------------------------------
@@ -430,17 +444,18 @@ class ListChange(APIView):
         try:
             base_list_id = int(params['list'])
             # UnweightedList retrieved by id
-            base_list = UnweightedList(base_list_id)
         except:
             raise ValidationException('The "list" parameter requires an existing list id.')
+        base_list = UnweightedList(base_list_id)
 
         change_ngram_ids = []
         try:
             change_ngram_ids = [int(n) for n in params['ngrams'].split(',')]
             # UnweightedList created from items
-            change_list = UnweightedList(change_ngram_ids)
         except:
-            raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
+            # ngrams no longer mandatory inline, see payload check afterwards
+            pass
+        change_list = UnweightedList(change_ngram_ids)
 
         return(base_list, change_list)
 

static/lib/gargantext/NGrams_dyna_chart_and_table.js

@@ -1818,14 +1818,25 @@ function SaveLocalChanges() {
 // For list modifications (add/delete), all http-requests
 function CRUD( list_id , ngram_ids , http_method , callback) {
     // ngramlists/change?node_id=42&ngram_ids=1,2
-    var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
+    // var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
+    var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id;
 
     // debug
     // console.log("  ajax target: " + the_url + " (" + http_method + ")")
 
+    // 2016-10-05 pass PUT and DELETE ngrams in payload as if it was POSTs
+    // (to avoid too long urls that trigger Bad Gateway in production)
+
+    var myNgramsData = new FormData();
+    myNgramsData.append("ngrams", ngram_ids.join(","))
+
     if(ngram_ids.length>0) {
         $.ajax({
           method: http_method,
+          async: true,
+          contentType: false,
+          processData: false,
+          data: myNgramsData,
           url: the_url,
             //    data: args,   // currently all data explicitly in the url (like a GET)
           beforeSend: function(xhr) {