Skip to content

gargantext
gargantext
Commit 48a58dad authored by delanoe's avatar delanoe
Browse files
Options

Merge remote-tracking branch 'origin/romain-list' into testing-merge

parents e09c6a94 c2ce1582
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 474 additions and 356 deletions
gargantext/util/ngramlists_tools.py
View file @ 48a58dad
......@@ -11,7 +11,7 @@ from gargantext.util.group_tools import query_groups, group_union
from gargantext.util.db import session, desc, func, \
bulk_insert_ifnotexists
from gargantext.models import Ngram, NodeNgram, NodeNodeNgram, \
NodeNgramNgram
NodeNgramNgram, Node
from gargantext.util.lists import UnweightedList, Translations
......@@ -327,7 +327,7 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
group_delimiter=DEFAULT_CSV_DELIM_GROUP):
'''
This function reads a CSV of an ngrams table for a Corpus,
......@@ -385,7 +385,7 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
-------
3 x UnweightedList + 1 x Translations
@param fname a local filename or a filehandle-like
@param the_file a local filename or file contents or a filehandle-like
@param delimiter a character used as separator in the CSV
@param group_delimiter a character used as grouped subforms separator
(in the last column)
......@@ -418,21 +418,27 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
# =============== READ CSV ===============
if isinstance(fname, str):
fh = open(fname, "r")
elif callable(getattr(fname, "read", None)):
fh = fname
if isinstance(the_file, list):
fname = 'imported_file'
contents = the_file
else:
raise TypeError("IMPORT: fname argument has unknown type %s" % type(fh))
if isinstance(the_file, str):
fh = open(the_file, "r")
fname = the_file
elif callable(getattr(the_file, "read", None)):
fh = the_file
fname = the_file
else:
raise TypeError("IMPORT: the_file argument has unknown type %s" % type(the_file))
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents = fh.read().decode("UTF-8").split("\n")
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents = fh.read().decode("UTF-8").split("\n")
# end of CSV read
fh.close()
# end of CSV read
fh.close()
# <class 'django.core.files.uploadedfile.InMemoryUploadedFile'>
......@@ -609,7 +615,6 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
return result
def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
"""
Integrates an external terms table to the current one:
......@@ -832,3 +837,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
# return a log
return("\n".join(my_log))
def import_and_merge_ngramlists(file_contents, onto_corpus_id):
"""
A single function to run import_ngramlists and merge_ngramlists together
"""
new_lists = import_ngramlists(file_contents)
corpus_node = session.query(Node).filter(Node.id == onto_corpus_id).first()
# merge the new_lists onto those of the target corpus
log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node)
return log_msg
gargantext/views/api/ngramlists.py
View file @ 48a58dad
......@@ -12,12 +12,13 @@ from gargantext.util.http import APIView, get_parameters, JsonHttpResponse,\
from gargantext.util.db import session, aliased, bulk_insert
from gargantext.util.db_cache import cache
from sqlalchemy import tuple_
from gargantext.models import Ngram, NodeNgram, NodeNodeNgram, NodeNgramNgram
from gargantext.models import Ngram, NodeNgram, NodeNodeNgram, NodeNgramNgram, Node
from gargantext.util.lists import UnweightedList, Translations
# useful subroutines
from gargantext.util.ngramlists_tools import query_list, export_ngramlists, \
import_ngramlists, merge_ngramlists
import_ngramlists, merge_ngramlists, \
import_and_merge_ngramlists
from gargantext.util.group_tools import query_grouped_ngrams
......@@ -82,23 +83,22 @@ class CSVLists(APIView):
# ----------------------
csv_file = request.data['csvfile']
# import the csv
try:
new_lists = import_ngramlists(csv_file)
print("======new_lists=========================!!!")
# print(new_lists) # very long
del csv_file
csv_contents = csv_file.read().decode("UTF-8").split("\n")
csv_file.close()
del csv_file
# merge the new_lists onto those of the target corpus
log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node)
return JsonHttpResponse({
'log': log_msg,
}, 200)
# import the csv
# try:
log_msg = import_and_merge_ngramlists(csv_contents,
onto_corpus_id = corpus_node.id)
return JsonHttpResponse({
'log': log_msg,
}, 200)
except Exception as e:
return JsonHttpResponse({
'err': str(e),
}, 400)
# except Exception as e:
# return JsonHttpResponse({
# 'err': str(e),
# }, 400)
def patch(self,request):
"""
......@@ -361,6 +361,7 @@ class ListChange(APIView):
1) Checks current user authentication to prevent remote DB manipulation
2) Prepares self.list_objects from params
"""
if not request.user.is_authenticated():
raise Http404()
# can't use return in initial() (although 401 maybe better than 404)
......@@ -368,8 +369,20 @@ class ListChange(APIView):
# get validated params
self.params = get_parameters(request)
(self.base_list, self.change_list) = ListChange._validate(self.params)
if not len(self.change_list.items):
payload_ngrams = request.data['ngrams']
# print("no change_list in params but we got:", payload_ngrams)
# change_list can be in payload too
change_ngram_ids = [int(n) for n in payload_ngrams.split(',')]
if (not len(change_ngram_ids)):
raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
else:
self.change_list = UnweightedList(change_ngram_ids)
def put(self, request):
"""
Adds one or more ngrams to a list.
......@@ -406,6 +419,7 @@ class ListChange(APIView):
'count_removed': len(self.base_list.items) - len(new_list.items),
}, 200)
@staticmethod
def _validate(params):
"""
......@@ -420,9 +434,9 @@ class ListChange(APIView):
if 'list' not in params:
raise ValidationException('The route /api/ngramlists/change requires a "list" \
parameter, for instance /api/ngramlists/change?list_id=42')
if 'ngrams' not in params:
raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
# if 'ngrams' not in params:
# raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
# parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
# 2 x retrieval => 2 x UnweightedLists
# ------------------------------------
......@@ -430,17 +444,18 @@ class ListChange(APIView):
try:
base_list_id = int(params['list'])
# UnweightedList retrieved by id
base_list = UnweightedList(base_list_id)
except:
raise ValidationException('The "list" parameter requires an existing list id.')
base_list = UnweightedList(base_list_id)
change_ngram_ids = []
try:
change_ngram_ids = [int(n) for n in params['ngrams'].split(',')]
# UnweightedList created from items
change_list = UnweightedList(change_ngram_ids)
except:
raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
# ngrams no longer mandatory inline, see payload check afterwards
pass
change_list = UnweightedList(change_ngram_ids)
return(base_list, change_list)
......
static/lib/gargantext/NGrams_dyna_chart_and_table.js
View file @ 48a58dad
......@@ -1818,14 +1818,25 @@ function SaveLocalChanges() {
// For list modifications (add/delete), all http-requests
function CRUD( list_id , ngram_ids , http_method , callback) {
// ngramlists/change?node_id=42&ngram_ids=1,2
var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
// var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id;
// debug
// console.log(" ajax target: " + the_url + " (" + http_method + ")")
// 2016-10-05 pass PUT and DELETE ngrams in payload as if it was POSTs
// (to avoid too long urls that trigger Bad Gateway in production)
var myNgramsData = new FormData();
myNgramsData.append("ngrams", ngram_ids.join(","))
if(ngram_ids.length>0) {
$.ajax({
method: http_method,
async: true,
contentType: false,
processData: false,
data: myNgramsData,
url: the_url,
// data: args, // currently all data explicitly in the url (like a GET)
beforeSend: function(xhr) {
......
templates/pages/corpora/myGraphs.html
View file @ 48a58dad
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment