Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
humanities
gargantext
Commits
45be2f40
Commit
45be2f40
authored
Oct 05, 2016
by
delanoe
Browse files
Options
Browse Files
Download
Plain Diff
Merge remote-tracking branch 'origin/romain-list' into testing-merge
parents
af1147b7
ef3f6f95
Changes
4
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
474 additions
and
356 deletions
+474
-356
ngramlists_tools.py
gargantext/util/ngramlists_tools.py
+33
-15
ngramlists.py
gargantext/views/api/ngramlists.py
+38
-23
NGrams_dyna_chart_and_table.js
static/lib/gargantext/NGrams_dyna_chart_and_table.js
+12
-1
myGraphs.html
templates/pages/corpora/myGraphs.html
+391
-317
No files found.
gargantext/util/ngramlists_tools.py
View file @
45be2f40
...
...
@@ -11,7 +11,7 @@ from gargantext.util.group_tools import query_groups, group_union
from
gargantext.util.db
import
session
,
desc
,
func
,
\
bulk_insert_ifnotexists
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
\
NodeNgramNgram
NodeNgramNgram
,
Node
from
gargantext.util.lists
import
UnweightedList
,
Translations
...
...
@@ -327,7 +327,7 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
def
import_ngramlists
(
fnam
e
,
delimiter
=
DEFAULT_CSV_DELIM
,
def
import_ngramlists
(
the_fil
e
,
delimiter
=
DEFAULT_CSV_DELIM
,
group_delimiter
=
DEFAULT_CSV_DELIM_GROUP
):
'''
This function reads a CSV of an ngrams table for a Corpus,
...
...
@@ -385,7 +385,7 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
-------
3 x UnweightedList + 1 x Translations
@param
fname a local filename
or a filehandle-like
@param
the_file a local filename or file contents
or a filehandle-like
@param delimiter a character used as separator in the CSV
@param group_delimiter a character used as grouped subforms separator
(in the last column)
...
...
@@ -418,21 +418,27 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
# =============== READ CSV ===============
if
isinstance
(
fname
,
str
):
fh
=
open
(
fname
,
"r"
)
elif
callable
(
getattr
(
fname
,
"read"
,
None
)):
fh
=
fname
if
isinstance
(
the_file
,
list
):
fname
=
'imported_file'
contents
=
the_file
else
:
raise
TypeError
(
"IMPORT: fname argument has unknown type
%
s"
%
type
(
fh
))
if
isinstance
(
the_file
,
str
):
fh
=
open
(
the_file
,
"r"
)
fname
=
the_file
elif
callable
(
getattr
(
the_file
,
"read"
,
None
)):
fh
=
the_file
fname
=
the_file
else
:
raise
TypeError
(
"IMPORT: the_file argument has unknown type
%
s"
%
type
(
the_file
))
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents
=
fh
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents
=
fh
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
# end of CSV read
fh
.
close
()
# end of CSV read
fh
.
close
()
# <class 'django.core.files.uploadedfile.InMemoryUploadedFile'>
...
...
@@ -609,7 +615,6 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
return
result
def
merge_ngramlists
(
new_lists
=
{},
onto_corpus
=
None
,
del_originals
=
[]):
"""
Integrates an external terms table to the current one:
...
...
@@ -832,3 +837,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
# return a log
return
(
"
\n
"
.
join
(
my_log
))
def
import_and_merge_ngramlists
(
file_contents
,
onto_corpus_id
):
"""
A single function to run import_ngramlists and merge_ngramlists together
"""
new_lists
=
import_ngramlists
(
file_contents
)
corpus_node
=
session
.
query
(
Node
)
.
filter
(
Node
.
id
==
onto_corpus_id
)
.
first
()
# merge the new_lists onto those of the target corpus
log_msg
=
merge_ngramlists
(
new_lists
,
onto_corpus
=
corpus_node
)
return
log_msg
gargantext/views/api/ngramlists.py
View file @
45be2f40
...
...
@@ -12,12 +12,13 @@ from gargantext.util.http import APIView, get_parameters, JsonHttpResponse,\
from
gargantext.util.db
import
session
,
aliased
,
bulk_insert
from
gargantext.util.db_cache
import
cache
from
sqlalchemy
import
tuple_
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
NodeNgramNgram
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
NodeNgramNgram
,
Node
from
gargantext.util.lists
import
UnweightedList
,
Translations
# useful subroutines
from
gargantext.util.ngramlists_tools
import
query_list
,
export_ngramlists
,
\
import_ngramlists
,
merge_ngramlists
import_ngramlists
,
merge_ngramlists
,
\
import_and_merge_ngramlists
from
gargantext.util.group_tools
import
query_grouped_ngrams
...
...
@@ -82,23 +83,22 @@ class CSVLists(APIView):
# ----------------------
csv_file
=
request
.
data
[
'csvfile'
]
# import the csv
try
:
new_lists
=
import_ngramlists
(
csv_file
)
print
(
"======new_lists=========================!!!"
)
# print(new_lists) # very long
del
csv_file
csv_contents
=
csv_file
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
csv_file
.
close
()
del
csv_file
# merge the new_lists onto those of the target corpus
log_msg
=
merge_ngramlists
(
new_lists
,
onto_corpus
=
corpus_node
)
return
JsonHttpResponse
({
'log'
:
log_msg
,
},
200
)
# import the csv
# try:
log_msg
=
import_and_merge_ngramlists
(
csv_contents
,
onto_corpus_id
=
corpus_node
.
id
)
return
JsonHttpResponse
({
'log'
:
log_msg
,
},
200
)
except
Exception
as
e
:
return
JsonHttpResponse
({
'err'
:
str
(
e
),
},
400
)
#
except Exception as e:
#
return JsonHttpResponse({
#
'err': str(e),
#
}, 400)
def
patch
(
self
,
request
):
"""
...
...
@@ -361,6 +361,7 @@ class ListChange(APIView):
1) Checks current user authentication to prevent remote DB manipulation
2) Prepares self.list_objects from params
"""
if
not
request
.
user
.
is_authenticated
():
raise
Http404
()
# can't use return in initial() (although 401 maybe better than 404)
...
...
@@ -368,8 +369,20 @@ class ListChange(APIView):
# get validated params
self
.
params
=
get_parameters
(
request
)
(
self
.
base_list
,
self
.
change_list
)
=
ListChange
.
_validate
(
self
.
params
)
if
not
len
(
self
.
change_list
.
items
):
payload_ngrams
=
request
.
data
[
'ngrams'
]
# print("no change_list in params but we got:", payload_ngrams)
# change_list can be in payload too
change_ngram_ids
=
[
int
(
n
)
for
n
in
payload_ngrams
.
split
(
','
)]
if
(
not
len
(
change_ngram_ids
)):
raise
ValidationException
(
'The "ngrams" parameter requires one or more ngram_ids separated by comma'
)
else
:
self
.
change_list
=
UnweightedList
(
change_ngram_ids
)
def
put
(
self
,
request
):
"""
Adds one or more ngrams to a list.
...
...
@@ -406,6 +419,7 @@ class ListChange(APIView):
'count_removed'
:
len
(
self
.
base_list
.
items
)
-
len
(
new_list
.
items
),
},
200
)
@
staticmethod
def
_validate
(
params
):
"""
...
...
@@ -420,9 +434,9 @@ class ListChange(APIView):
if
'list'
not
in
params
:
raise
ValidationException
(
'The route /api/ngramlists/change requires a "list"
\
parameter, for instance /api/ngramlists/change?list_id=42'
)
if
'ngrams'
not
in
params
:
raise
ValidationException
(
'The route /api/ngramlists/change requires an "ngrams"
\
parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4'
)
#
if 'ngrams' not in params:
#
raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
#
parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
# 2 x retrieval => 2 x UnweightedLists
# ------------------------------------
...
...
@@ -430,17 +444,18 @@ class ListChange(APIView):
try
:
base_list_id
=
int
(
params
[
'list'
])
# UnweightedList retrieved by id
base_list
=
UnweightedList
(
base_list_id
)
except
:
raise
ValidationException
(
'The "list" parameter requires an existing list id.'
)
base_list
=
UnweightedList
(
base_list_id
)
change_ngram_ids
=
[]
try
:
change_ngram_ids
=
[
int
(
n
)
for
n
in
params
[
'ngrams'
]
.
split
(
','
)]
# UnweightedList created from items
change_list
=
UnweightedList
(
change_ngram_ids
)
except
:
raise
ValidationException
(
'The "ngrams" parameter requires one or more ngram_ids separated by comma'
)
# ngrams no longer mandatory inline, see payload check afterwards
pass
change_list
=
UnweightedList
(
change_ngram_ids
)
return
(
base_list
,
change_list
)
...
...
static/lib/gargantext/NGrams_dyna_chart_and_table.js
View file @
45be2f40
...
...
@@ -1818,14 +1818,25 @@ function SaveLocalChanges() {
// For list modifications (add/delete), all http-requests
function
CRUD
(
list_id
,
ngram_ids
,
http_method
,
callback
)
{
// ngramlists/change?node_id=42&ngram_ids=1,2
var
the_url
=
window
.
location
.
origin
+
"/api/ngramlists/change?list="
+
list_id
+
"&ngrams="
+
ngram_ids
.
join
(
","
);
// var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
var
the_url
=
window
.
location
.
origin
+
"/api/ngramlists/change?list="
+
list_id
;
// debug
// console.log(" ajax target: " + the_url + " (" + http_method + ")")
// 2016-10-05 pass PUT and DELETE ngrams in payload as if it was POSTs
// (to avoid too long urls that trigger Bad Gateway in production)
var
myNgramsData
=
new
FormData
();
myNgramsData
.
append
(
"ngrams"
,
ngram_ids
.
join
(
","
))
if
(
ngram_ids
.
length
>
0
)
{
$
.
ajax
({
method
:
http_method
,
async
:
true
,
contentType
:
false
,
processData
:
false
,
data
:
myNgramsData
,
url
:
the_url
,
// data: args, // currently all data explicitly in the url (like a GET)
beforeSend
:
function
(
xhr
)
{
...
...
templates/pages/corpora/myGraphs.html
View file @
45be2f40
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment