Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
humanities
gargantext
Commits
744369cb
Commit
744369cb
authored
Oct 05, 2016
by
Romain Loth
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
terms view: import lists in one function for celery tests
parent
86106b07
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
50 additions
and
32 deletions
+50
-32
ngramlists_tools.py
gargantext/util/ngramlists_tools.py
+33
-15
ngramlists.py
gargantext/views/api/ngramlists.py
+17
-17
No files found.
gargantext/util/ngramlists_tools.py
View file @
744369cb
...
...
@@ -11,7 +11,7 @@ from gargantext.util.group_tools import query_groups, group_union
from
gargantext.util.db
import
session
,
desc
,
func
,
\
bulk_insert_ifnotexists
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
\
NodeNgramNgram
NodeNgramNgram
,
Node
from
gargantext.util.lists
import
UnweightedList
,
Translations
...
...
@@ -327,7 +327,7 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
def
import_ngramlists
(
fnam
e
,
delimiter
=
DEFAULT_CSV_DELIM
,
def
import_ngramlists
(
the_fil
e
,
delimiter
=
DEFAULT_CSV_DELIM
,
group_delimiter
=
DEFAULT_CSV_DELIM_GROUP
):
'''
This function reads a CSV of an ngrams table for a Corpus,
...
...
@@ -385,7 +385,7 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
-------
3 x UnweightedList + 1 x Translations
@param
fname a local filename
or a filehandle-like
@param
the_file a local filename or file contents
or a filehandle-like
@param delimiter a character used as separator in the CSV
@param group_delimiter a character used as grouped subforms separator
(in the last column)
...
...
@@ -418,21 +418,27 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
# =============== READ CSV ===============
if
isinstance
(
fname
,
str
):
fh
=
open
(
fname
,
"r"
)
elif
callable
(
getattr
(
fname
,
"read"
,
None
)):
fh
=
fname
if
isinstance
(
the_file
,
list
):
fname
=
'imported_file'
contents
=
the_file
else
:
raise
TypeError
(
"IMPORT: fname argument has unknown type
%
s"
%
type
(
fh
))
if
isinstance
(
the_file
,
str
):
fh
=
open
(
the_file
,
"r"
)
fname
=
the_file
elif
callable
(
getattr
(
the_file
,
"read"
,
None
)):
fh
=
the_file
fname
=
the_file
else
:
raise
TypeError
(
"IMPORT: the_file argument has unknown type
%
s"
%
type
(
the_file
))
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents
=
fh
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
# reading all directly b/c csv.reader takes only lines or a real fh in bytes
# and we usually have a "false" fh (uploadedfile.InMemoryUploadedFile) in strings
# (but we checked its size before!)
contents
=
fh
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
# end of CSV read
fh
.
close
()
# end of CSV read
fh
.
close
()
# <class 'django.core.files.uploadedfile.InMemoryUploadedFile'>
...
...
@@ -609,7 +615,6 @@ def import_ngramlists(fname, delimiter=DEFAULT_CSV_DELIM,
return
result
def
merge_ngramlists
(
new_lists
=
{},
onto_corpus
=
None
,
del_originals
=
[]):
"""
Integrates an external terms table to the current one:
...
...
@@ -832,3 +837,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
# return a log
return
(
"
\n
"
.
join
(
my_log
))
def
import_and_merge_ngramlists
(
file_contents
,
onto_corpus_id
):
"""
A single function to run import_ngramlists and merge_ngramlists together
"""
new_lists
=
import_ngramlists
(
file_contents
)
corpus_node
=
session
.
query
(
Node
)
.
filter
(
Node
.
id
==
onto_corpus_id
)
.
first
()
# merge the new_lists onto those of the target corpus
log_msg
=
merge_ngramlists
(
new_lists
,
onto_corpus
=
corpus_node
)
return
log_msg
gargantext/views/api/ngramlists.py
View file @
744369cb
...
...
@@ -12,12 +12,13 @@ from gargantext.util.http import APIView, get_parameters, JsonHttpResponse,\
from
gargantext.util.db
import
session
,
aliased
,
bulk_insert
from
gargantext.util.db_cache
import
cache
from
sqlalchemy
import
tuple_
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
NodeNgramNgram
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
,
NodeNgramNgram
,
Node
from
gargantext.util.lists
import
UnweightedList
,
Translations
# useful subroutines
from
gargantext.util.ngramlists_tools
import
query_list
,
export_ngramlists
,
\
import_ngramlists
,
merge_ngramlists
import_ngramlists
,
merge_ngramlists
,
\
import_and_merge_ngramlists
from
gargantext.util.group_tools
import
query_grouped_ngrams
...
...
@@ -82,23 +83,22 @@ class CSVLists(APIView):
# ----------------------
csv_file
=
request
.
data
[
'csvfile'
]
# import the csv
try
:
new_lists
=
import_ngramlists
(
csv_file
)
print
(
"======new_lists=========================!!!"
)
# print(new_lists) # very long
del
csv_file
csv_contents
=
csv_file
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
csv_file
.
close
()
del
csv_file
# merge the new_lists onto those of the target corpus
log_msg
=
merge_ngramlists
(
new_lists
,
onto_corpus
=
corpus_node
)
return
JsonHttpResponse
({
'log'
:
log_msg
,
},
200
)
# import the csv
# try:
log_msg
=
import_and_merge_ngramlists
(
csv_contents
,
onto_corpus_id
=
corpus_node
.
id
)
return
JsonHttpResponse
({
'log'
:
log_msg
,
},
200
)
except
Exception
as
e
:
return
JsonHttpResponse
({
'err'
:
str
(
e
),
},
400
)
#
except Exception as e:
#
return JsonHttpResponse({
#
'err': str(e),
#
}, 400)
def
patch
(
self
,
request
):
"""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment