Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
humanities
gargantext
Commits
c7526ad7
Commit
c7526ad7
authored
Apr 01, 2016
by
delanoe
Browse files
Options
Browse Files
Download
Plain Diff
[FIX] merge url conflicts.
parents
8be7e5a7
a77ea0cf
Changes
10
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
348 additions
and
137 deletions
+348
-137
ngram_parsing_flow.dot
doc/schemas/ngram_parsing_flow.dot
+20
-0
ngram_parsing_flow.png
doc/schemas/ngram_parsing_flow.png
+0
-0
constants.py
gargantext/constants.py
+4
-4
db.py
gargantext/util/db.py
+1
-2
CSV.py
gargantext/util/parsers/CSV.py
+20
-27
__init__.py
gargantext/util/parsers/__init__.py
+1
-1
ngramlists.py
gargantext/views/api/ngramlists.py
+148
-27
urls.py
gargantext/views/api/urls.py
+8
-7
NGrams_dyna_chart_and_table.js
static/js/gargantext/NGrams_dyna_chart_and_table.js
+139
-67
terms.html
templates/pages/corpora/terms.html
+7
-2
No files found.
doc/schemas/ngram_parsing_flow.dot
0 → 100644
View file @
c7526ad7
// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
digraph
ngramflow
{
edge
[
fontsize
=
10
]
;
label
=<
<B><U>
gargantext.util.toolchain
</U></B><BR/>
(ngram extraction flow)
>;
labelloc
=
"t"
;
"extracted_ngrams"
->
"grouplist"
;
"extracted_ngrams"
->
"occs+tfidfs"
;
"main_user_stoplist"
->
"stoplist"
;
"stoplist"
->
"mainlist"
;
"occs+tfidfs"
->
"mainlist"
[
label
=
" TFIDF_LIMIT"
]
;
"mainlist"
->
"coocs"
[
label
=
" COOCS_THRESHOLD"
]
;
"coocs"
->
"specificity"
;
"specificity"
->
"maplist"
[
label
=
"MAPLIST_LIMIT\nMONOGRAM_PART"
]
;
"maplist"
->
"explore"
;
"grouplist"
->
"maplist"
;
}
doc/ngram_parsing_flow.png
→
doc/
schemas/
ngram_parsing_flow.png
View file @
c7526ad7
File moved
gargantext/constants.py
View file @
c7526ad7
...
...
@@ -156,10 +156,10 @@ RESOURCETYPES = [
'parser'
:
RISParser
,
'default_language'
:
'en'
,
},
#
{ 'name': 'CSV',
# #
'parser': CSVParser,
#
'default_language': 'en',
#
},
{
'name'
:
'CSV'
,
'parser'
:
CSVParser
,
'default_language'
:
'en'
,
},
# { 'name': 'ISTex',
# # 'parser': ISTexParser,
# 'default_language': 'en',
...
...
gargantext/util/db.py
View file @
c7526ad7
...
...
@@ -35,8 +35,7 @@ Double = DOUBLE_PRECISION
# useful for queries
from
sqlalchemy.orm
import
aliased
from
sqlalchemy
import
func
from
sqlalchemy
import
func
,
desc
# bulk insertions
...
...
gargantext/util/parsers/CSV
Parser
.py
→
gargantext/util/parsers/CSV.py
View file @
c7526ad7
...
...
@@ -8,34 +8,32 @@ import os
class
CSVParser
(
Parser
):
def
CSVsample
(
self
,
filename
,
delim
)
:
ifile
=
open
(
filename
,
"r"
)
reader
=
csv
.
reader
(
ifile
,
delimiter
=
delim
)
def
CSVsample
(
self
,
small_contents
,
delim
)
:
reader
=
csv
.
reader
(
small_contents
,
delimiter
=
delim
)
Freqs
=
[]
for
row
in
reader
:
Freqs
.
append
(
len
(
row
))
ifile
.
close
()
return
Freqs
def
parse
(
self
,
filename
):
print
(
"CSV: parsing (assuming UTF-8 and LF line endings)"
)
contents
=
filename
.
read
()
.
decode
(
"UTF-8"
)
.
split
(
"
\n
"
)
sample_size
=
10
sample_
file
=
filename
.
replace
(
".csv"
,
"_sample.csv"
)
sample_
contents
=
contents
[
0
:
sample_size
]
hyperdata_list
=
[]
command_for_sample
=
"cat '"
+
filename
+
"' | head -n "
+
str
(
sample_size
)
+
" > '"
+
sample_file
+
"'"
os
.
system
(
command_for_sample
)
# you just created a *_sample.csv
# # = = = = [ Getting delimiters frequency ] = = = = #
PossibleDelimiters
=
[
','
,
' '
,
'
\t
'
,
';'
,
'|'
,
':'
]
AllDelimiters
=
{}
for
delim
in
PossibleDelimiters
:
AllDelimiters
[
delim
]
=
self
.
CSVsample
(
sample_
file
,
delim
)
AllDelimiters
[
delim
]
=
self
.
CSVsample
(
sample_
contents
,
delim
)
# # = = = = [ / Getting delimiters frequency ] = = = = #
# # OUTPUT example:
# # AllDelimiters = {
...
...
@@ -68,23 +66,22 @@ class CSVParser(Parser):
# # = = = = [ Delimiter selection ] = = = = #
Sorted_Delims
=
sorted
(
Delimiters
,
key
=
lambda
x
:
x
[
1
],
reverse
=
True
)
HighestDelim
=
Sorted_Delims
[
0
][
0
]
#
print("selected delimiter:",[HighestDelim]
# print
#
HighestDelim = ","
print
(
"CSV selected delimiter:"
,[
HighestDelim
])
# # = = = = [ / Delimiter selection ] = = = = #
# # = = = = [ First data coordinate ] = = = = #
Coords
=
{
"row"
:
-
1
,
"column"
:
-
1
}
ifile
=
open
(
sample_file
,
"r"
)
reader
=
csv
.
reader
(
ifile
,
delimiter
=
HighestDelim
)
reader
=
csv
.
reader
(
contents
,
delimiter
=
HighestDelim
)
for
rownum
,
tokens
in
enumerate
(
reader
):
if
rownum
%
250
==
0
:
print
(
"CSV row: "
,
rownum
)
joined_tokens
=
""
.
join
(
tokens
)
if
Coords
[
"row"
]
<
0
and
len
(
joined_tokens
)
>
0
:
Coords
[
"row"
]
=
rownum
...
...
@@ -93,22 +90,21 @@ class CSVParser(Parser):
if
len
(
t
)
>
0
:
Coords
[
"column"
]
=
columnum
break
ifile
.
close
()
# # = = = = [ / First data coordinate ] = = = = #
# # = = = = [ Setting Headers ] = = = = #
Headers_Int2Str
=
{}
ifile
=
open
(
sample_file
,
"r"
)
reader
=
csv
.
reader
(
ifile
,
delimiter
=
HighestDelim
)
reader
=
csv
.
reader
(
contents
,
delimiter
=
HighestDelim
)
for
rownum
,
tokens
in
enumerate
(
reader
):
if
rownum
>=
Coords
[
"row"
]:
for
columnum
in
range
(
Coords
[
"column"
],
len
(
tokens
)
):
t
=
tokens
[
columnum
]
Headers_Int2Str
[
columnum
]
=
t
break
ifile
.
close
()
# print("Headers_Int2Str")
# print(Headers_Int2Str)
# # = = = = [ / Setting Headers ] = = = = #
# # OUTPUT example:
# # Headers_Int2Str = {
...
...
@@ -119,11 +115,9 @@ class CSVParser(Parser):
# # }
# # = = = = [ Reading the whole CSV and saving ] = = = = #
hyperdata_list
=
[]
ifile
=
open
(
filename
,
"r"
)
reader
=
csv
.
reader
(
ifile
,
delimiter
=
HighestDelim
)
reader
=
csv
.
reader
(
contents
,
delimiter
=
HighestDelim
)
for
rownum
,
tokens
in
enumerate
(
reader
):
if
rownum
>
Coords
[
"row"
]:
RecordDict
=
{}
...
...
@@ -131,7 +125,6 @@ class CSVParser(Parser):
data
=
tokens
[
columnum
]
RecordDict
[
Headers_Int2Str
[
columnum
]
]
=
data
hyperdata_list
.
append
(
RecordDict
)
ifile
.
close
()
# # = = = = [ / Reading the whole CSV and saving ] = = = = #
return
hyperdata_list
gargantext/util/parsers/__init__.py
View file @
c7526ad7
...
...
@@ -8,4 +8,4 @@ from .Pubmed import PubmedParser
from
.Europress
import
EuropressParser
# from .ISTex import ISTexParser
#
from .CSV import CSVParser
from
.CSV
import
CSVParser
gargantext/views/api/ngramlists.py
View file @
c7526ad7
"""
API views for advanced operations on ngrams and ngramlists
-----------------------------------------------------------
- retrieve several lists together ("family")
- retrieve detailed list infos (ngram_id, term strings, scores...)
- modify NodeNgram lists (PUT/DEL an ngram to a MAINLIST OR MAPLIST...)
"""
from
gargantext.util.http
import
APIView
,
get_parameters
,
JsonHttpResponse
,
\
ValidationException
from
gargantext.util.db
import
session
,
aliased
ValidationException
,
Http404
from
gargantext.util.db
import
session
,
aliased
,
desc
from
gargantext.util.db_cache
import
cache
from
gargantext.models
import
Ngram
,
NodeNgram
,
NodeNodeNgram
from
gargantext.util.lists
import
Translations
from
sqlalchemy
import
desc
# from gargantext.constants import *
# from gargantext.util.validation import validate
# from collections import defaultdict
from
gargantext.util.lists
import
UnweightedList
,
Translations
def
_query_list
(
list_id
,
...
...
@@ -64,6 +68,116 @@ class List(APIView):
"""
pass
class
ListChange
(
APIView
):
"""
Any ngram action on standard NodeNgram lists (MAIN, MAP, STOP)
USAGE EXEMPLE:
HOST/api/ngramlists/change?list=42&ngrams=1,2,3,4,5
vvvvvv ||||||
old list vvvvvv
to modify new list items
| |
v v
2 x UnweightedLists: self.base_list self.change_list
We use DEL/PUT HTTP methods to differentiate the 2 basic rm/add actions
They rely only on inline parameters (no need for payload data)
No chained effects: eg removing from MAPLIST will not remove
automatically from associated MAINLIST
NB: request.user is also checked for current authentication status
"""
def
initial
(
self
,
request
):
"""
Before dispatching to put(), delete()...
1) Checks current user authentication to prevent remote DB manipulation
2) Prepares self.list_objects from params
"""
if
not
request
.
user
.
is_authenticated
():
raise
Http404
()
# can't use return in initial() (although 401 maybe better than 404)
# can't use @requires_auth because of positional 'self' within class
# get validated params
self
.
params
=
get_parameters
(
request
)
(
self
.
base_list
,
self
.
change_list
)
=
ListChange
.
_validate
(
self
.
params
)
def
put
(
self
,
request
):
"""
Adds one or more ngrams to a list.
"""
# union of items ----------------------------
new_list
=
self
.
base_list
+
self
.
change_list
# -------------------------------------------
# save
new_list
.
save
(
self
.
base_list
.
id
)
return
JsonHttpResponse
({
'parameters'
:
self
.
params
,
'count_added'
:
len
(
new_list
.
items
)
-
len
(
self
.
base_list
.
items
),
},
201
)
def
delete
(
self
,
request
):
"""
Removes one or more ngrams from a list.
"""
# removal (set difference) ------------------
new_list
=
self
.
base_list
-
self
.
change_list
# -------------------------------------------
# save
new_list
.
save
(
self
.
base_list
.
id
)
return
JsonHttpResponse
({
'parameters'
:
self
.
params
,
'count_removed'
:
len
(
self
.
base_list
.
items
)
-
len
(
new_list
.
items
),
},
200
)
@
staticmethod
def
_validate
(
params
):
"""
Checks "list" and "ngrams" parameters for their:
- presence
- type
These two parameters are mandatory for any ListChange methods.
ngrams are also converted to an UnweightedList object for easy add/remove
"""
if
'list'
not
in
params
:
raise
ValidationException
(
'The route /api/ngramlists/change requires a "list"
\
parameter, for instance /api/ngramlists/change?list_id=42'
)
if
'ngrams'
not
in
params
:
raise
ValidationException
(
'The route /api/ngramlists/change requires an "ngrams"
\
parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4'
)
# 2 x retrieval => 2 x UnweightedLists
# ------------------------------------
base_list_id
=
None
try
:
base_list_id
=
int
(
params
[
'list'
])
# UnweightedList retrieved by id
base_list
=
UnweightedList
(
base_list_id
)
except
:
raise
ValidationException
(
'The "list" parameter requires an existing list id.'
)
change_ngram_ids
=
[]
try
:
change_ngram_ids
=
[
int
(
n
)
for
n
in
params
[
'ngrams'
]
.
split
(
','
)]
# UnweightedList created from items
change_list
=
UnweightedList
(
change_ngram_ids
)
except
:
raise
ValidationException
(
'The "ngrams" parameter requires one or more ngram_ids separated by comma'
)
return
(
base_list
,
change_list
)
class
ListFamily
(
APIView
):
"""
Compact combination of *multiple* list info
...
...
@@ -78,14 +192,14 @@ class ListFamily(APIView):
USAGE EXEMPLES
HOST/api/ngramlists/family?corpus=2
HOST/api/ngramlists/family?corpus=2&
glance
=10
HOST/api/ngramlists/family?corpus=2&
head
=10
HOST/api/ngramlists/family?mainlist=91&scoring=94
HOST/api/ngramlists/family?mainlist=91&scoring=94&
glance
=10
HOST/api/ngramlists/family?mainlist=91&scoring=94&
head
=10
HOST/api/ngramlists/family?mainlist=91&stoplist=90&scoring=94
etc.
REST Parameters:
"
glance
=20"
"
head
=20"
use pagination to only load the k top ngrams of the mainlist
(useful for fast loading of terms view)
"corpus=ID"
...
...
@@ -100,10 +214,10 @@ class ListFamily(APIView):
parameters
=
get_parameters
(
request
)
glance_limit
=
None
mainlist
=
None
mainlist
_id
=
None
scores_id
=
None
groups_id
=
None
secondary_list
s
=
{
'maplist'
:
None
,
'stoplist'
:
None
}
other_list_id
s
=
{
'maplist'
:
None
,
'stoplist'
:
None
}
# 1) retrieve a mainlist_id and other lists
##########################################
...
...
@@ -119,22 +233,22 @@ class ListFamily(APIView):
else
:
scores_id
=
corpus
.
children
(
'OCCURRENCES'
)
.
first
()
.
id
# retrieve the family of lists that have corpus as parent
mainlist
=
corpus
.
children
(
'MAINLIST'
)
.
first
()
.
id
,
mainlist
_id
=
corpus
.
children
(
'MAINLIST'
)
.
first
()
.
id
groups_id
=
corpus
.
children
(
'GROUPLIST'
)
.
first
()
.
id
secondary_list
s
[
'stoplist'
]
=
corpus
.
children
(
'STOPLIST'
)
.
first
()
.
id
secondary_lists
[
'maplist'
]
=
corpus
.
children
(
'MAPLIST'
)
.
first
()
.
id
,
other_list_id
s
[
'stoplist'
]
=
corpus
.
children
(
'STOPLIST'
)
.
first
()
.
id
other_list_ids
[
'maplist'
]
=
corpus
.
children
(
'MAPLIST'
)
.
first
()
.
id
# custom request: refers to each list individually
# -------------------------------------------------
elif
"mainlist"
in
parameters
and
"scoring"
in
parameters
:
mainlist
=
parameters
[
'mainlist'
]
mainlist
_id
=
parameters
[
'mainlist'
]
scores_id
=
parameters
[
'scoring'
]
groups_id
=
None
if
'groups'
in
parameters
:
groups_id
=
parameters
[
'scoring'
]
for
k
in
[
'stoplist'
,
'maplist'
]:
if
k
in
parameters
:
secondary_list
s
[
k
]
=
parameters
[
k
]
other_list_id
s
[
k
]
=
parameters
[
k
]
# or request has an error
# -----------------------
...
...
@@ -149,21 +263,22 @@ class ListFamily(APIView):
ngraminfo
=
{}
# ngram details sorted per ngram id
linkinfo
=
{}
# ngram groups sorted per ngram id
listmembers
=
{}
# ngram ids sorted per list name
if
"
glance
"
in
parameters
:
#
glance
<=> only mainlist AND only k top ngrams
glance_limit
=
int
(
parameters
[
'
glance
'
])
mainlist_query
=
_query_list
(
mainlist
,
details
=
True
,
if
"
head
"
in
parameters
:
#
head
<=> only mainlist AND only k top ngrams
glance_limit
=
int
(
parameters
[
'
head
'
])
mainlist_query
=
_query_list
(
mainlist
_id
,
details
=
True
,
pagination_limit
=
glance_limit
,
scoring_metric_id
=
scores_id
)
else
:
# infos for all ngrams
mainlist_query
=
_query_list
(
mainlist
,
details
=
True
,
mainlist_query
=
_query_list
(
mainlist
_id
,
details
=
True
,
scoring_metric_id
=
scores_id
)
# and for the other lists (stop and map)
for
li
in
secondary_list
s
:
li_elts
=
_query_list
(
secondary_list
s
[
li
],
details
=
False
for
li
in
other_list_id
s
:
li_elts
=
_query_list
(
other_list_id
s
[
li
],
details
=
False
)
.
all
()
listmembers
[
li
]
=
{
ng
[
0
]:
True
for
ng
in
li_elts
}
# simple array of ngram_ids
listmembers
[
li
]
=
[
ng
[
0
]
for
ng
in
li_elts
]
# and the groupings
if
groups_id
:
...
...
@@ -171,7 +286,6 @@ class ListFamily(APIView):
linkinfo
=
links
.
groups
# the output form
ngraminfo
=
{}
for
ng
in
mainlist_query
.
all
():
ng_id
=
ng
[
0
]
# id => [term, weight]
...
...
@@ -180,5 +294,12 @@ class ListFamily(APIView):
return
JsonHttpResponse
({
'ngraminfos'
:
ngraminfo
,
'listmembers'
:
listmembers
,
'links'
:
linkinfo
'links'
:
linkinfo
,
'nodeids'
:
{
'mainlist'
:
mainlist_id
,
'maplist'
:
other_list_ids
[
'maplist'
],
'stoplist'
:
other_list_ids
[
'stoplist'
],
'groups'
:
groups_id
,
'scores'
:
scores_id
,
}
})
gargantext/views/api/urls.py
View file @
c7526ad7
...
...
@@ -11,17 +11,18 @@ urlpatterns = [
url
(
r'^nodes/(\d+)/facets$'
,
nodes
.
CorpusFacet
.
as_view
()),
url
(
r'^nodes/(\d+)/having$'
,
nodes
.
NodeListHaving
.
as_view
()),
# get a list of ngram_ids or ngram_infos by list_id
#
# url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
# add or remove ngram from a list
# ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
# rm <=> DEL ngramlists/change?list=42&ngrams=1,2
url
(
r'^ngramlists/change$'
,
ngramlists
.
ListChange
.
as_view
()),
# entire combination of lists from a corpus
#
get
entire combination of lists from a corpus
# (or any combination of lists that go together :
# - a mainlist
# - an optional stoplist
# - an optional maplist
# - an optional grouplist
# aka lexical model
url
(
r'^ngramlists/family$'
,
ngramlists
.
ListFamily
.
as_view
()),
# - an optional grouplist
)
url
(
r'^ngramlists/family$'
,
ngramlists
.
ListFamily
.
as_view
()),
]
static/js/gargantext/NGrams_dyna_chart_and_table.js
View file @
c7526ad7
...
...
@@ -318,8 +318,8 @@ function Final_UpdateTable( action ) {
// debug
console
.
log
(
"
\n
FUN Final_UpdateTable()"
)
console
.
log
(
"AjaxRecords"
)
console
.
log
(
AjaxRecords
)
//
console.log("AjaxRecords")
//
console.log(AjaxRecords)
// (1) Identifying if the button is collapsed:
var
isCollapsed
=
false
;
...
...
@@ -346,9 +346,8 @@ function Final_UpdateTable( action ) {
var
dataini
=
(
TheBuffer
[
0
])?
TheBuffer
[
0
]:
oldest
;
var
datafin
=
(
TheBuffer
[
1
])?
TheBuffer
[
1
]:
latest
;
pr
(
"show me the pubs of the selected
period
"
)
pr
(
"show me the pubs of the selected
score range
"
)
pr
(
"
\
tfrom ["
+
dataini
+
"] to ["
+
datafin
+
"]"
)
pr
(
"
\
tfrom ["
+
oldest
+
"] to ["
+
latest
+
"]"
)
TimeRange
=
[]
for
(
var
i
in
AjaxRecords
)
{
...
...
@@ -557,7 +556,7 @@ function transformContent(rec_id) {
result
[
"ngramId"
]
=
ngram_info
[
"id"
]
;
// uncomment if column state (here and in Main_test)
result
[
"state"
]
=
AjaxRecords
[
rec_id
].
state
//
result["state"] = AjaxRecords[rec_id].state
// -------------------------------------------
// check box state columns 'will_be_map' and 'will_be_stop'
...
...
@@ -870,16 +869,16 @@ function SaveLocalChanges() {
}
// [ = = = = / For deleting subforms = = = = ]
console
.
log
(
" = = = = = = = = = == "
)
console
.
log
(
"FlagsBuffer:"
)
console
.
log
(
JSON
.
stringify
(
FlagsBuffer
))
//
console.log(" = = = = = = = = = == ")
//
console.log("FlagsBuffer:")
//
console.log(JSON.stringify(FlagsBuffer))
var
nodes_2del
=
Object
.
keys
(
FlagsBuffer
[
"delete"
]).
map
(
Number
)
var
nodes_2keep
=
Object
.
keys
(
FlagsBuffer
[
"keep"
]).
map
(
Number
)
var
nodes_2del
=
Object
.
keys
(
FlagsBuffer
[
"delete"
]).
map
(
Number
)
// main => stop
var
nodes_2keep
=
Object
.
keys
(
FlagsBuffer
[
"keep"
]).
map
(
Number
)
// ??? stop => main ???
var
nodes_2group
=
$
.
extend
({},
FlagsBuffer
[
"group"
])
var
nodes_2inmap
=
$
.
extend
({},
FlagsBuffer
[
"inmap"
])
var
nodes_2outmap
=
$
.
extend
({},
FlagsBuffer
[
"outmap"
])
var
nodes_2inmap
=
$
.
extend
({},
FlagsBuffer
[
"inmap"
])
// add to map
var
nodes_2outmap
=
$
.
extend
({},
FlagsBuffer
[
"outmap"
])
// remove from map
// console.log("")
// console.log("")
...
...
@@ -896,8 +895,12 @@ function SaveLocalChanges() {
// console.log("")
// console.log("")
var
list_id
=
$
(
"#list_id"
).
val
()
var
corpus_id
=
getIDFromURL
(
"corpora"
)
// retrieve node_ids from hidden input
var
mainlist_id
=
$
(
"#mainlist_id"
).
val
()
var
maplist_id
=
$
(
"#maplist_id"
).
val
()
var
stoplist_id
=
$
(
"#stoplist_id"
).
val
()
// var corpus_id = getIDFromURL( "corpora" )
$
(
"#stoplist_content"
).
html
()
...
...
@@ -906,15 +909,59 @@ function SaveLocalChanges() {
// });
$
(
"#Save_All"
).
append
(
'<img width="8%" src="/static/img/ajax-loader.gif"></img>'
)
CRUD
(
corpus_id
,
"/group"
,
[]
,
nodes_2group
,
"PUT"
,
function
(
result
)
{
CRUD
(
corpus_id
,
"/keep"
,
[]
,
nodes_2inmap
,
"PUT"
,
function
(
result
)
{
CRUD
(
corpus_id
,
"/keep"
,
[]
,
nodes_2outmap
,
"DELETE"
,
function
(
result
)
{
CRUD
(
list_id
,
""
,
nodes_2del
,
[]
,
"DELETE"
,
function
(
result
)
{
window
.
location
.
reload
()
// chained CRUD calls
CRUD_1_AddMap
()
// add some ngrams to maplist
function
CRUD_1_AddMap
()
{
CRUD
(
maplist_id
,
Object
.
keys
(
nodes_2inmap
),
"PUT"
,
function
(
success
)
{
if
(
success
)
{
CRUD_2_RmMap
()
// chained AJAX 1 -> 2
}
else
{
console
.
warn
(
'CRUD error on ngrams add to maplist ('
+
maplist_id
+
')'
)
}
});
}
// remove some ngrams from maplist
function
CRUD_2_RmMap
()
{
CRUD
(
maplist_id
,
Object
.
keys
(
nodes_2outmap
),
"DELETE"
,
function
(
success
)
{
if
(
success
)
{
CRUD_3_AddStopRmMain
()
// chained AJAX 2 -> 3
}
else
{
console
.
warn
(
'CRUD error on ngrams remove from maplist ('
+
maplist_id
+
')'
)
}
});
}
// 2 operations going together: add ngrams to stoplist and remove them from mainlist
function
CRUD_3_AddStopRmMain
()
{
CRUD
(
stoplist_id
,
nodes_2del
,
"PUT"
,
function
(
success
)
{
if
(
success
)
{
// console.log("OK CRUD 3a add stop")
CRUD
(
mainlist_id
,
nodes_2del
,
"DELETE"
,
function
(
success
)
{
if
(
success
)
{
// console.log("OK CRUD 3b rm main")
CRUD_4
()
// chained AJAX 3 -> 4
}
else
{
console
.
warn
(
'CRUD error on ngrams remove from mainlist ('
+
mainlist_id
+
')'
)
}
});
}
else
{
console
.
warn
(
'CRUD error on ngrams add to stoplist ('
+
stoplist_id
+
')'
)
}
});
}
// TODO add to groups
function
CRUD_4
()
{
window
.
location
.
reload
()
// refresh whole page if all OK
}
}
...
...
@@ -938,37 +985,40 @@ $("#Save_All").click(function(){
SaveLocalChanges
()
});
// For list
s
, all http-requests
function
CRUD
(
parent_id
,
action
,
nodes
,
arg
s
,
http_method
,
callback
)
{
var
the_url
=
window
.
location
.
origin
+
"/api/node/"
+
parent_id
+
"/ngrams"
+
action
+
"/"
+
nodes
.
join
(
"+"
);
the_url
=
the_url
.
replace
(
/
\/
$/
,
""
);
//remove trailing slash
// For list
modifications (add/delete)
, all http-requests
function
CRUD
(
list_id
,
ngram_id
s
,
http_method
,
callback
)
{
// ngramlists/change?node_id=42&ngram_ids=1,2
var
the_url
=
window
.
location
.
origin
+
"/api/ngramlists/change?list="
+
list_id
+
"&ngrams="
+
ngram_ids
.
join
(
","
);
// debug
// console.log("CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
// console.log("
starting
CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
if
(
n
odes
.
length
>
0
||
Object
.
keys
(
args
)
.
length
>
0
)
{
if
(
n
gram_ids
.
length
>
0
)
{
$
.
ajax
({
method
:
http_method
,
url
:
the_url
,
data
:
args
,
// data: args, // currently all data explicitly in the url (like a GET)
beforeSend
:
function
(
xhr
)
{
xhr
.
setRequestHeader
(
"X-CSRFToken"
,
getCookie
(
"csrftoken"
));
},
success
:
function
(
data
){
console
.
log
(
"-- CRUD ----------"
)
console
.
log
(
http_method
+
" ok!!"
)
console
.
log
(
nodes
)
console
.
log
(
data
)
console
.
log
(
JSON
.
stringify
(
data
)
)
console
.
log
(
"------------------"
)
callback
(
true
);
},
error
:
function
(
result
)
{
console
.
log
(
"-- CRUD ----------"
)
console
.
log
(
"Data not found in #Save_All"
);
console
.
log
(
result
)
console
.
log
(
"------------------"
)
callback
(
false
);
}
});
}
else
callback
(
fals
e
);
}
}
else
callback
(
tru
e
);
}
...
...
@@ -1003,6 +1053,13 @@ function Main_test( data , initial , search_filter) {
console
.
log
(
" = = = = / MAIN_TEST: = = = = "
)
console
.
log
(
""
)
// Expected infos in "data.ngrams" should have the form:
// { "1": { id: "1", name: "réalité", score: 36 },
// "9": { id: "9", name: "pdg", score: 116 },
// "10": { id:"10", name: "infrastructure", score: 12 } etc. }
// (see filling of rec_info below)
// console.log(data.ngrams)
var
DistributionDict
=
{}
for
(
var
i
in
DistributionDict
)
...
...
@@ -1032,7 +1089,7 @@ function Main_test( data , initial , search_filter) {
div_table
+=
"
\
t"
+
"
\
t"
+
'<th data-dynatable-column="ngramId" style="background-color:grey">ngramId</th>'
+
"
\n
"
;
// uncomment for column stateId (here and in transformContent)
div_table
+=
"
\
t"
+
"
\
t"
+
'<th data-dynatable-column="state" style="background-color:grey">State</th>'
+
"
\n
"
;
//
div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
div_table
+=
"
\
t"
+
"
\
t"
+
'<th data-dynatable-column="name">Terms</th>'
+
"
\n
"
;
div_table
+=
"
\
t"
+
"
\
t"
+
'<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>'
+
"
\n
"
;
...
...
@@ -1089,15 +1146,14 @@ function Main_test( data , initial , search_filter) {
$
(
"#stats"
).
html
(
div_stats
)
AjaxRecords
=
{}
console
.
log
(
"data.ngrams"
)
console
.
log
(
data
.
ngrams
)
for
(
var
id
in
data
.
ngrams
)
{
// console.log(i)
// console.log(data.ngrams[i])
var
le_ngram
=
data
.
ngrams
[
id
]
;
var
node
_info
=
{
var
rec
_info
=
{
"id"
:
le_ngram
.
id
,
"name"
:
le_ngram
.
name
,
"score"
:
le_ngram
.
score
,
...
...
@@ -1112,14 +1168,14 @@ function Main_test( data , initial , search_filter) {
"will_be_map"
:
null
,
"will_be_stop"
:
null
}
// AjaxRecords.push(
node
_info)
AjaxRecords
[
id
]
=
node
_info
// AjaxRecords.push(
rec
_info)
AjaxRecords
[
id
]
=
rec
_info
if
(
!
DistributionDict
[
node_info
.
score
]
)
DistributionDict
[
node
_info
.
score
]
=
0
;
DistributionDict
[
node
_info
.
score
]
++
;
if
(
!
DistributionDict
[
rec_info
.
score
]
)
DistributionDict
[
rec
_info
.
score
]
=
0
;
DistributionDict
[
rec
_info
.
score
]
++
;
}
console
.
log
(
FirstScore
)
//
console.log(FirstScore)
// console.log("The Distribution!:")
// console.log(Distribution)
...
...
@@ -1418,8 +1474,17 @@ GET_(new_url, function(res) {
"nb_ngrams"
:
Object
.
keys
(
main_ngrams_objects
).
length
,
}
}
;
NGrams
[
"map"
]
=
res
.
listmembers
.
maplist
;
NGrams
[
"stop"
]
=
res
.
listmembers
.
stoplist
;
// TODO use simpler struct
// map & stop: 2x(array of ids) ==> 2x(lookup hash)
NGrams
[
"map"
]
=
{}
;
for
(
var
i
in
res
.
listmembers
.
maplist
)
{
var
map_ng_id
=
res
.
listmembers
.
maplist
[
i
]
;
NGrams
[
"map"
][
map_ng_id
]
=
true
;
}
NGrams
[
"stop"
]
=
{}
;
for
(
var
i
in
res
.
listmembers
.
stoplist
)
{
var
stop_ng_id
=
res
.
listmembers
.
stoplist
[
i
]
;
NGrams
[
"stop"
][
stop_ng_id
]
=
true
;
}
NGrams
[
"group"
]
=
{
"links"
:
res
.
links
,
"nodes"
:
{}};
for
(
var
parent_ng_id
in
res
.
links
)
{
NGrams
[
"group"
][
"nodes"
][
parent_ng_id
]
=
false
;
...
...
@@ -1429,8 +1494,15 @@ GET_(new_url, function(res) {
}
}
}
console
.
log
(
'after init NGrams["main"].ngrams'
)
console
.
log
(
NGrams
[
"main"
].
ngrams
)
// console.log('after init NGrams["main"].ngrams')
// console.log(NGrams["main"].ngrams)
// cache all DB node_ids
$
(
"input#mainlist_id"
).
val
(
res
.
nodeids
[
'mainlist'
])
$
(
"input#maplist_id"
).
val
(
res
.
nodeids
[
'maplist'
])
$
(
"input#stoplist_id"
).
val
(
res
.
nodeids
[
'stoplist'
])
$
(
"input#groups_id"
).
val
(
res
.
nodeids
[
'groups'
])
$
(
"input#scores_id"
).
val
(
res
.
nodeids
[
'scores'
])
AfterAjax
()
;
});
...
...
@@ -1537,8 +1609,8 @@ function AfterAjax() {
// console.log('NGrams["group"]["nodes"]')
// console.log( NGrams["group"]["nodes"] )
console
.
log
(
'after subforms deletion NGrams["main"].ngrams'
)
console
.
log
(
NGrams
[
"main"
].
ngrams
)
//
console.log('after subforms deletion NGrams["main"].ngrams')
//
console.log(NGrams["main"].ngrams)
// initialize state of maplist items
if
(
Object
.
keys
(
NGrams
[
"map"
]).
length
>
0
)
{
...
...
templates/pages/corpora/terms.html
View file @
c7526ad7
...
...
@@ -22,7 +22,7 @@
<div
class=
"row"
>
<div
id=
"monthly-move-chart"
>
<center>
Select a
time
range in the chart with blue bars to zoom in
Select a
score/frequency
range in the chart with blue bars to zoom in
<p
align=
"center"
>
<a
class=
"btn btn-xs btn-default"
role=
"button"
href=
"/chart/corpus/{{ corpus.id }}/data.csv"
>
Save
</a>
<a
class=
"btn btn-xs btn-default"
href=
"javascript:volumeChart.filterAll();dc.redrawAll();"
>
Reset
</a></p>
...
...
@@ -41,7 +41,12 @@
<br>
</div>
<input
type=
"hidden"
id=
"list_id"
value=
"{{ list_id }}"
></input>
<!-- (values set by js) caching our DB ids (handy for list update commands) -->
<input
type=
"hidden"
id=
"mainlist_id"
value=
""
></input>
<input
type=
"hidden"
id=
"maplist_id"
value=
""
></input>
<input
type=
"hidden"
id=
"stoplist_id"
value=
""
></input>
<input
type=
"hidden"
id=
"groups_id"
value=
""
></input>
<input
type=
"hidden"
id=
"scores_id"
value=
""
></input>
<div
class=
"row"
>
<div
class=
"panel panel-default"
>
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment