Skip to content

gargantext
gargantext
Commit a77ea0cf authored by Romain Loth's avatar Romain Loth
Browse files
Options

Update/delete possible for terms table (TODO: groupings)

parent e8bde84e
Hide whitespace changes
Inline Side-by-side
Showing with 303 additions and 104 deletions
gargantext/util/db.py
View file @ a77ea0cf
......@@ -35,8 +35,7 @@ from sqlalchemy.ext.mutable import MutableDict, MutableList
# useful for queries
from sqlalchemy.orm import aliased
from sqlalchemy import func
from sqlalchemy import func, desc
# bulk insertions
......
gargantext/views/api/ngramlists.py
View file @ a77ea0cf
"""
API views for advanced operations on ngrams and ngramlists
-----------------------------------------------------------
- retrieve several lists together ("family")
- retrieve detailed list infos (ngram_id, term strings, scores...)
- modify NodeNgram lists (PUT/DEL an ngram to a MAINLIST OR MAPLIST...)
"""
from gargantext.util.http import APIView, get_parameters, JsonHttpResponse,\
ValidationException
from gargantext.util.db import session, aliased
ValidationException, Http404
from gargantext.util.db import session, aliased, desc
from gargantext.util.db_cache import cache
from gargantext.models import Ngram, NodeNgram, NodeNodeNgram
from gargantext.util.lists import Translations
from sqlalchemy import desc
# from gargantext.constants import *
# from gargantext.util.validation import validate
# from collections import defaultdict
from gargantext.util.lists import UnweightedList, Translations
def _query_list(list_id,
......@@ -64,6 +68,116 @@ class List(APIView):
"""
pass
class ListChange(APIView):
"""
Any ngram action on standard NodeNgram lists (MAIN, MAP, STOP)
USAGE EXEMPLE:
HOST/api/ngramlists/change?list=42&ngrams=1,2,3,4,5
vvvvvv ||||||
old list vvvvvv
to modify new list items
| |
v v
2 x UnweightedLists: self.base_list self.change_list
We use DEL/PUT HTTP methods to differentiate the 2 basic rm/add actions
They rely only on inline parameters (no need for payload data)
No chained effects: eg removing from MAPLIST will not remove
automatically from associated MAINLIST
NB: request.user is also checked for current authentication status
"""
def initial(self, request):
"""
Before dispatching to put(), delete()...
1) Checks current user authentication to prevent remote DB manipulation
2) Prepares self.list_objects from params
"""
if not request.user.is_authenticated():
raise Http404()
# can't use return in initial() (although 401 maybe better than 404)
# can't use @requires_auth because of positional 'self' within class
# get validated params
self.params = get_parameters(request)
(self.base_list, self.change_list) = ListChange._validate(self.params)
def put(self, request):
"""
Adds one or more ngrams to a list.
"""
# union of items ----------------------------
new_list = self.base_list + self.change_list
# -------------------------------------------
# save
new_list.save(self.base_list.id)
return JsonHttpResponse({
'parameters': self.params,
'count_added': len(new_list.items) - len(self.base_list.items),
}, 201)
def delete(self, request):
"""
Removes one or more ngrams from a list.
"""
# removal (set difference) ------------------
new_list = self.base_list - self.change_list
# -------------------------------------------
# save
new_list.save(self.base_list.id)
return JsonHttpResponse({
'parameters': self.params,
'count_removed': len(self.base_list.items) - len(new_list.items),
}, 200)
@staticmethod
def _validate(params):
"""
Checks "list" and "ngrams" parameters for their:
- presence
- type
These two parameters are mandatory for any ListChange methods.
ngrams are also converted to an UnweightedList object for easy add/remove
"""
if 'list' not in params:
raise ValidationException('The route /api/ngramlists/change requires a "list" \
parameter, for instance /api/ngramlists/change?list_id=42')
if 'ngrams' not in params:
raise ValidationException('The route /api/ngramlists/change requires an "ngrams"\
parameter, for instance /api/ngramlists/change?ngrams=1,2,3,4')
# 2 x retrieval => 2 x UnweightedLists
# ------------------------------------
base_list_id = None
try:
base_list_id = int(params['list'])
# UnweightedList retrieved by id
base_list = UnweightedList(base_list_id)
except:
raise ValidationException('The "list" parameter requires an existing list id.')
change_ngram_ids = []
try:
change_ngram_ids = [int(n) for n in params['ngrams'].split(',')]
# UnweightedList created from items
change_list = UnweightedList(change_ngram_ids)
except:
raise ValidationException('The "ngrams" parameter requires one or more ngram_ids separated by comma')
return(base_list, change_list)
class ListFamily(APIView):
"""
Compact combination of *multiple* list info
......@@ -78,14 +192,14 @@ class ListFamily(APIView):
USAGE EXEMPLES
HOST/api/ngramlists/family?corpus=2
HOST/api/ngramlists/family?corpus=2&glance=10
HOST/api/ngramlists/family?corpus=2&head=10
HOST/api/ngramlists/family?mainlist=91&scoring=94
HOST/api/ngramlists/family?mainlist=91&scoring=94&glance=10
HOST/api/ngramlists/family?mainlist=91&scoring=94&head=10
HOST/api/ngramlists/family?mainlist=91&stoplist=90&scoring=94
etc.
REST Parameters:
"glance=20"
"head=20"
use pagination to only load the k top ngrams of the mainlist
(useful for fast loading of terms view)
"corpus=ID"
......@@ -100,10 +214,10 @@ class ListFamily(APIView):
parameters = get_parameters(request)
glance_limit = None
mainlist = None
mainlist_id = None
scores_id = None
groups_id = None
secondary_lists = {'maplist':None, 'stoplist':None}
other_list_ids = {'maplist':None, 'stoplist':None}
# 1) retrieve a mainlist_id and other lists
##########################################
......@@ -119,22 +233,22 @@ class ListFamily(APIView):
else:
scores_id = corpus.children('OCCURRENCES').first().id
# retrieve the family of lists that have corpus as parent
mainlist = corpus.children('MAINLIST').first().id,
mainlist_id = corpus.children('MAINLIST').first().id
groups_id = corpus.children('GROUPLIST').first().id
secondary_lists['stoplist'] = corpus.children('STOPLIST').first().id
secondary_lists['maplist'] = corpus.children('MAPLIST').first().id,
other_list_ids['stoplist'] = corpus.children('STOPLIST').first().id
other_list_ids['maplist'] = corpus.children('MAPLIST').first().id
# custom request: refers to each list individually
# -------------------------------------------------
elif "mainlist" in parameters and "scoring" in parameters:
mainlist = parameters['mainlist']
mainlist_id = parameters['mainlist']
scores_id = parameters['scoring']
groups_id = None
if 'groups' in parameters:
groups_id = parameters['scoring']
for k in ['stoplist', 'maplist']:
if k in parameters:
secondary_lists[k] = parameters[k]
other_list_ids[k] = parameters[k]
# or request has an error
# -----------------------
......@@ -149,21 +263,22 @@ class ListFamily(APIView):
ngraminfo = {} # ngram details sorted per ngram id
linkinfo = {} # ngram groups sorted per ngram id
listmembers = {} # ngram ids sorted per list name
if "glance" in parameters:
# glance <=> only mainlist AND only k top ngrams
glance_limit = int(parameters['glance'])
mainlist_query = _query_list(mainlist, details=True,
if "head" in parameters:
# head <=> only mainlist AND only k top ngrams
glance_limit = int(parameters['head'])
mainlist_query = _query_list(mainlist_id, details=True,
pagination_limit = glance_limit,
scoring_metric_id= scores_id)
else:
# infos for all ngrams
mainlist_query = _query_list(mainlist, details=True,
mainlist_query = _query_list(mainlist_id, details=True,
scoring_metric_id= scores_id)
# and for the other lists (stop and map)
for li in secondary_lists:
li_elts = _query_list(secondary_lists[li], details=False
for li in other_list_ids:
li_elts = _query_list(other_list_ids[li], details=False
).all()
listmembers[li] = {ng[0]:True for ng in li_elts}
# simple array of ngram_ids
listmembers[li] = [ng[0] for ng in li_elts]
# and the groupings
if groups_id:
......@@ -171,7 +286,6 @@ class ListFamily(APIView):
linkinfo = links.groups
# the output form
ngraminfo = {}
for ng in mainlist_query.all():
ng_id = ng[0]
# id => [term, weight]
......@@ -180,5 +294,12 @@ class ListFamily(APIView):
return JsonHttpResponse({
'ngraminfos' : ngraminfo,
'listmembers' : listmembers,
'links' : linkinfo
'links' : linkinfo,
'nodeids' : {
'mainlist': mainlist_id,
'maplist' : other_list_ids['maplist'],
'stoplist': other_list_ids['stoplist'],
'groups': groups_id,
'scores': scores_id,
}
})
gargantext/views/api/urls.py
View file @ a77ea0cf
......@@ -10,16 +10,18 @@ urlpatterns = [
url(r'^nodes/(\d+)/facets$', nodes.CorpusFacet.as_view()),
# get a list of ngram_ids or ngram_infos by list_id
#
# url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
# add or remove ngram from a list
# ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
# rm <=> DEL ngramlists/change?list=42&ngrams=1,2
url(r'^ngramlists/change$', ngramlists.ListChange.as_view()),
# entire combination of lists from a corpus
# get entire combination of lists from a corpus
# (or any combination of lists that go together :
# - a mainlist
# - an optional stoplist
# - an optional maplist
# - an optional grouplist
# aka lexical model
# - an optional grouplist)
url(r'^ngramlists/family$', ngramlists.ListFamily.as_view()),
]
static/js/gargantext/NGrams_dyna_chart_and_table.js
View file @ a77ea0cf
......@@ -318,8 +318,8 @@ function Final_UpdateTable( action ) {
// debug
console.log("\nFUN Final_UpdateTable()")
console.log("AjaxRecords")
console.log(AjaxRecords)
// console.log("AjaxRecords")
// console.log(AjaxRecords)
// (1) Identifying if the button is collapsed:
var isCollapsed=false;
......@@ -346,9 +346,8 @@ function Final_UpdateTable( action ) {
var dataini = (TheBuffer[0])?TheBuffer[0]:oldest;
var datafin = (TheBuffer[1])?TheBuffer[1]:latest;
pr("show me the pubs of the selected period")
pr("show me the pubs of the selected score range")
pr("\tfrom ["+dataini+"] to ["+datafin+"]")
pr("\tfrom ["+oldest+"] to ["+latest+"]")
TimeRange = []
for (var i in AjaxRecords) {
......@@ -557,7 +556,7 @@ function transformContent(rec_id) {
result["ngramId"] = ngram_info["id"] ;
// uncomment if column state (here and in Main_test)
result["state"] = AjaxRecords[rec_id].state
// result["state"] = AjaxRecords[rec_id].state
// -------------------------------------------
// check box state columns 'will_be_map' and 'will_be_stop'
......@@ -870,16 +869,16 @@ function SaveLocalChanges() {
}
// [ = = = = / For deleting subforms = = = = ]
console.log(" = = = = = = = = = == ")
console.log("FlagsBuffer:")
console.log(JSON.stringify(FlagsBuffer))
// console.log(" = = = = = = = = = == ")
// console.log("FlagsBuffer:")
// console.log(JSON.stringify(FlagsBuffer))
var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number)
var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number)
var nodes_2del = Object.keys(FlagsBuffer["delete"]).map(Number) // main => stop
var nodes_2keep = Object.keys(FlagsBuffer["keep"]).map(Number) // ??? stop => main ???
var nodes_2group = $.extend({}, FlagsBuffer["group"])
var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"])
var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"])
var nodes_2inmap = $.extend({}, FlagsBuffer["inmap"]) // add to map
var nodes_2outmap = $.extend({}, FlagsBuffer["outmap"]) // remove from map
// console.log("")
// console.log("")
......@@ -896,8 +895,12 @@ function SaveLocalChanges() {
// console.log("")
// console.log("")
var list_id = $("#list_id").val()
var corpus_id = getIDFromURL( "corpora" )
// retrieve node_ids from hidden input
var mainlist_id = $("#mainlist_id").val()
var maplist_id = $("#maplist_id" ).val()
var stoplist_id = $("#stoplist_id" ).val()
// var corpus_id = getIDFromURL( "corpora" )
$("#stoplist_content").html()
......@@ -906,15 +909,59 @@ function SaveLocalChanges() {
// });
$("#Save_All").append('<img width="8%" src="/static/img/ajax-loader.gif"></img>')
CRUD( corpus_id , "/group" , [] , nodes_2group , "PUT" , function(result) {
CRUD( corpus_id , "/keep" , [] , nodes_2inmap , "PUT" , function(result) {
CRUD( corpus_id , "/keep" , [] , nodes_2outmap , "DELETE" , function(result) {
CRUD( list_id , "" , nodes_2del , [] , "DELETE", function(result) {
window.location.reload()
});
});
});
});
// chained CRUD calls
CRUD_1_AddMap()
// add some ngrams to maplist
function CRUD_1_AddMap() {
CRUD( maplist_id , Object.keys(nodes_2inmap), "PUT" , function(success) {
if (success) {
CRUD_2_RmMap() // chained AJAX 1 -> 2
}
else {
console.warn('CRUD error on ngrams add to maplist ('+maplist_id+')')
}
});
}
// remove some ngrams from maplist
function CRUD_2_RmMap() {
CRUD( maplist_id , Object.keys(nodes_2outmap), "DELETE" , function(success) {
if (success) {
CRUD_3_AddStopRmMain() // chained AJAX 2 -> 3
}
else {
console.warn('CRUD error on ngrams remove from maplist ('+maplist_id+')')
}
});
}
// 2 operations going together: add ngrams to stoplist and remove them from mainlist
function CRUD_3_AddStopRmMain() {
CRUD( stoplist_id , nodes_2del, "PUT" , function(success) {
if (success) {
// console.log("OK CRUD 3a add stop")
CRUD( mainlist_id , nodes_2del, "DELETE" , function(success) {
if (success) {
// console.log("OK CRUD 3b rm main")
CRUD_4() // chained AJAX 3 -> 4
}
else {
console.warn('CRUD error on ngrams remove from mainlist ('+mainlist_id+')')
}
});
}
else {
console.warn('CRUD error on ngrams add to stoplist ('+stoplist_id+')')
}
});
}
// TODO add to groups
function CRUD_4() {
window.location.reload() // refresh whole page if all OK
}
}
......@@ -938,37 +985,40 @@ $("#Save_All").click(function(){
SaveLocalChanges()
});
// For lists, all http-requests
function CRUD( parent_id , action , nodes , args , http_method , callback) {
var the_url = window.location.origin+"/api/node/"+parent_id+"/ngrams"+action+"/"+nodes.join("+");
the_url = the_url.replace(/\/$/, ""); //remove trailing slash
// For list modifications (add/delete), all http-requests
function CRUD( list_id , ngram_ids , http_method , callback) {
// ngramlists/change?node_id=42&ngram_ids=1,2
var the_url = window.location.origin+"/api/ngramlists/change?list="+list_id+"&ngrams="+ngram_ids.join(",");
// debug
// console.log("CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
if(nodes.length>0 || Object.keys(args).length>0) {
$.ajax({
method: http_method,
url: the_url,
data: args,
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data){
console.log(http_method + " ok!!")
console.log(nodes)
console.log(data)
callback(true);
},
error: function(result) {
console.log("Data not found in #Save_All");
console.log(result)
callback(false);
}
});
} else callback(false);
}
// console.log("starting CRUD AJAX => URL: " + the_url + " (" + http_method + ")")
if(ngram_ids.length>0) {
$.ajax({
method: http_method,
url: the_url,
// data: args, // currently all data explicitly in the url (like a GET)
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data){
console.log("-- CRUD ----------")
console.log(http_method + " ok!!")
console.log(JSON.stringify(data))
console.log("------------------")
callback(true);
},
error: function(result) {
console.log("-- CRUD ----------")
console.log("Data not found in #Save_All");
console.log(result)
console.log("------------------")
callback(false);
}
});
} else callback(true);
}
......@@ -1003,6 +1053,13 @@ function Main_test( data , initial , search_filter) {
console.log(" = = = = / MAIN_TEST: = = = = ")
console.log("")
// Expected infos in "data.ngrams" should have the form:
// { "1": { id: "1", name: "réalité", score: 36 },
// "9": { id: "9", name: "pdg", score: 116 },
// "10": { id:"10", name: "infrastructure", score: 12 } etc. }
// (see filling of rec_info below)
// console.log(data.ngrams)
var DistributionDict = {}
for(var i in DistributionDict)
......@@ -1032,7 +1089,7 @@ function Main_test( data , initial , search_filter) {
div_table += "\t"+"\t"+'<th data-dynatable-column="ngramId" style="background-color:grey">ngramId</th>'+"\n";
// uncomment for column stateId (here and in transformContent)
div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
// div_table += "\t"+"\t"+'<th data-dynatable-column="state" style="background-color:grey">State</th>'+"\n" ;
div_table += "\t"+"\t"+'<th data-dynatable-column="name">Terms</th>'+"\n";
div_table += "\t"+"\t"+'<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>'+"\n";
......@@ -1089,15 +1146,14 @@ function Main_test( data , initial , search_filter) {
$("#stats").html(div_stats)
AjaxRecords = {}
console.log("data.ngrams")
console.log(data.ngrams)
for(var id in data.ngrams) {
// console.log(i)
// console.log(data.ngrams[i])
var le_ngram = data.ngrams[id] ;
var node_info = {
var rec_info = {
"id" : le_ngram.id,
"name": le_ngram.name,
"score": le_ngram.score,
......@@ -1112,14 +1168,14 @@ function Main_test( data , initial , search_filter) {
"will_be_map": null,
"will_be_stop": null
}
// AjaxRecords.push(node_info)
AjaxRecords[id] = node_info
// AjaxRecords.push(rec_info)
AjaxRecords[id] = rec_info
if ( ! DistributionDict[node_info.score] ) DistributionDict[node_info.score] = 0;
DistributionDict[node_info.score]++;
if ( ! DistributionDict[rec_info.score] ) DistributionDict[rec_info.score] = 0;
DistributionDict[rec_info.score]++;
}
console.log(FirstScore)
// console.log(FirstScore)
// console.log("The Distribution!:")
// console.log(Distribution)
......@@ -1418,8 +1474,17 @@ GET_(new_url, function(res) {
"nb_ngrams":Object.keys(main_ngrams_objects).length,
}
} ;
NGrams["map"] = res.listmembers.maplist ;
NGrams["stop"] = res.listmembers.stoplist ; // TODO use simpler struct
// map & stop: 2x(array of ids) ==> 2x(lookup hash)
NGrams["map"] = {} ;
for (var i in res.listmembers.maplist) {
var map_ng_id = res.listmembers.maplist[i] ;
NGrams["map"][map_ng_id] = true ;
}
NGrams["stop"] = {} ;
for (var i in res.listmembers.stoplist) {
var stop_ng_id = res.listmembers.stoplist[i] ;
NGrams["stop"][stop_ng_id] = true ;
}
NGrams["group"] = {"links" : res.links , "nodes" : {}};
for (var parent_ng_id in res.links) {
NGrams["group"]["nodes"][parent_ng_id] = false ;
......@@ -1429,8 +1494,15 @@ GET_(new_url, function(res) {
}
}
}
console.log('after init NGrams["main"].ngrams')
console.log(NGrams["main"].ngrams)
// console.log('after init NGrams["main"].ngrams')
// console.log(NGrams["main"].ngrams)
// cache all DB node_ids
$("input#mainlist_id").val(res.nodeids['mainlist'])
$("input#maplist_id" ).val(res.nodeids['maplist'])
$("input#stoplist_id").val(res.nodeids['stoplist'])
$("input#groups_id").val(res.nodeids['groups'])
$("input#scores_id").val(res.nodeids['scores'])
AfterAjax() ;
});
......@@ -1537,8 +1609,8 @@ function AfterAjax() {
// console.log('NGrams["group"]["nodes"]')
// console.log( NGrams["group"]["nodes"] )
console.log('after subforms deletion NGrams["main"].ngrams')
console.log(NGrams["main"].ngrams)
// console.log('after subforms deletion NGrams["main"].ngrams')
// console.log(NGrams["main"].ngrams)
// initialize state of maplist items
if( Object.keys(NGrams["map"]).length>0 ) {
......
templates/pages/corpora/terms.html
View file @ a77ea0cf
......@@ -22,7 +22,7 @@
<div class="row">
<div id="monthly-move-chart">
<center>
Select a time range in the chart with blue bars to zoom in
Select a score/frequency range in the chart with blue bars to zoom in
<p align="center">
<a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
<a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
......@@ -41,7 +41,12 @@
<br>
</div>
<input type="hidden" id="list_id" value="{{ list_id }}"></input>
<!-- (values set by js) caching our DB ids (handy for list update commands) -->
<input type="hidden" id="mainlist_id" value=""></input>
<input type="hidden" id="maplist_id" value=""></input>
<input type="hidden" id="stoplist_id" value=""></input>
<input type="hidden" id="groups_id" value=""></input>
<input type="hidden" id="scores_id" value=""></input>
<div class="row">
<div class="panel panel-default">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment