Skip to content

gargantext
gargantext
Commit c33bcce3 authored by delanoe's avatar delanoe
Browse files
Options

Merge branch 'samuel' of ssh://delanoe.org:1979/gargantext into samuel

parents 6189bebd 322bb883
Hide whitespace changes
Inline Side-by-side
Showing with 185 additions and 56 deletions
analysis/cooccurrences.py
View file @ c33bcce3
...@@ -18,7 +18,7 @@ def do_cooc(corpus=None ...@@ -18,7 +18,7 @@ def do_cooc(corpus=None
, start=None, end=None , start=None, end=None
, limit=1000 , limit=1000
, isMonopartite=True , isMonopartite=True
, apax = 2): , apax = 1):
''' '''
Compute the cooccurence matrix and save it, returning NodeNgramNgram.node_id Compute the cooccurence matrix and save it, returning NodeNgramNgram.node_id
For the moment list of paramters are not supported because, lists need to For the moment list of paramters are not supported because, lists need to
......
analysis/functions.py
View file @ c33bcce3
...@@ -30,7 +30,7 @@ def diag_null(x): ...@@ -30,7 +30,7 @@ def diag_null(x):
return x - x * scipy.eye(x.shape[0]) return x - x * scipy.eye(x.shape[0])
def do_distance(cooc_id, field1=None, field2=None, isMonopartite=True, apax=2): def do_distance(cooc_id, field1=None, field2=None, isMonopartite=True):
''' '''
do_distance :: Int -> (Graph, Partition, {ids}, {weight}) do_distance :: Int -> (Graph, Partition, {ids}, {weight})
''' '''
...@@ -118,7 +118,7 @@ def get_cooc(request=None, corpus=None ...@@ -118,7 +118,7 @@ def get_cooc(request=None, corpus=None
, field1='ngrams', field2='ngrams' , field1='ngrams', field2='ngrams'
, cooc_id=None, type='node_link', size=1000 , cooc_id=None, type='node_link', size=1000
, start=None, end=None , start=None, end=None
, apax=2 , apax=1
): ):
''' '''
get_ccoc : to compute the graph. get_ccoc : to compute the graph.
...@@ -291,7 +291,8 @@ def get_graphA( nodeA_type , NodesB , links , corpus ): ...@@ -291,7 +291,8 @@ def get_graphA( nodeA_type , NodesB , links , corpus ):
# = = = = [ 04. Getting A-elems and making the dictionaries] = = = = ] # = = = = [ 04. Getting A-elems and making the dictionaries] = = = = ]
sql_query = 'select node_id,value_string from node_node_hyperdata where node_id IN (' + ','.join(map(str, Docs_and_["nodesB"].keys())) + ")"+' and hyperdata_id='+str(nodeA_type_id) sql_query = 'select node_id,value_string from node_node_hyperdata where node_id IN (' + \
','.join(map(str, Docs_and_["nodesB"].keys())) + ")"+' and hyperdata_id='+str(nodeA_type_id)
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute(sql_query) cursor.execute(sql_query)
results = cursor.fetchall() results = cursor.fetchall()
......
ngram/tools.py
View file @ c33bcce3
from gargantext_web.db import Ngram, NodeNgramNgram from gargantext_web.db import Ngram, NodeNgramNgram
from parsing.corpustools import get_cursor, bulk_insert
from gargantext_web.db import get_cursor, bulk_insert
def insert_ngrams(ngrams,get='terms-id'): def insert_ngrams(ngrams,get='terms-id'):
......
rest_v1_0/api.py
View file @ c33bcce3
...@@ -259,6 +259,8 @@ class Ngrams(APIView): ...@@ -259,6 +259,8 @@ class Ngrams(APIView):
], ],
}) })
class NodesChildrenDuplicates(APIView): class NodesChildrenDuplicates(APIView):
def _fetch_duplicates(self, request, node_id, extra_columns=None, min_count=1): def _fetch_duplicates(self, request, node_id, extra_columns=None, min_count=1):
......
rest_v1_0/graph.py
View file @ c33bcce3
...@@ -21,7 +21,7 @@ class Graph(APIView): ...@@ -21,7 +21,7 @@ class Graph(APIView):
format_ = request.GET.get('format', 'json') format_ = request.GET.get('format', 'json')
type_ = request.GET.get('type', 'node_link') type_ = request.GET.get('type', 'node_link')
apax = request.GET.get('apax', 2) apax = request.GET.get('apax', 1)
corpus = session.query(Node).filter(Node.id==corpus_id).first() corpus = session.query(Node).filter(Node.id==corpus_id).first()
......
rest_v1_0/ngrams.py
View file @ c33bcce3
...@@ -11,12 +11,11 @@ from sqlalchemy.orm import aliased ...@@ -11,12 +11,11 @@ from sqlalchemy.orm import aliased
import datetime import datetime
import copy import copy
from gargantext_web.views import move_to_trash
from gargantext_web.db import session, Node, NodeNgram, NodeNgramNgram, NodeNodeNgram, Ngram, Hyperdata, Node_Ngram
from gargantext_web.db import get_or_create_node
from gargantext_web.validation import validate, ValidationException from gargantext_web.validation import validate, ValidationException
from node import models
from gargantext_web.db import session, Node, NodeNgram, NodeNgramNgram\
, NodeNodeNgram, Ngram, Hyperdata, Node_Ngram, get_or_create_node
def DebugHttpResponse(data): def DebugHttpResponse(data):
return HttpResponse('<html><body style="background:#000;color:#FFF"><pre>%s</pre></body></html>' % (str(data), )) return HttpResponse('<html><body style="background:#000;color:#FFF"><pre>%s</pre></body></html>' % (str(data), ))
...@@ -71,6 +70,12 @@ class APIException(_APIException): ...@@ -71,6 +70,12 @@ class APIException(_APIException):
from rest_framework.decorators import api_view from rest_framework.decorators import api_view
#@login_required #@login_required
# TODO how to secure REST ? # TODO how to secure REST ?
class List(APIView):
pass
class Ngrams(APIView): class Ngrams(APIView):
''' '''
REST application to manage ngrams REST application to manage ngrams
...@@ -212,3 +217,116 @@ class Ngrams(APIView): ...@@ -212,3 +217,116 @@ class Ngrams(APIView):
], ],
}) })
class Group(APIView):
'''
REST API to manage groups of Ngrams
Groups can be synonyms, a cathegory or ngrams groups with stems or lems.
'''
def get_group_id(self , node_id):
node_id = int(node_id)
corpus = session.query(Node).filter(Node.id==node_id).first()
group = get_or_create_node(corpus=corpus, nodetype='Group')
return(group.id)
def get(self, request, corpus_id):
# query ngrams
group_id = self.get_group_id(corpus_id)
#api/node/$corpus_id/ngrams?ngram_id=12
# ngram_id = 1 #request.GET.get('ngram_id', False)
# ngram_id = int(node_id)
# #api/node/$corpus_id/ngrams?all=True
# all_option = request.GET.get('all', False)
# all_option = 1 #int(all_option)
# IMPORTANT: Algorithm for getting the groups:
# 1. pairs_list <- Get all pairs from get_group_id()
# 2. G <- Do a non-directed graph of pairs_list
# 3. DG <- Do a directed graph of pairs_list
# 4. cliques_list <- find_cliques of G
# 5. groups <- Iterate in sinonims_cliques and set the mainNode per each clique: take the highest max_outdegree-node of each clique, using DG
import networkx as nx
G = nx.Graph()
DG = nx.DiGraph()
ngrams_ngrams = (session
.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id==group_id)
)
# ngramy_id=476996, score=1.0, node_id=75081, id=1282846, ngramx_id=493431
for ng in ngrams_ngrams:
# n_x = ( session.query(Ngram).filter(Ngram.id==ng.ngramx_id) ).first()
# n_y = ( session.query(Ngram).filter(Ngram.id==ng.ngramy_id) ).first()
G.add_edge( ng.ngramx_id , ng.ngramy_id )
DG.add_edge( ng.ngramx_id , ng.ngramy_id )
# group = dict(list())
sinonims_cliques = nx.find_cliques( G )
# for nn in ngrams_ngrams.all():
# group[nn.ngramx_id] = group.get(nn.ngramx_id, []) + [nn.ngramy_id]
groups = { "nodes": {} , "links": {} }
for clique in sinonims_cliques:
max_deg = -1
mainNode = -1
mainNode_sinonims = []
for node in clique:
groups["nodes"][node] = "nom_"+str(node)
node_outdeg = DG.out_degree(node)
if node_outdeg>max_deg:
max_deg = node_outdeg
mainNode = node
for node in clique:
if mainNode!=node:
mainNode_sinonims.append( node )
groups["links"][ mainNode ] = mainNode_sinonims
return JsonHttpResponse(groups)
def post(self, request, node_id):
# input validation
input = validate(request.DATA, {'data' : {'source': int, 'target': list}})
group_id = get_group_id(node_id)
for data in input['data']:
if data['source'] > 0 and len(data['target']) > 0:
for target_id in data['target']:
if target_id > 0:
session.add(NodeNgramNgram(node_id=group_id, \
ngramx_id=output['source'], ngramy_id=target_id, score=1))
session.commit()
return JsonHttpResponse(True, 201)
else:
raise APIException('Missing parameter: "{\'data\' : [\'source\': Int, \'target\': [Int]}"', 400)
def delete(self, request, corpus_id):
# input validation
input = validate(request.DATA, {'data' : {'source': int, 'target': list}})
group_id = get_group_id(corpus_id)
for data in input['data']:
if data['source'] > 0 and len(data['target']) > 0:
for target_id in data['target']:
(session.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id==group_id)
.filter(NodeNgramNgram.ngramx_id==data['source'])
.delete()
)
return JsonHttpResponse(True, 201)
else:
raise APIException('Missing parameter: "{\'data\' : [\'source\': Int, \'target\': [Int]}"', 400)
class Keep(APIView):
pass
rest_v1_0/urls.py
View file @ c33bcce3
...@@ -19,8 +19,11 @@ urlpatterns = patterns('', ...@@ -19,8 +19,11 @@ urlpatterns = patterns('',
url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ? url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ?
url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ? url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ?
url(r'node/(\d+)/ngrams$', ngrams.Ngrams.as_view()), # NGRAMS table & annotations
url(r'node/(\d+)/ngrams$', ngrams.Ngrams.as_view()), url(r'node/(\d+)/ngrams$' , ngrams.Ngrams.as_view()),
url(r'node/(\d+)/ngrams/group$', ngrams.Group.as_view()),
url(r'node/(\d+)/ngrams/keep$' , ngrams.Keep.as_view()),
url(r'node/(\d+)/ngrams/list$' , ngrams.List.as_view()),
#url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()), #url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()),
#url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()), #url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()),
......
static/js/NGrams_dyna_chart_and_table.js
View file @ c33bcce3
...@@ -209,6 +209,7 @@ function getRecords() { ...@@ -209,6 +209,7 @@ function getRecords() {
return MyTable.data('dynatable').settings.dataset.originalRecords; return MyTable.data('dynatable').settings.dataset.originalRecords;
} }
// new
function group_mode ( elem ) { function group_mode ( elem ) {
GState=1 GState=1
var elem_id = $( elem ).data("stuff") var elem_id = $( elem ).data("stuff")
...@@ -226,6 +227,7 @@ function group_mode ( elem ) { ...@@ -226,6 +227,7 @@ function group_mode ( elem ) {
MyTable.data('dynatable').dom.update(); MyTable.data('dynatable').dom.update();
} }
// new
function SaveSinonims( gheader , gcontent) { function SaveSinonims( gheader , gcontent) {
console.log("GHEADER:") console.log("GHEADER:")
$(gheader).children('span').each(function () { $(gheader).children('span').each(function () {
...@@ -237,6 +239,7 @@ function SaveSinonims( gheader , gcontent) { ...@@ -237,6 +239,7 @@ function SaveSinonims( gheader , gcontent) {
}); });
} }
// new
$('#group_box_content').bind("DOMSubtreeModified",function(){ $('#group_box_content').bind("DOMSubtreeModified",function(){
console.log( $(this).has( "span" ).length ) console.log( $(this).has( "span" ).length )
var groupdiv = "#group_box" var groupdiv = "#group_box"
...@@ -257,6 +260,7 @@ $('#group_box_content').bind("DOMSubtreeModified",function(){ ...@@ -257,6 +260,7 @@ $('#group_box_content').bind("DOMSubtreeModified",function(){
} }
}) })
// new
function add2group ( elem ) { function add2group ( elem ) {
var elem_id = $( elem ).data("stuff") var elem_id = $( elem ).data("stuff")
...@@ -285,6 +289,7 @@ function add2group ( elem ) { ...@@ -285,6 +289,7 @@ function add2group ( elem ) {
MyTable.data('dynatable').dom.update(); MyTable.data('dynatable').dom.update();
} }
// new
function clickngram_action ( elem ) { function clickngram_action ( elem ) {
var elem_id = $( elem ).data("stuff") var elem_id = $( elem ).data("stuff")
AjaxRecords[elem_id].state = (AjaxRecords[elem_id].state==(System[0]["states"].length-2))?0:(AjaxRecords[elem_id].state+1); AjaxRecords[elem_id].state = (AjaxRecords[elem_id].state==(System[0]["states"].length-2))?0:(AjaxRecords[elem_id].state+1);
...@@ -292,6 +297,7 @@ function clickngram_action ( elem ) { ...@@ -292,6 +297,7 @@ function clickngram_action ( elem ) {
MyTable.data('dynatable').dom.update(); MyTable.data('dynatable').dom.update();
} }
// modified
function transformContent(rec_id) { function transformContent(rec_id) {
var elem = AjaxRecords[rec_id]; var elem = AjaxRecords[rec_id];
var result = {} var result = {}
...@@ -301,7 +307,7 @@ function transformContent(rec_id) { ...@@ -301,7 +307,7 @@ function transformContent(rec_id) {
plus_event = " <a class=\"plusclass\" onclick=\"group_mode(this.parentNode.parentNode)\">(+)</a>" plus_event = " <a class=\"plusclass\" onclick=\"group_mode(this.parentNode.parentNode)\">(+)</a>"
if(GState==1 ) { if(GState==1 ) {
if(elem.state!=1 && elem.state!=3) { // if deleted and already grouped, no Up button if(elem.state!=1 && elem.state!=3) { // if deleted and already grouped, no Up button
plus_event = " <a class=\"plusclass\" onclick=\"add2group(this.parentNode.parentNode)\">(↑)</a>" plus_event = " <a class=\"plusclass\" onclick=\"add2group(this.parentNode.parentNode)\">(▲)</a>"
} }
} }
result["id"] = elem["id"] result["id"] = elem["id"]
...@@ -389,11 +395,6 @@ function Mark_NGram( ngram_id , old_flag , new_flag ) { ...@@ -389,11 +395,6 @@ function Mark_NGram( ngram_id , old_flag , new_flag ) {
return new_flag; return new_flag;
} }
function GroupNGrams() {
for (var i in FlagsBuffer["to_group"]){
console.log( AjaxRecords[i] )
}
}
//generic enough //generic enough
function ulWriter(rowIndex, record, columns, cellWriter) { function ulWriter(rowIndex, record, columns, cellWriter) {
...@@ -414,6 +415,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) { ...@@ -414,6 +415,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
} }
function SelectAll( the_checkbox ) { function SelectAll( the_checkbox ) {
console.log(the_checkbox)
var current_flag = $("input[type='radio'][name='radios']:checked").val() var current_flag = $("input[type='radio'][name='radios']:checked").val()
$("tbody tr").each(function (i, row) { $("tbody tr").each(function (i, row) {
var id = $(row).data('stuff') var id = $(row).data('stuff')
...@@ -499,8 +501,6 @@ $("#Save_All").click(function(){ ...@@ -499,8 +501,6 @@ $("#Save_All").click(function(){
}); });
function Main_test( data , initial) { function Main_test( data , initial) {
...@@ -738,38 +738,46 @@ function Main_test( data , initial) { ...@@ -738,38 +738,46 @@ function Main_test( data , initial) {
return "OK" return "OK"
} }
function getIDFromURL( item ) {
function SearchFilters( elem ) { var pageurl = window.location.href.split("/")
var MODE = elem.value; var cid;
for(var i in pageurl) {
if( MODE == "filter_all") { if(pageurl[i]==item) {
var result = Main_test(AjaxRecords , MODE) cid=parseInt(i);
console.log( result ) break;
return ; }
} }
return pageurl[cid+1];
// if( MODE == "filter_stoplist") {
// }
// if( MODE == "filter_miamlist") {
// }
} }
console.log(window.location.href+"/ngrams.json") // [ = = = = = = = = = = INIT = = = = = = = = = = ]
$.ajax({ var corpus_id = getIDFromURL( "corpus" )
url: window.location.href+"/ngrams.json", var url1=window.location.origin+"/api/node/"+corpus_id+"/ngrams/group",
success: function(data){ url2=window.location.href+"/ngrams.json";
var ngrams_groups, ngrams_data;
$.when(
$.ajax({
type: "GET",
url: url1,
dataType: "json",
success : function(data, textStatus, jqXHR) { ngrams_groups = data },
error: function(exception) {
console.log("first ajax, exception!: "+exception.status)
}
}),
$.ajax({
type: "GET",
url: url2,
dataType: "json",
success : function(data, textStatus, jqXHR) { ngrams_data = data },
error: function(exception) {
console.log("second ajax, exception!: "+exception.status)
}
})
).then(function() {
// Building the Score-Selector // Building the Score-Selector
var FirstScore = data.scores.initial var FirstScore = ngrams_data.scores.initial
var possible_scores = Object.keys( data.ngrams[0].scores ); var possible_scores = Object.keys( ngrams_data.ngrams[0].scores );
var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n"; var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n" scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
for( var i in possible_scores ) { for( var i in possible_scores ) {
...@@ -778,20 +786,16 @@ $.ajax({ ...@@ -778,20 +786,16 @@ $.ajax({
} }
} }
// Initializing the Charts and Table // Initializing the Charts and Table
var result = Main_test( data , FirstScore ) var result = Main_test( ngrams_data , FirstScore )
console.log( result ) console.log( result )
// Listener for onchange Score-Selector // Listener for onchange Score-Selector
scores_div += "<select>"+"\n"; scores_div += "<select>"+"\n";
$("#ScoresBox").html(scores_div) $("#ScoresBox").html(scores_div)
$("#scores_selector").on('change', function() { $("#scores_selector").on('change', function() {
console.log( this.value ) console.log( this.value )
var result = Main_test( data , this.value ) var result = Main_test( ngrams_data , this.value )
console.log( result ) console.log( result )
}); });
}
}); });
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment