Skip to content

gargantext
gargantext
Commit c33bcce3 authored by delanoe's avatar delanoe
Browse files
Options

Merge branch 'samuel' of ssh://delanoe.org:1979/gargantext into samuel

parents 6189bebd 322bb883
Hide whitespace changes
Inline Side-by-side
Showing with 185 additions and 56 deletions
analysis/cooccurrences.py
View file @ c33bcce3
......@@ -18,7 +18,7 @@ def do_cooc(corpus=None
, start=None, end=None
, limit=1000
, isMonopartite=True
, apax = 2):
, apax = 1):
'''
Compute the cooccurence matrix and save it, returning NodeNgramNgram.node_id
For the moment list of paramters are not supported because, lists need to
......
analysis/functions.py
View file @ c33bcce3
......@@ -30,7 +30,7 @@ def diag_null(x):
return x - x * scipy.eye(x.shape[0])
def do_distance(cooc_id, field1=None, field2=None, isMonopartite=True, apax=2):
def do_distance(cooc_id, field1=None, field2=None, isMonopartite=True):
'''
do_distance :: Int -> (Graph, Partition, {ids}, {weight})
'''
......@@ -118,7 +118,7 @@ def get_cooc(request=None, corpus=None
, field1='ngrams', field2='ngrams'
, cooc_id=None, type='node_link', size=1000
, start=None, end=None
, apax=2
, apax=1
):
'''
get_ccoc : to compute the graph.
......@@ -291,7 +291,8 @@ def get_graphA( nodeA_type , NodesB , links , corpus ):
# = = = = [ 04. Getting A-elems and making the dictionaries] = = = = ]
sql_query = 'select node_id,value_string from node_node_hyperdata where node_id IN (' + ','.join(map(str, Docs_and_["nodesB"].keys())) + ")"+' and hyperdata_id='+str(nodeA_type_id)
sql_query = 'select node_id,value_string from node_node_hyperdata where node_id IN (' + \
','.join(map(str, Docs_and_["nodesB"].keys())) + ")"+' and hyperdata_id='+str(nodeA_type_id)
cursor = connection.cursor()
cursor.execute(sql_query)
results = cursor.fetchall()
......
ngram/tools.py
View file @ c33bcce3
from gargantext_web.db import Ngram, NodeNgramNgram
from parsing.corpustools import get_cursor, bulk_insert
from gargantext_web.db import get_cursor, bulk_insert
def insert_ngrams(ngrams,get='terms-id'):
......
rest_v1_0/api.py
View file @ c33bcce3
......@@ -259,6 +259,8 @@ class Ngrams(APIView):
],
})
class NodesChildrenDuplicates(APIView):
def _fetch_duplicates(self, request, node_id, extra_columns=None, min_count=1):
......
rest_v1_0/graph.py
View file @ c33bcce3
......@@ -21,7 +21,7 @@ class Graph(APIView):
format_ = request.GET.get('format', 'json')
type_ = request.GET.get('type', 'node_link')
apax = request.GET.get('apax', 2)
apax = request.GET.get('apax', 1)
corpus = session.query(Node).filter(Node.id==corpus_id).first()
......
rest_v1_0/ngrams.py
View file @ c33bcce3
......@@ -11,12 +11,11 @@ from sqlalchemy.orm import aliased
import datetime
import copy
from gargantext_web.views import move_to_trash
from gargantext_web.db import session, Node, NodeNgram, NodeNgramNgram, NodeNodeNgram, Ngram, Hyperdata, Node_Ngram
from gargantext_web.db import get_or_create_node
from gargantext_web.validation import validate, ValidationException
from node import models
from gargantext_web.db import session, Node, NodeNgram, NodeNgramNgram\
, NodeNodeNgram, Ngram, Hyperdata, Node_Ngram, get_or_create_node
def DebugHttpResponse(data):
return HttpResponse('<html><body style="background:#000;color:#FFF"><pre>%s</pre></body></html>' % (str(data), ))
......@@ -71,6 +70,12 @@ class APIException(_APIException):
from rest_framework.decorators import api_view
#@login_required
# TODO how to secure REST ?
class List(APIView):
pass
class Ngrams(APIView):
'''
REST application to manage ngrams
......@@ -212,3 +217,116 @@ class Ngrams(APIView):
],
})
class Group(APIView):
'''
REST API to manage groups of Ngrams
Groups can be synonyms, a cathegory or ngrams groups with stems or lems.
'''
def get_group_id(self , node_id):
node_id = int(node_id)
corpus = session.query(Node).filter(Node.id==node_id).first()
group = get_or_create_node(corpus=corpus, nodetype='Group')
return(group.id)
def get(self, request, corpus_id):
# query ngrams
group_id = self.get_group_id(corpus_id)
#api/node/$corpus_id/ngrams?ngram_id=12
# ngram_id = 1 #request.GET.get('ngram_id', False)
# ngram_id = int(node_id)
# #api/node/$corpus_id/ngrams?all=True
# all_option = request.GET.get('all', False)
# all_option = 1 #int(all_option)
# IMPORTANT: Algorithm for getting the groups:
# 1. pairs_list <- Get all pairs from get_group_id()
# 2. G <- Do a non-directed graph of pairs_list
# 3. DG <- Do a directed graph of pairs_list
# 4. cliques_list <- find_cliques of G
# 5. groups <- Iterate in sinonims_cliques and set the mainNode per each clique: take the highest max_outdegree-node of each clique, using DG
import networkx as nx
G = nx.Graph()
DG = nx.DiGraph()
ngrams_ngrams = (session
.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id==group_id)
)
# ngramy_id=476996, score=1.0, node_id=75081, id=1282846, ngramx_id=493431
for ng in ngrams_ngrams:
# n_x = ( session.query(Ngram).filter(Ngram.id==ng.ngramx_id) ).first()
# n_y = ( session.query(Ngram).filter(Ngram.id==ng.ngramy_id) ).first()
G.add_edge( ng.ngramx_id , ng.ngramy_id )
DG.add_edge( ng.ngramx_id , ng.ngramy_id )
# group = dict(list())
sinonims_cliques = nx.find_cliques( G )
# for nn in ngrams_ngrams.all():
# group[nn.ngramx_id] = group.get(nn.ngramx_id, []) + [nn.ngramy_id]
groups = { "nodes": {} , "links": {} }
for clique in sinonims_cliques:
max_deg = -1
mainNode = -1
mainNode_sinonims = []
for node in clique:
groups["nodes"][node] = "nom_"+str(node)
node_outdeg = DG.out_degree(node)
if node_outdeg>max_deg:
max_deg = node_outdeg
mainNode = node
for node in clique:
if mainNode!=node:
mainNode_sinonims.append( node )
groups["links"][ mainNode ] = mainNode_sinonims
return JsonHttpResponse(groups)
def post(self, request, node_id):
# input validation
input = validate(request.DATA, {'data' : {'source': int, 'target': list}})
group_id = get_group_id(node_id)
for data in input['data']:
if data['source'] > 0 and len(data['target']) > 0:
for target_id in data['target']:
if target_id > 0:
session.add(NodeNgramNgram(node_id=group_id, \
ngramx_id=output['source'], ngramy_id=target_id, score=1))
session.commit()
return JsonHttpResponse(True, 201)
else:
raise APIException('Missing parameter: "{\'data\' : [\'source\': Int, \'target\': [Int]}"', 400)
def delete(self, request, corpus_id):
# input validation
input = validate(request.DATA, {'data' : {'source': int, 'target': list}})
group_id = get_group_id(corpus_id)
for data in input['data']:
if data['source'] > 0 and len(data['target']) > 0:
for target_id in data['target']:
(session.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id==group_id)
.filter(NodeNgramNgram.ngramx_id==data['source'])
.delete()
)
return JsonHttpResponse(True, 201)
else:
raise APIException('Missing parameter: "{\'data\' : [\'source\': Int, \'target\': [Int]}"', 400)
class Keep(APIView):
pass
rest_v1_0/urls.py
View file @ c33bcce3
......@@ -19,8 +19,11 @@ urlpatterns = patterns('',
url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ?
url(r'nodes/(\d+)/children/ngrams$', api.NodesChildrenNgrams.as_view()), # => repeated children ?
url(r'node/(\d+)/ngrams$', ngrams.Ngrams.as_view()),
url(r'node/(\d+)/ngrams$', ngrams.Ngrams.as_view()),
# NGRAMS table & annotations
url(r'node/(\d+)/ngrams$' , ngrams.Ngrams.as_view()),
url(r'node/(\d+)/ngrams/group$', ngrams.Group.as_view()),
url(r'node/(\d+)/ngrams/keep$' , ngrams.Keep.as_view()),
url(r'node/(\d+)/ngrams/list$' , ngrams.List.as_view()),
#url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()),
#url(r'nodes/(\d+)/children/hyperdata$', api.NodesChildrenMetatadata.as_view()),
......
static/js/NGrams_dyna_chart_and_table.js
View file @ c33bcce3
......@@ -209,6 +209,7 @@ function getRecords() {
return MyTable.data('dynatable').settings.dataset.originalRecords;
}
// new
function group_mode ( elem ) {
GState=1
var elem_id = $( elem ).data("stuff")
......@@ -226,6 +227,7 @@ function group_mode ( elem ) {
MyTable.data('dynatable').dom.update();
}
// new
function SaveSinonims( gheader , gcontent) {
console.log("GHEADER:")
$(gheader).children('span').each(function () {
......@@ -237,6 +239,7 @@ function SaveSinonims( gheader , gcontent) {
});
}
// new
$('#group_box_content').bind("DOMSubtreeModified",function(){
console.log( $(this).has( "span" ).length )
var groupdiv = "#group_box"
......@@ -257,6 +260,7 @@ $('#group_box_content').bind("DOMSubtreeModified",function(){
}
})
// new
function add2group ( elem ) {
var elem_id = $( elem ).data("stuff")
......@@ -285,6 +289,7 @@ function add2group ( elem ) {
MyTable.data('dynatable').dom.update();
}
// new
function clickngram_action ( elem ) {
var elem_id = $( elem ).data("stuff")
AjaxRecords[elem_id].state = (AjaxRecords[elem_id].state==(System[0]["states"].length-2))?0:(AjaxRecords[elem_id].state+1);
......@@ -292,6 +297,7 @@ function clickngram_action ( elem ) {
MyTable.data('dynatable').dom.update();
}
// modified
function transformContent(rec_id) {
var elem = AjaxRecords[rec_id];
var result = {}
......@@ -301,7 +307,7 @@ function transformContent(rec_id) {
plus_event = " <a class=\"plusclass\" onclick=\"group_mode(this.parentNode.parentNode)\">(+)</a>"
if(GState==1 ) {
if(elem.state!=1 && elem.state!=3) { // if deleted and already grouped, no Up button
plus_event = " <a class=\"plusclass\" onclick=\"add2group(this.parentNode.parentNode)\">(↑)</a>"
plus_event = " <a class=\"plusclass\" onclick=\"add2group(this.parentNode.parentNode)\">(▲)</a>"
}
}
result["id"] = elem["id"]
......@@ -389,11 +395,6 @@ function Mark_NGram( ngram_id , old_flag , new_flag ) {
return new_flag;
}
function GroupNGrams() {
for (var i in FlagsBuffer["to_group"]){
console.log( AjaxRecords[i] )
}
}
//generic enough
function ulWriter(rowIndex, record, columns, cellWriter) {
......@@ -414,6 +415,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
}
function SelectAll( the_checkbox ) {
console.log(the_checkbox)
var current_flag = $("input[type='radio'][name='radios']:checked").val()
$("tbody tr").each(function (i, row) {
var id = $(row).data('stuff')
......@@ -499,8 +501,6 @@ $("#Save_All").click(function(){
});
function Main_test( data , initial) {
......@@ -738,38 +738,46 @@ function Main_test( data , initial) {
return "OK"
}
function SearchFilters( elem ) {
var MODE = elem.value;
if( MODE == "filter_all") {
var result = Main_test(AjaxRecords , MODE)
console.log( result )
return ;
}
// if( MODE == "filter_stoplist") {
// }
// if( MODE == "filter_miamlist") {
// }
function getIDFromURL( item ) {
var pageurl = window.location.href.split("/")
var cid;
for(var i in pageurl) {
if(pageurl[i]==item) {
cid=parseInt(i);
break;
}
}
return pageurl[cid+1];
}
console.log(window.location.href+"/ngrams.json")
$.ajax({
url: window.location.href+"/ngrams.json",
success: function(data){
// [ = = = = = = = = = = INIT = = = = = = = = = = ]
var corpus_id = getIDFromURL( "corpus" )
var url1=window.location.origin+"/api/node/"+corpus_id+"/ngrams/group",
url2=window.location.href+"/ngrams.json";
var ngrams_groups, ngrams_data;
$.when(
$.ajax({
type: "GET",
url: url1,
dataType: "json",
success : function(data, textStatus, jqXHR) { ngrams_groups = data },
error: function(exception) {
console.log("first ajax, exception!: "+exception.status)
}
}),
$.ajax({
type: "GET",
url: url2,
dataType: "json",
success : function(data, textStatus, jqXHR) { ngrams_data = data },
error: function(exception) {
console.log("second ajax, exception!: "+exception.status)
}
})
).then(function() {
// Building the Score-Selector
var FirstScore = data.scores.initial
var possible_scores = Object.keys( data.ngrams[0].scores );
var FirstScore = ngrams_data.scores.initial
var possible_scores = Object.keys( ngrams_data.ngrams[0].scores );
var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
for( var i in possible_scores ) {
......@@ -778,20 +786,16 @@ $.ajax({
}
}
// Initializing the Charts and Table
var result = Main_test( data , FirstScore )
var result = Main_test( ngrams_data , FirstScore )
console.log( result )
// Listener for onchange Score-Selector
scores_div += "<select>"+"\n";
$("#ScoresBox").html(scores_div)
$("#scores_selector").on('change', function() {
console.log( this.value )
var result = Main_test( data , this.value )
var result = Main_test( ngrams_data , this.value )
console.log( result )
});
}
});
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment