Skip to content

gargantext
gargantext
Commit 024733e2 authored by Romain Loth's avatar Romain Loth
Browse files
Options

Merge branch 'refactoring' into romain-refactoring

parents f920e0c3 2e480551
Show whitespace changes
Inline Side-by-side
Showing with 343 additions and 217 deletions
gargantext/models/nodes.py
View file @ 024733e2
......@@ -7,8 +7,7 @@ from datetime import datetime
from .users import User
__all__ = ['Node']
__all__ = ['Node', 'NodeNode']
class NodeType(TypeDecorator):
"""Define a new type of column to describe a Node's type.
......@@ -174,3 +173,10 @@ class Node(Base):
{'action':action, 'progress':progress, 'complete':complete, 'error':error, 'date':date}
))
return self['statuses'][-1]
class NodeNode(Base):
__tablename__ = 'nodes_nodes'
id = Column(Integer, primary_key=True)
node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
score = Column(Float(precision=24))
gargantext/urls.py
View file @ 024733e2
......@@ -9,8 +9,9 @@ Views are shared between these modules:
"""
from django.conf.urls import include, url
from django.contrib import admin
from django.views.generic.base import RedirectView as Redirect
from django.contrib.staticfiles.storage import staticfiles_storage as static
import gargantext.views.api.urls
import gargantext.views.pages.urls
......@@ -22,28 +23,26 @@ from annotations.views import main as annotations_main_view
# Module "Graph Explorer"
#from graphExplorer import urls as graphExplorer_urls
from graphExplorer.rest import Graph
from graphExplorer.views import explorer
import graphExplorer.urls
# Module Scrapers
from scrapers import urls as scrapers_urls
import scrapers.urls
urlpatterns = [ url(r'^admin/' , admin.site.urls )
, url(r'^api/' , include( gargantext.views.api.urls ) )
, url(r'^' , include( gargantext.views.pages.urls ) )
, url(r'^favicon.ico$', Redirect.as_view( url=static.url('favicon.ico')
, permanent=False), name="favicon")
# Module "Graph Explorer"
, url(r'^' , include( graphExplorer.urls ) )
# Module Annotation
# tempo: unchanged doc-annotations routes --
, url(r'^annotations/', include( annotations_urls ) )
, url(r'^projects/(\d+)/corpora/(\d+)/documents/(\d+)/$', annotations_main_view)
# Module "Graph Explorer"
, url(r'^projects/(\d+)/corpora/(\d+)/explorer$', explorer )
, url(r'^projects/(\d+)/corpora/(\d+)/graph$' , Graph.as_view())
# to be removed:
, url(r'^projects/(\d+)/corpora/(\d+)/node_link.json$', Graph.as_view())
#url(r'^projects/(\d+)/corpora/(\d+)/explorer$', include(graphExplorer.urls))
# Scrapers module
, url(r'^scrapers/' , include( scrapers_urls ) )
# Module Scrapers
, url(r'^scrapers/' , include( scrapers.urls ) )
]
graphExplorer/distances.py
View file @ 024733e2
......@@ -60,8 +60,8 @@ def clusterByDistances( cooc_id
# top generic or specific
m = ( xs - ys) / (2 * (x.shape[0] - 1))
n = n.sort(inplace=False)
m = m.sort(inplace=False)
n = n.sort_index(inplace=False)
m = m.sort_index(inplace=False)
nodes_included = 500 #int(round(size/20,0))
#nodes_excluded = int(round(size/10,0))
......
graphExplorer/intersection.py 0 → 100644
View file @ 024733e2
from gargantext.models import Node, Ngram, NodeNgram, NodeNgramNgram, \
HyperdataKey
from gargantext.util.db import session, aliased, bulk_insert, func
from gargantext.util.lists import WeightedMatrix, UnweightedList, Translations
from gargantext.util.http import JsonHttpResponse
from sqlalchemy import desc, asc, or_, and_
import datetime
def intersection(request , corpuses_ids, measure='cooc'):
FinalDict = False
if request.method == 'POST' and "nodeids" in request.POST and len(request.POST["nodeids"])>0 :
import ast
import networkx as nx
node_ids = [int(i) for i in (ast.literal_eval( request.POST["nodeids"] )) ]
# Here are the visible nodes of the initial semantic map.
corpuses_ids = corpuses_ids.split('a')
corpuses_ids = [int(i) for i in corpuses_ids]
print(corpuses_ids)
# corpus[1] will be the corpus to compare
def get_score(corpus_id):
cooc_ids = (session.query(Node.id)
.filter(Node.user_id == request.user.id
, Node.parent_id==corpus_id
, Node.typename == 'COOCCURRENCES' )
.first()
)
if len(cooc_ids)==0:
return JsonHttpResponse(FinalDict)
# If corpus[1] has a coocurrence.id then lets continue
Coocs = {}
G = nx.Graph()
# undirected graph only
# because direction doesnt matter here
# coocs is triangular matrix
ngrams_data = ( session.query(NodeNgramNgram)
.filter( NodeNgramNgram.node_id==cooc_ids[0]
, or_( NodeNgramNgram.ngram1_id.in_( node_ids )
, NodeNgramNgram.ngram2_id.in_( node_ids )
)
)
.group_by(NodeNgramNgram)
.all()
)
for ngram in ngrams_data :
# are there visible nodes in the X-axis of corpus to compare ?
G.add_edge( ngram.ngram1_id , ngram.ngram2_id , weight=ngram.weight)
print(corpus_id, ngram)
for e in G.edges_iter() :
n1 = e[0]
n2 = e[1]
# print( G[n1][n2]["weight"] , "\t", n1,",",n2 )
if n1 not in Coocs :
Coocs[n1] = 0
if n2 not in Coocs :
Coocs[n2] = 0
Coocs[n1] += G[n1][n2]["weight"]
Coocs[n2] += G[n1][n2]["weight"]
return(Coocs,G)
Coocs_0,G_0 = get_score( corpuses_ids[0] )
Coocs_1,G_1 = get_score( corpuses_ids[1] )
FinalDict = {}
if measure == 'jacquard':
for node in node_ids :
if node in G_1.nodes() and node in G_0.nodes():
neighbors_0 = set(G_0.neighbors(node))
neighbors_1 = set(G_1.neighbors(node))
jacquard = len(neighbors_0.intersection(neighbors_1)) / len(neighbors_0.union(neighbors_1))
FinalDict[node] = jacquard * 3
elif node in G_0.nodes() and node not in G_1.nodes() :
FinalDict[node] = 2
elif node not in G_0.nodes() and node in G_1.nodes() :
FinalDict[node] = 1
else:
FinalDict[node] = 0
elif measure == 'degree':
for node in node_ids :
if node in G_1.nodes() and node in G_0.nodes():
score_0 = Coocs_0[node] / G_0.degree(node)
score_1 = Coocs_1[node] / G_1.degree(node)
FinalDict[node] = 5 * score_0 / score_1
elif node in G_0.nodes() and node not in G_1.nodes() :
FinalDict[node] = 0.5
elif node not in G_0.nodes() and node in G_1.nodes() :
FinalDict[node] = 0.2
else:
FinalDict[node] = 0
elif measure == 'cooc':
for node in node_ids :
if node in G_1.nodes() and node in G_0.nodes():
#FinalDict[node] = Coocs_1[node] / Coocs_0[node]
FinalDict[node] = Coocs_0[node] / Coocs_1[node]
elif node in G_0.nodes() and node not in G_1.nodes() :
FinalDict[node] = 0.0
elif node not in G_0.nodes() and node in G_1.nodes() :
FinalDict[node] = 0.0
else:
FinalDict[node] = 0
print(FinalDict)
#print(node,score)
# Getting AVG-COOC of each ngram that exists in the cooc-matrix of the compared-corpus.
return JsonHttpResponse(FinalDict)
graphExplorer/urls.py
View file @ 024733e2
from django.conf.urls import patterns, url
from graphExplorer import views
# /!\ urls patterns here are *without* the trailing slash
# Module "Graph Explorer"
from graphExplorer.rest import Graph
from graphExplorer.views import explorer
from graphExplorer.intersection import intersection
urlpatterns = patterns('',
url(r'^register/$', views.Register.as_view()), # Register
url(r'^login/$', views.Login.as_view()), # Login
)
# TODO : factor urls
# url will have this pattern:
# ^explorer/$corpus_id/view
# ^explorer/$corpus_id/data.json
# ^explorer/$corpus_id/intersection
urlpatterns = [ url(r'^explorer/intersection/(\w+)$', intersection )
, url(r'^projects/(\d+)/corpora/(\d+)/explorer$', explorer )
, url(r'^projects/(\d+)/corpora/(\d+)/graph$' , Graph.as_view())
, url(r'^projects/(\d+)/corpora/(\d+)/node_link.json$', Graph.as_view())
]
install/postgres/indexes.sql
View file @ 024733e2
......@@ -26,9 +26,11 @@
-- create INDEX on nodes_ngrams_ngrams (node_id, ngram1_id, ngram2_id) ;
----------------------------------------------------------------------
-- DELETE optimization of Nodes
create INDEX on nodes_nodes_ngrams (node1_id);
create INDEX on nodes_nodes_ngrams (node2_id);
-- DELETE optimization of Nodes -- todo on dev
-- create INDEX on nodes_nodes_ngrams (node1_id);
-- create INDEX on nodes_nodes_ngrams (node2_id);
create INDEX on nodes_nodes (node1_id, node2_id);
-- Maybe needed soon:
-- create INDEX on nodes_nodes_ngrams (node1_id, node2_id);
......
static/js/graphExplorer/extras_explorerjs.js
View file @ 024733e2
......@@ -671,7 +671,7 @@ function getCookie(name) {
// Just for Garg
function printCorpuses() {
console.clear()
console.log( "!!!!!!!! in printCorpuses() !!!!!!!! " )
console.log( "!!!!!!!! Corpus chosen, going to make the diff !!!!!!!! " )
pr(corpusesList)
var selected = $('input[name=optradio]:checked')[0].id.split("_")
......@@ -680,18 +680,18 @@ function printCorpuses() {
var pageurl = window.location.href.split("/")
var cid;
for(var i in pageurl) {
if(pageurl[i]=="corpus") {
if(pageurl[i]=="corpora") {
cid=parseInt(i);
break;
}
}
var current_corpus = pageurl[cid+1];
pr("corpus id, selected: "+corpusesList[sel_p]["corpuses"][sel_c]["id"])
pr("corpus id, selected: "+sel_c)
pr("current corpus: "+current_corpus)
var the_ids = []
the_ids.push( current_corpus )
the_ids.push( corpusesList[sel_p]["corpuses"][sel_c]["id"] )
the_ids.push( sel_c )
$("#closecorpuses").click();
......@@ -702,7 +702,7 @@ function printCorpuses() {
console.log( thenodes )
$.ajax({
type: 'GET',
url: window.location.origin+'/api/corpusintersection/'+the_ids.join("a"),
url: window.location.origin+'/explorer/intersection/'+the_ids.join("a"),
data: "nodeids="+JSON.stringify(thenodes),
type: 'POST',
beforeSend: function(xhr) {
......@@ -720,7 +720,7 @@ function printCorpuses() {
cancelSelection(false)
ChangeGraphAppearanceByAtt(true)
console.log("YOLOYOLYOLYOYKOYYKYOY")
console.log("Getting the clusters")
clustersBy("inter" , "color")
clustersBy("inter" , "size")
......@@ -814,44 +814,68 @@ function GetUserPortfolio() {
if( Object.keys( corpusesList ).length > 0 )
return true;
var query_url = window.location.origin+'/api/userportfolio/project/'+project_id+'/corpuses'
var query_url = window.location.origin+'/api/nodes?types[]=PROJECT&types[]=CORPUS&pagination_limit=100'
$.ajax({
type: 'GET',
dataType : 'JSON',
url: query_url,
success : function(data) {
var html_ = ""
var portfolio = {}
html_ += '<div class="panel-group" id="accordion">'+"\n"
html_ += ' <form id="corpuses_form" role="form">'+"\n"
corpusesList = data;
for (var k1 in data) {
var v1 = data[k1]
for (var record in data["records"]) {
console.log( " ici le record " + record )
if ( data["records"][record]["typename"] === 'PROJECT' ) {
var project_id = data["records"][record]["id"]
var project_name = data["records"][record]["name"]
portfolio[project_id] = project_name
html_ += ' <div class="panel panel-default">'+"\n"
html_ += ' <div class="panel-heading">'+"\n"
html_ += ' <h4 class="panel-title">'+"\n"
html_ += ' <a data-toggle="collapse" data-parent="#accordion" href="#collapse_'+k1+'">'+v1["proj_name"]+'</a>'+"\n"
html_ += ' <a data-toggle="collapse" data-parent="#accordion" href="#collapse_' + project_id+'">'
html_ += ' <span class="glyphicon glyphicon-book" aria-hidden="true"></span> ' + project_name
html_ += ' </a>'+"\n"
html_ += ' </h4>'+"\n"
html_ += ' </div>'+"\n"
html_ += ' <div id="collapse_'+k1+'" class="panel-collapse collapse">'+"\n"
html_ += ' <div id="collapse_'+project_id+'" class="panel-collapse collapse">'+"\n"
html_ += ' <div class="panel-body" style="input[type=radio] {display: none;}">'+"\n"
html_ += ' <ul>'+"\n"
for(var c in v1["corpuses"]) {
var Ci = v1["corpuses"][c]
if( Ci["id"]!= corpus_id) {
html_ += ' <li>'+"\n"
for (var record2 in data["records"]) {
if ( data["records"][record2]["typename"] == 'CORPUS' ) {
var corpus_parent_id = data["records"][record2]["parent_id"]
if (corpus_parent_id !== null) {
if ( corpus_parent_id == project_id) {
var corpus_id = data["records"][record2]["id"]
var corpus_name = data["records"][record2]["name"]
portfolio[corpus_id] = corpus_name
html_ += ' <div class="row">'+"\n"
html_ += ' <div class="radio">'+"\n"
html_ += ' <label><input type="radio" id="'+k1+"_"+c+'" name="optradio">'+"\n"
html_ += ' <a target="_blank" href="/project/'+k1+'/corpus/'+Ci["id"]+'/">'+Ci["name"] +' ('+Ci["c"]+' docs.)</a>'+"\n"
html_ += ' <label><input type="radio" id="'+project_id+"_"+corpus_id+'" name="optradio">'+"\n"
html_ += ' <a target="_blank" href="/projects/'+project_id+'/corpora/'+corpus_id+'/">'
html_ += ' <span class="glyphicon glyphicon-file" aria-hidden="true"></span> ' + corpus_name +'</a>'+"\n"
html_ += ' </label>'+"\n"
html_ += ' </div>'+"\n"
html_ += ' </li>'+"\n"
html_ += ' </div>'+"\n"
}
}
}
}
html_ += ' </ul>'+"\n"
html_ += ' </div>'+"\n"
html_ += ' </div>'+"\n"
html_ += ' </div>'+"\n"
}
}
html_ += ' </form>'+"\n"
html_ += '</div>'+"\n"
......@@ -860,8 +884,17 @@ function GetUserPortfolio() {
$('#corpuses_form input:radio').change(function() {
$("#add_corpus_tab").prop("disabled",false)
var selected = $('input[name=optradio]:checked')[0].id.split("_")
var sel_p = selected[0], sel_c=selected[1]
$("#selected_corpus").html( "<center>"+data[sel_p]["proj_name"] + " , " + data[sel_p]["corpuses"][sel_c]["name"]+"</center><br>" )
var sel_p_id = selected[0], sel_c_id =selected[1]
var html_selection = ""
html_selection += '<center>You are comparing :<br><span class="glyphicon glyphicon-hand-down" aria-hidden="true"></span></center>'+"\n"
html_selection += '<center>'
html_selection += '( current graph ) '
html_selection += '<span class="glyphicon glyphicon-resize-horizontal" aria-hidden="true"></span>'
html_selection += ' (' + portfolio[sel_p_id] + ' / ' + portfolio[sel_c_id] + ')'
// html_selection += ' (' + portfolio[sel_p_id] + '/' + sel_c_id + portfolio[sel_c_id] + ')'
html_selection += '</center><br>'+"\n"
$("#selected_corpus").html( html_selection )
});
......
templates/graphExplorer/explorer.html
View file @ 024733e2
<html>
<head>
{% load staticfiles %}
<link rel="stylesheet" href="{% static "js/bootstrap/3.0.2/bootstrap.css" %}">
<link rel="stylesheet" href="{% static "js/bootstrap/3.1.1/bootstrap-theme.min.css" %}">
{% extends "pages/menu.html" %}
{% load staticfiles %}
{% block css %}
<link rel="stylesheet" href="{% static "js/jquery/1.11.2/jquery-ui.css" %}" media="screen">
<link rel="stylesheet" href="{% static "js/libs/css2/freshslider.css" %}" media="screen">
<link rel="stylesheet" href="{% static "js/libs/css2/custom.css" %}" media="screen">
<link rel="stylesheet" href="{% static "js/libs/css2/sidebar.css" %}" media="screen">
<link rel="stylesheet" type="text/css" href="{% static "js/gargantext/menu.css"%}"/>
<style>
#leftcolumn {
......@@ -42,80 +43,11 @@
-->
</head>
{% endblock %}
<body>
<div id="graphsfixedtop" class="navbar navbar-inverse" role="navigation">
<div class="container">
<div class="navbar-inner">
<button class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" style="line-height:15px; height:10px; padding: 10px 10px;" href="/"><img src="{% static "img/logoSmall.png"%}" title="Back to home."></a>
</div>
<div class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li><a href="/about/" title="More informations about the project, its sponsors and its authors.">
<span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span>
About
</a>
</li>
{% if user.is_authenticated %}
<li><a href="/projects/" title="All your projects are here.">
<span class="glyphicon glyphicon-home" aria-hidden="true"></span>
Projects
</a></li>
{% endif %}
{% if project %}
<li><a href="/projects/{{project.id}}">
<span class="glyphicon glyphicon-book" aria-hidden="true"></span>
{{project.name | truncatechars:15}}</a></li>
{% endif %}
{% if corpus %}
<li><a href="/projects/{{project.id}}/corpora/{{corpus.id}}">
<span class="glyphicon glyphicon-file" aria-hidden="true"></span>
{{corpus.name | truncatechars:15}}
</a></li>
{% endif %}
</ul>
<ul class="nav pull-right">
<li class="dropdown">
<a href="#" role="button" class="dropdown-toggle navbar-text" data-toggle="dropdown" title="That is your username">
<i class="icon-user"></i>
<span class="glyphicon glyphicon-user" aria-hidden="true" style="color:white"></span>
{{ user.username | truncatechars:15}}
<i class="caret"></i>
</a>
<ul class="dropdown-menu">
<li><a tabindex="-1" href="http://www.iscpif.fr/tiki-index.php?page=gargantext_feedback" title="Send us a message (bug, thanks, congrats...)">
<span class="glyphicon glyphicon-bullhorn" aria-hidden="true"></span>
Report Feedback</a></li>
<li class="divider"></li>
{% if user.is_authenticated %}
<li>
<a tabindex="-1" href="/auth/logout"
title="Click here to logout especially on public devices">
<span class="glyphicon glyphicon-log-out" aria-hidden="true"></span>
Logout
</a></li>
{% else %}
<li><a tabindex="-1" href="/auth/login">Login</a></li>
{% endif %}
</ul>
</li>
</ul>
</div>
</div>
</div>
{% block content %}
<!-- this is the tweakbar -->
<div id="defaultop" class="navbar navbar-default">
......@@ -441,13 +373,19 @@
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
<h3 class="modal-title">Corpus Comparison Tool</h3>
<h3 class="modal-title">
<span class="glyphicon glyphicon-transfer" aria-hidden="true"></span>
Corpus Comparison Tool
</h3>
</div>
<div class="modal-body form-horizontal">
<h4>Choose one corpus:</h4>
<h4>
<span class="glyphicon glyphicon-hand-right" aria-hidden="true"></span>
Choose one corpus among your projects:
</h4>
<div style="color:red;" id="selected_corpus"></div>
<div id="user_portfolio">
......@@ -455,8 +393,16 @@
<div class="modal-footer">
<button id="closecorpuses" type="button" class="btn btn-default" data-dismiss="modal">Close</button>
<button id="add_corpus_tab" type="button" class="btn btn-primary" disabled onclick='printCorpuses();'>Add Tab</button>
<button id="closecorpuses" type="button" class="btn btn-default" data-dismiss="modal">
<span class="glyphicon glyphicon-remove" aria-hidden="true" ></span>
Close
</button>
<button id="add_corpus_tab" type="button" class="btn btn-primary" disabled onclick='printCorpuses();'>
<span class="glyphicon glyphicon-ok" aria-hidden="true" ></span>
Compare
</button>
</div>
</div>
......@@ -483,7 +429,6 @@
<script type="text/javascript" src="{% static "js/libs/jquery/jquery.easytabs.min.js" %}"></script>
<script src="{% static "js/libs/bootstrap/js/bootstrap.min.js" %}"></script>
<script src="{% static "js/libs/bootstrap/js/bootstrap-modal.js" %}" type="text/javascript"></script>
<script src="{% static "js/libs/bootstrap/js/bootstrap-hover-dropdown.min.js" %}" type="text/javascript"></script>
......@@ -512,12 +457,8 @@
// $('#tab-container-top').easytabs({updateHash:false});
</script>
</body>
{% endblock %}
</html>
templates/pages/menu.html
View file @ 024733e2
......@@ -132,8 +132,8 @@
<!--/.nav-collapse -->
{% block corpusBannerTop %}
{% if corpus %}
{% if view != "graph" %}
<div class="container theme-showcase">
<div class="jumbotron" style="margin-bottom:0">
<br>
......@@ -183,9 +183,13 @@
</div>
</div>
</div>
{% else %}
<div class="container theme-showcase">
<div class="jumbotron" style="margin-bottom:0">
</div>
</div>
{% endif %}
{% endif %}
{% endblock %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment