Skip to content

gargantext
gargantext
Commit e85005be authored by delanoe's avatar delanoe
Browse files
Options

[SQLA+SESSIONS] one session for the web interactions and not for each page (for tests).

parent 55136392
Hide whitespace changes
Inline Side-by-side
Showing with 125 additions and 87 deletions
gargantext_web/api2.py
View file @ e85005be
......@@ -13,7 +13,7 @@ import collections
from gargantext_web.views import move_to_trash
from gargantext_web.db import *
from gargantext_web.db import get_session
from gargantext_web.db import session
from gargantext_web.validation import validate, ValidationException
from node import models
......@@ -102,7 +102,6 @@ class NodeNgramsQueries(APIView):
def post(self, request, project_id):
# example only
session = get_session()
input = request.data or {
'x': {
......@@ -260,4 +259,3 @@ class NodeNgramsQueries(APIView):
elif input['format'] == 'csv':
return CsvHttpResponse(sorted(result.items()), ('date', 'value'), 201)
session.remove()
gargantext_web/views.py
View file @ e85005be
......@@ -46,7 +46,7 @@ from gargantext_web import about
from gargantext_web.celery import empty_trash
from gargantext_web.db import *
from gargantext_web.db import get_session, cache, NodeNgram, NodeNgramNgram
from gargantext_web.db import session, cache, NodeNgram, NodeNgramNgram
def login_user(request):
logout(request)
......@@ -220,7 +220,6 @@ def projects(request):
Each project is described with hyperdata that are updateded on each following view.
To each project, we can link a resource that can be an image.
'''
session = get_session()
if not request.user.is_authenticated():
return redirect('/auth/')
......@@ -288,7 +287,6 @@ def projects(request):
'common_projects':common_projects,
'common_users':common_users,
})
session.remove()
def update_nodes(request, project_id, corpus_id, view=None):
'''
......@@ -297,7 +295,6 @@ def update_nodes(request, project_id, corpus_id, view=None):
- permanent deletion of Trash
'''
session = get_session()
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
......@@ -359,11 +356,9 @@ def update_nodes(request, project_id, corpus_id, view=None):
# context_instance=RequestContext(request)
# )
#
session.remove()
def corpus(request, project_id, corpus_id):
session = get_session()
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
......@@ -409,11 +404,9 @@ def corpus(request, project_id, corpus_id):
}))
return HttpResponse(html)
session.remove()
def newpaginatorJSON(request , corpus_id):
session = get_session()
# t = get_template('tests/newpag/thetable.html')
......@@ -468,10 +461,8 @@ def newpaginatorJSON(request , corpus_id):
"totalRecordCount":len(results)
}
return JsonHttpResponse(finaldict)
session.remove()
def move_to_trash(node_id):
session = get_session()
try:
node = session.query(Node).filter(Node.id == node_id).first()
......@@ -491,10 +482,8 @@ def move_to_trash(node_id):
except Exception as error:
print("can not move to trash Node" + str(node_id) + ":" + str(error))
session.remove()
def move_to_trash_multiple(request):
session = get_session()
user = request.user
......@@ -517,11 +506,9 @@ def move_to_trash_multiple(request):
results = ["tudo","fixe"]
return JsonHttpResponse(results)
session.remove()
def delete_node(request, node_id):
session = get_session()
# do we have a valid user?
user = request.user
......@@ -542,7 +529,6 @@ def delete_node(request, node_id):
else:
return HttpResponseRedirect('/projects/')
session.remove()
def delete_corpus(request, project_id, node_id):
# ORM Django
......@@ -564,7 +550,6 @@ def delete_corpus(request, project_id, node_id):
def chart(request, project_id, corpus_id):
''' Charts to compare, filter, count'''
session = get_session()
t = get_template('chart.html')
user = request.user
......@@ -581,10 +566,8 @@ def chart(request, project_id, corpus_id):
'corpus' : corpus,
}))
return HttpResponse(html)
session.remove()
def sankey(request, corpus_id):
session = get_session()
t = get_template('sankey.html')
user = request.user
......@@ -600,11 +583,9 @@ def sankey(request, corpus_id):
}))
return HttpResponse(html)
session.remove()
def matrix(request, project_id, corpus_id):
session = get_session()
t = get_template('matrix.html')
user = request.user
date = datetime.datetime.now()
......@@ -621,10 +602,8 @@ def matrix(request, project_id, corpus_id):
}))
return HttpResponse(html)
session.remove()
def graph(request, project_id, corpus_id, generic=100, specific=100):
session = get_session()
t = get_template('explorer.html')
user = request.user
......@@ -654,7 +633,6 @@ def graph(request, project_id, corpus_id, generic=100, specific=100):
}))
return HttpResponse(html)
session.remove()
def exploration(request):
t = get_template('exploration.html')
......@@ -689,7 +667,6 @@ def corpus_csv(request, project_id, corpus_id):
'''
Create the HttpResponse object with the appropriate CSV header.
'''
session = get_session()
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="corpus.csv"'
......@@ -718,7 +695,6 @@ def corpus_csv(request, project_id, corpus_id):
return response
session.remove()
def send_csv(request, corpus_id):
'''
......@@ -767,34 +743,49 @@ def node_link(request, corpus_id):
'''
Create the HttpResponse object with the node_link dataset.
'''
session = get_session()
data = []
corpus = session.query(Node).filter(Node.id==corpus_id).first()
data = get_cooc(request=request, corpus=corpus, type="node_link")
return JsonHttpResponse(data)
session.remove()
from analysis.periods import phylo_clusters
def sankey_csv(request, corpus_id):
session = get_session()
data = []
corpus = session.query(Node).filter(Node.id==corpus_id).first()
#
# header = ["source", "target", "value"]
# data.append(header)
#
# flows = phylo_clusters(corpus, range(2005,2013))
# for flow in flows:
# data.append(flow)
# print(data)
#
data = [
["source", "target", "value"]
, ["Comment_1", "Theme_1", 1]
, ["Comment_2", "Theme_2", 2]
, ["Comment_3", "Theme_2", 2]
, ["Comment_7", "Theme_1", 2]
, ["Comment_8", "Theme_3", 2]
, ["Theme_1", "Reco_par_1", 2]
, ["Theme_2", "Reco_par_2", 2]
, ["Theme_2", "Reco_par_5", 2]
, ["Theme_3", "Reco_par_5", 1]
['source', 'target', 'value']
, ['inégalités,rapports sociaux,P1', 'critique,travail social,P2', 8]
, ['inégalités,rapports sociaux,P1', 'inégalités,éducation,P2', 21]
, ['éducation,institutions,P1', 'critique,travail social,P2', 7]
, ['éducation,institutions,P1', 'inégalités,éducation,P2', 10]
#, ['éducation,institutions,P1', 'personnes âgées,pouvoirs publics,P2', 8]
, ['éducation,institutions,P1', 'politiques publiques,personnes âgées dépendantes,P2', 8]
#, ['éducation,institutions,P1', 'intervention sociale,travailleur social,P2', 8]
#, ['intervention sociale,travailleur social,2011-01-01 2013-12-31', 'intervention sociale,travailleur social,P3', 0]
, ['critique,enseignement supérieur,P1', 'critique,travail social,P2', 6]
#, ['critique,enseignement supérieur,P1', 'personnes âgées,pouvoirs publics,P2', 7]
, ['justice,exclusion,violence,P1', 'inégalités,éducation,P2', 12]
, ['critique,travail social,P2', 'justice,travail social,P3', 14]
, ['inégalités,éducation,P2', 'justice,travail social,P3', 20]
, ['inégalités,éducation,P2', 'justice sociale,éducation,P3', 8]
, ['inégalités,éducation,P2', 'action publique,institutions,P3', 9]
, ['inégalités,éducation,P2', 'inégalités,inégalités sociales,P3', 18]
, ['politiques publiques,personnes âgées dépendantes,P2', 'justice sociale,éducation,P3', 20]
]
return(CsvHttpResponse(data))
session.remove()
def adjacency(request, corpus_id):
'''
......@@ -853,7 +844,6 @@ def ngrams(request):
}))
return HttpResponse(html)
def nodeinfo(request , node_id):
'''Structure of the popUp for topPapers div '''
t = get_template('node-info.html')
......
gargantext_web/views_optimized.py
View file @ e85005be
......@@ -15,7 +15,7 @@ from threading import Thread
from node.admin import CustomForm
from gargantext_web.db import *
from gargantext_web.db import get_or_create_node, get_session
from gargantext_web.db import get_or_create_node, session
from gargantext_web.settings import DEBUG, MEDIA_ROOT
from rest_v1_0.api import JsonHttpResponse
from django.db import connection
......@@ -39,7 +39,6 @@ def project(request, project_id):
raise Http404()
# do we have a valid project?
session = get_session()
project = (session
.query(Node)
......@@ -146,7 +145,6 @@ def project(request, project_id):
language_id = None,
hyperdata = {'Processing' : "Parsing documents",}
)
session = get_session()
session.add(corpus)
session.commit()
......@@ -199,7 +197,6 @@ def project(request, project_id):
'number' : corpora_count,
})
session.remove()
def tfidf(request, corpus_id, ngram_ids):
"""Takes IDs of corpus and ngram and returns list of relevent documents in json format
......@@ -211,7 +208,6 @@ def tfidf(request, corpus_id, ngram_ids):
ngram_ids = ngram_ids.split('a')
ngram_ids = [int(i) for i in ngram_ids]
session = get_session()
corpus = session.query(Node).filter(Node.id==corpus_id).first()
tfidf_id = get_or_create_node(corpus=corpus, nodetype='Tfidf').id
......@@ -256,61 +252,116 @@ def tfidf(request, corpus_id, ngram_ids):
nodes_list.append(node_dict)
return JsonHttpResponse(nodes_list)
session.remove()
def getCorpusIntersection(request , corpuses_ids):
session = get_session()
FinalDict = False
if request.method == 'POST' and "nodeids" in request.POST and len(request.POST["nodeids"])>0 :
if request.method == 'POST' and "nodeids" in request.POST and len(request.POST["nodeids"])>0:
import ast
import networkx as nx
node_ids = [int(i) for i in (ast.literal_eval( request.POST["nodeids"] )) ]
# Here are the visible nodes of the initial semantic map.
corpuses_ids = corpuses_ids.split('a')
corpuses_ids = [int(i) for i in corpuses_ids] # corpus[1] will be the corpus to compare
cooc_type_id = cache.NodeType['Cooccurrence'].id
cooc_ids = session.query(Node.id).filter(Node.user_id == request.user.id , Node.parent_id==corpuses_ids[1] , Node.type_id == cooc_type_id ).first()
if len(cooc_ids)==0:
return JsonHttpResponse(FinalDict)
# If corpus[1] has a coocurrence.id then lets continue
corpuses_ids = [int(i) for i in corpuses_ids]
print(corpuses_ids)
# corpus[1] will be the corpus to compare
Coocs = {}
import networkx as nx
G = nx.Graph() # I use an undirected graph, because direction doesnt matter here, coocs should be a triangular matrix, so...
ngrams_data1 = session.query(NodeNgramNgram).filter( NodeNgramNgram.node_id==cooc_ids[0], NodeNgramNgram.ngramx_id.in_( node_ids )).all()
for ngram in ngrams_data1: # are there visible nodes in the X-axis of corpus to compare ?
G.add_edge( ngram.ngramx_id , ngram.ngramy_id , weight=ngram.score)
ngrams_data2 = session.query(NodeNgramNgram).filter( NodeNgramNgram.node_id==cooc_ids[0], NodeNgramNgram.ngramy_id.in_( node_ids )).all()
for ngram in ngrams_data2: # are there visible nodes in the Y-axis of corpus to compare ?
if not G.has_edge(ngram.ngramx_id,ngram.ngramy_id):
def get_score(corpus_id):
cooc_type_id = cache.NodeType['Cooccurrence'].id
cooc_ids = (session.query(Node.id)
.filter(Node.user_id == request.user.id
, Node.parent_id==corpus_id
, Node.type_id == cooc_type_id )
.first()
)
if len(cooc_ids)==0:
return JsonHttpResponse(FinalDict)
# If corpus[1] has a coocurrence.id then lets continue
Coocs = {}
G = nx.Graph()
# undirected graph only
# because direction doesnt matter here
# coocs is triangular matrix
ngrams_data = ( session.query(NodeNgramNgram)
.filter( NodeNgramNgram.node_id==cooc_ids[0]
, or_(
NodeNgramNgram.ngramx_id.in_( node_ids )
, NodeNgramNgram.ngramy_id.in_( node_ids )
)
)
.group_by(NodeNgramNgram)
.all()
)
for ngram in ngrams_data :
# are there visible nodes in the X-axis of corpus to compare ?
G.add_edge( ngram.ngramx_id , ngram.ngramy_id , weight=ngram.score)
for e in G.edges_iter():
n1 = e[0]
n2 = e[1]
# print( G[n1][n2]["weight"] , "\t", n1,",",n2 )
if n1 not in Coocs:
Coocs[n1]=0
if n2 not in Coocs:
Coocs[n2]=0
Coocs[n1]+=G[n1][n2]["weight"]
Coocs[n2]+=G[n1][n2]["weight"]
print(corpus_id, ngram)
for e in G.edges_iter() :
n1 = e[0]
n2 = e[1]
# print( G[n1][n2]["weight"] , "\t", n1,",",n2 )
if n1 not in Coocs :
Coocs[n1] = 0
if n2 not in Coocs :
Coocs[n2] = 0
Coocs[n1] += G[n1][n2]["weight"]
Coocs[n2] += G[n1][n2]["weight"]
return(Coocs,G)
Coocs_0,G_0 = get_score( corpuses_ids[0] )
Coocs_1,G_1 = get_score( corpuses_ids[1] )
FinalDict = {}
for node in node_ids:
if node in Coocs:
FinalDict[node] = Coocs[node]/G.degree(node)
# Getting AVG-COOC of each ngram that exists in the cooc-matrix of the compared-corpus.
measure = 'cooc'
if measure == 'jacquard':
for node in node_ids :
if node in G_1.nodes() and node in G_0.nodes():
neighbors_0 = set(G_0.neighbors(node))
neighbors_1 = set(G_1.neighbors(node))
jacquard = len(neighbors_0.intersection(neighbors_1)) / len(neighbors_0.union(neighbors_1))
FinalDict[node] = jacquard * 3
elif node in G_0.nodes() and node not in G_1.nodes() :
FinalDict[node] = 2
elif node not in G_0.nodes() and node in G_1.nodes() :
FinalDict[node] = 1
else:
FinalDict[node] = 0
elif measure == 'cooc':
for node in node_ids :
if node in G_1.nodes() and node in G_0.nodes():
score_0 = Coocs_0[node] / G_0.degree(node)
score_1 = Coocs_1[node] / G_1.degree(node)
FinalDict[node] = 5 * score_0 / score_1
elif node in G_0.nodes() and node not in G_1.nodes() :
FinalDict[node] = 0.5
elif node not in G_0.nodes() and node in G_1.nodes() :
FinalDict[node] = 0.2
else:
FinalDict[node] = 0
print(FinalDict)
#print(node,score)
# Getting AVG-COOC of each ngram that exists in the cooc-matrix of the compared-corpus.
return JsonHttpResponse(FinalDict)
session.remove()
def getUserPortfolio(request , project_id):
session = get_session()
user = request.user
user_id = cache.User[request.user.username].id
project_type_id = cache.NodeType['Project'].id
......@@ -354,4 +405,3 @@ def getUserPortfolio(request , project_id):
return JsonHttpResponse( results )
session.remove()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment