Skip to content

gargantext
gargantext
Commit 25584756 authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into unstable-mat

parents d6cc0e32 225cb308
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 830 additions and 614 deletions
analysis/functions.py
View file @ 25584756
......@@ -21,7 +21,10 @@ def create_synonymes(user, corpus):
size = 1000
def create_whitelist(user, corpus_id, size=size, count_min=2):
def create_whitelist(user, corpus_id, size=size, count_min=2, miam_id=None):
if miam_id is None:
PrintException()
cursor = connection.cursor()
whitelist_type_id = cache.NodeType['WhiteList'].id
......@@ -53,10 +56,14 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
node_node_ngram AS nngX ON nngX.node_id = n.id
INNER JOIN
node_ngram AS ngX ON ngX.id = nngX.ngram_id
INNER JOIN
node_node_ngram AS miam ON ngX.id = miam.ngram_id
WHERE
n.parent_id = %d
AND
n.type_id = %d
AND
miam.node_id = %d
AND
ngX.n >= 2
AND
......@@ -72,7 +79,7 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
LIMIT
%d
;
""" % (white_list.id, int(corpus_id), int(type_document_id), count_min, size)
""" % (white_list.id, int(corpus_id), int(type_document_id), int(miam_id), count_min, size)
# print("PRINTING QYERY OF WHITELIST:")
# print(query_whitelist)
cursor.execute(query_whitelist)
......@@ -155,6 +162,7 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
from gargantext_web.api import JsonHttpResponse
from analysis.louvain import best_partition
from ngram.lists import listIds
#print(corpus_id, cooc_id)
......@@ -168,7 +176,9 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
if session.query(Node).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first() is None:
print("Coocurrences do not exist yet, create it.")
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size)
miam_id = listIds(typeList='MiamList', user_id=request.user.id, corpus_id=corpus_id)[0][0]
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size, miam_id=miam_id)
cooccurrence_node_id = create_cooc(user=request.user, corpus_id=corpus_id, whitelist=whitelist, size=size)
else:
cooccurrence_node_id = session.query(Node.id).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first()
......@@ -211,8 +221,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
n = n.sort(inplace=False)
m = m.sort(inplace=False)
print(n)
print(m)
#print(n)
#print(m)
nodes_included = 300 #int(round(size/20,0))
#nodes_excluded = int(round(size/10,0))
......@@ -237,25 +247,25 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
xxx = xx.values
threshold = min(xxx.max(axis=1))
matrix_filtered = np.where(xxx >= threshold, xxx, 0)
#matrix_filtered = matrix_filtered.resize((90,90))
except:
PrintException()
try:
G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph())
G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
G = nx.from_numpy_matrix(np.matrix(matrix_filtered))
#G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph())
#print(G)
G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
# Removing too connected nodes (find automatic way to do it)
#edges_to_remove = [ e for e in G.edges_iter() if
degree = G.degree()
nodes_to_remove = [n for n in degree if degree[n] <= 1]
G.remove_nodes_from(nodes_to_remove)
partition = best_partition(G)
uG = G.to_undirected()
partition = best_partition(uG)
except:
print("-" * 30)
PrintException()
......@@ -271,7 +281,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
G.node[node]['group'] = partition[node]
# G.add_edge(node, "cluster " + str(partition[node]), weight=3)
except Exception as error:
print("error01: ",error)
pass#PrintException()
#print("error01: ",error)
data = json_graph.node_link_data(G)
......
annotations/static/annotations/app.css
View file @ 25584756
/* app css stylesheet */
/*
* Class names corresponding to server-side list names
* To display another list name, please add a new class under this
*/
.MiamList {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.StopList {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.delete-keyword, .occurrences {
vertical-align: super;
font-size: 70%;
......@@ -27,27 +43,9 @@
border-bottom: none;
}
.miamword {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.stopword {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.global-stopword {
color: black;
background-color: rgba(169, 68, 66, 0.05);
cursor: pointer;
}
.main-panel, .text-panel, .words-panel {
height: 800px;
margin: 10px 0px;
margin: 10px 0;
}
.text-panel {
......@@ -59,11 +57,7 @@
height: 250px;
}
.keyword-container {
/*display: inline-block;*/
}
.keyword {
.keyword-text {
word-break: break-all;
}
......
annotations/static/annotations/app.js
View file @ 25584756
This diff is collapsed.
annotations/static/annotations/http.js
View file @ 25584756
......@@ -57,11 +57,11 @@
{
post: {
method: 'POST',
params: {'listId': '@listId', 'ngramId': ''}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
},
delete: {
method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@id'}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
}
}
);
......
annotations/static/annotations/keyword_tpl.html
View file @ 25584756
<span ng-if="keyword.category == 'miamlist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="miamlist">×</span>
<span ng-if="keyword.category == 'miamlist'" data-toggle="tooltip" class="keyword miamword">{[{keyword.text}]}</span>
<span ng-if="keyword.category == 'stoplist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="stoplist">×</span>
<span ng-if="keyword.category == 'stoplist'" data-toggle="tooltip" class="keyword stopword">{[{keyword.text}]}</span>
<span ng-click='onDeleteClick()' class="delete-keyword">×</span>
<span data-toggle="tooltip" class="keyword-text {[{keyword.listName}]}">{[{keyword.text}]}</span>
<span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span>
annotations/static/annotations/selection_tpl.html deleted 100644 → 0
View file @ d6cc0e32
<ul class="noselection">
<li class="miamword" ng-if="local_miamlist === true" ng-click="onClick($event, 'post', miamListId, 'local')">add to miam-list</li>
<li class="miamword" ng-if="local_miamlist === false" ng-click="onClick($event, 'delete', miamListId, 'local')">remove from miam-list</li>
<li class="stopword" ng-if="local_stoplist === true" ng-click="onClick($event, 'post', stopListId, 'local')">add to stop-list</li>
<li class="stopword" ng-if="local_stoplist === false" ng-click="onClick($event, 'delete', stopListId, 'local')">remove from stop-list</li>
<!--<li class="stopword" ng-if="global_stoplist === true" ng-click="onClick($event, 'post', 'stoplist', 'global')">add to global stop-list</li>
<li class="stopword" ng-if="global_stoplist === false" ng-click="onClick($event, 'delete', 'stoplist', 'global')">remove from global stop-list</li>-->
</ul>
annotations/templates/annotations/main.html
View file @ 25584756
......@@ -9,8 +9,8 @@
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>Gargantext corpus annotations editor</title>
<meta name="description" content="">
<title>Gargantext article editor</title>
<meta name="description" content="Gargantext">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}">
......@@ -18,51 +18,38 @@
<script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script>
</head>
<body>
<!-- TODO integrate this later into the corpus.html django template -->
<!-- TODO integrate this later into the any other django template -->
<div id="annotationsApp">
<div class="container-fluid">
<div class="row-fluid main-panel" ng-controller="IntraTextController">
<div class="col-md-4 col-xs-4 tabbable words-panel">
<ul class="nav nav-pills nav-justified">
<li class="active"><a href="#tab1" data-toggle="tab">Miamwords</a></li>
<li><a href="#tab2" data-toggle="tab">Stopwords</a></li>
<li ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}">
<a href="#tab-{[{listId}]}" data-toggle="tab">{[{listName}]}</a>
</li>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="tab1">
<div ng-if="extra_miamlist.length == 0" class="alert alert-info" role="alert">No extra text miam-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_miamlist | startFrom:currentMiamPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li>
</ul>
<nav ng-class="{invisible: numMiamPages() - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentMiamPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentMiamPage >= numMiamPages()-1}"><a ng-click="nextMiamPage()" class="glyphicon glyphicon-forward"></a></li>
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="miamlist-input" ng-keypress="onMiamlistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onMiamlistSubmit($event)">Add</button>
<div ng-controller="ExtraTextPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}">
<div ng-if="extraNgramList[listId].length == 0" class="alert alert-info" role="alert">
Input any keyword you want to link to this article and the list named '{[{listName}]}'
</div>
</div>
<div class="tab-pane" id="tab2">
<div ng-if="extra_stoplist.length == 0" class="alert alert-info" role="alert">No extra text stop-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_stoplist | startFrom:currentStopPage*pageSize | limitTo:pageSize" class="list-group-item">
<ul class="list-group words-list clearfix">
<li ng-repeat="keyword in extraNgramList[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li>
</ul>
<nav ng-class="{invisible: numStopPages() - 1 == 0}" class="clearfix">
<nav ng-class="{invisible: totalListPages - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentStopPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentStopPage >= numStopPages()-1}"><a ng-click="nextStopPage()" class="glyphicon glyphicon-forward"></a></li>
<li ng-class="{disabled: currentListPage == 0}"><a ng-click="previousListPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentListPage >= totalListPages - 1}"><a ng-click="nextListPage()" class="glyphicon glyphicon-forward"></a></li>
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="stoplist-input" ng-keypress="onStoplistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onStoplistSubmit($event)">Add</button>
<input type="text" class="form-control" id="{[{ listId }]}-input" ng-keypress="onListSubmit($event, listId)">
<button type="submit" class="form-control btn btn-default btn-primary" ng-click="onListSubmit($event, listId)">Add to {[{listName}]}</button>
</div>
</div>
</div>
......@@ -70,7 +57,7 @@
<div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document">
<div class="row-fluid clearfix">
<div class="col-md-7 col-xs-7">
<h3>{[{title}]}</h3>
<h3 class="text-container" id="title">{[{title}]}</h3>
</div>
<div class="col-md-5 col-xs-5">
<nav>
......@@ -90,17 +77,21 @@
</div>
<h4 ng-if="abstract_text != null">Abstract</h4>
<p id="abstract-text" class="text-container">
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">No abstract text</div>
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">Empty abstract text</div>
</p>
<h4 ng-if="full_text != null">Full Article</h4>
<p id="full-text" class="text-container">
<div ng-if="full_text == null" class="alert alert-info" role="alert">No full text</div>
<div ng-if="full_text == null" class="alert alert-info" role="alert">Empty full text</div>
</p>
</div>
</div> <!-- end of the main row -->
</div>
<!-- this menu is over the text -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu" selection-template></div>
<!-- this menu is over the text on mouse selection -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu">
<ul class="noselection">
<li ng-repeat="item in menuItems" class="{[{item.listName}]}" ng-click="onMenuClick($event, item.action, item.listId)">{[{item.verb}]} {[{item.listName}]}</li>
</ul>
</div>
</div>
<!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
......
annotations/views.py
View file @ 25584756
......@@ -40,15 +40,16 @@ class NgramList(APIView):
"""Get All for a doc id"""
corpus_id = int(corpus_id)
doc_id = int(doc_id)
lists = dict()
lists = {}
for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
# ngrams of list_id of corpus_id:
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id)
#doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
doc_ngram_list = [(i, 'miam', i, 1931) for i in range(500)]
doc_ngram_list += [(i, 'stop', i, 1932) for i in range(501, 600)]
# doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
data = { '%s' % corpus_id : {
'%s' % doc_id : [
......
gargantext.ini
View file @ 25584756
......@@ -39,6 +39,10 @@ vacuum = true
pidfile = /tmp/gargantext.pid
# touch /tmp/gargantext.reload to reload configuration (after git pull for instance)
touch-reload = /tmp/gargantext.reload
# respawn processes taking more than 20 seconds
harakiri = 120
......
gargantext_web/about.py
View file @ 25584756
......@@ -29,6 +29,11 @@ def get_team():
'picture' : 'david.jpg',
'role':'principal investigator'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
{ 'first_name' : 'Mathieu', 'last_name' : 'Rodic',
'mail' : '',
'website' : 'http://rodic.fr',
......@@ -41,11 +46,6 @@ def get_team():
'picture' : 'samuel.jpg',
'role' : 'developer'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
#{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''},
# copy paste the line above and write your informations please
]
......
gargantext_web/corpus_views.py
View file @ 25584756
......@@ -43,7 +43,9 @@ from scrappers.scrap_pubmed.admin import Logger
from gargantext_web.db import *
from sqlalchemy import or_, func
from sqlalchemy.sql import func
from sqlalchemy import desc, asc, or_, and_, Date, cast, select
from gargantext_web import about
......@@ -128,7 +130,7 @@ def get_ngrams(request , project_id , corpus_id ):
'processing' : processing,
'number' : number,
'list_id': list_id[0][0],
'view' : "Terms",
'view' : "terms",
}))
return HttpResponse(html)
......@@ -195,7 +197,7 @@ def get_journals(request , project_id , corpus_id ):
'corpus' : corpus,
'processing' : processing,
'number' : number,
'view' : "Journals",
'view' : "journals",
}))
return HttpResponse(html)
......@@ -205,16 +207,18 @@ def test_journals(request , project_id, corpus_id ):
JournalsDict = {}
user_id = request.user.id
document_type_id = cache.NodeType['Document'].id
documents = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
for doc in documents:
if "journal" in doc.hyperdata:
journal = doc.hyperdata["journal"]
if journal not in JournalsDict:
JournalsDict [journal] = 0
JournalsDict[journal] += 1
return JsonHttpResponse(JournalsDict)
journal_count = (session.query(Node.hyperdata['journal'].label("journal"), func.count().label("count"))
.filter(Node.parent_id == corpus_id, Node.type_id==cache.NodeType['Document'].id)
.group_by("journal")
.order_by(desc("count"))
.all())
jc = dict()
for journal,count in journal_count:
jc[journal] = count
return JsonHttpResponse(jc)
def test_ngrams(request , project_id, corpus_id ):
results = ["hola" , "mundo"]
......@@ -341,7 +345,11 @@ def test_ngrams(request , project_id, corpus_id ):
ngrams_ids = Ngrams_Scores.keys()
import math
occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) )
if len(ngrams_ids) != 0:
occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) )
else:
occs_threshold = 10
Metrics = {
"ngrams":[],
......
gargantext_web/urls.py
View file @ 25584756
......@@ -48,16 +48,20 @@ urlpatterns = patterns('',
# Corpus management
# Document view (main)
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
url(r'^project/(\d+)/corpus/(\d+)/documents/?$', views.corpus),
# Journals view
url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals$', corpus_views.get_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals', corpus_views.get_journals),
# Terms view
url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', corpus_views.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/ngrams$', corpus_views.get_ngrams),
############################################################################
# # Terms view
url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', corpus_views.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/terms/?$', corpus_views.get_ngrams),
# Update corpus
url(r'^project/(\d+)/corpus/(\d+)/(\w+)/update$', views.update_nodes),
############################################################################
# annotations App
url(r'^project/(\d+)/corpus/(\d+)/document/(\d+)/$', annotations_main_view),
url(r'^annotations/', include(annotations_urls)),
......@@ -104,7 +108,9 @@ urlpatterns = patterns('',
url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX),
url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test),
url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', samtest.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/terms', samtest.get_ngrams)
)
......
gargantext_web/views.py
View file @ 25584756
from django.shortcuts import redirect
from django.shortcuts import render
from admin.utils import PrintException
from django.shortcuts import redirect, render, render_to_response
from django.db import transaction
from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden
from django.template.loader import get_template
from django.template import Context
from django.template import Context, RequestContext
# remove this
from node import models
......@@ -43,7 +44,7 @@ from sqlalchemy import or_, func
from gargantext_web import about
from gargantext_web.db import NodeNgram, NodeNgramNgram
def login_user(request):
logout(request)
......@@ -251,6 +252,71 @@ def projects(request):
})
def update_nodes(request, project_id, corpus_id, view):
'''
update function:
- remove previous computations (temporary lists and coocurrences)
- permanent deletion of Trash
'''
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = int(project_id)
offset = int(corpus_id)
offset = str(view)
except ValueError:
raise Http404()
try:
print(corpus_id)
whites = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['WhiteList'].id)
.all()
)
print(whites)
for white in whites:
session.query(NodeNgram).filter(NodeNgram.node_id==white.id).delete()
session.delete(white)
cooc = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['Cooccurrence'].id)
.first()
)
session.query(NodeNgramNgram).filter(NodeNgramNgram.node_id==cooc.id).delete()
session.delete(cooc)
session.commit()
except :
PrintException()
nodes = models.Node.objects.filter(type_id=cache.NodeType['Trash'].id, user_id=request.user.id).all()
if nodes is not None:
with transaction.atomic():
for node in nodes:
try:
node.children.delete()
except Exception as error:
print(error)
node.delete()
#return redirect(request.path.replace('update', ''))
return redirect('/project/%s/corpus/%s/%s' % (project_id, corpus_id, view))
#
# return render_to_response(
# request.path,
# { 'title': 'Corpus view' },
# context_instance=RequestContext(request)
# )
#
def corpus(request, project_id, corpus_id):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
......@@ -287,7 +353,7 @@ def corpus(request, project_id, corpus_id):
'processing' : processing,
# 'documents': documents,\
'number' : number,
'view' : "Documents"
'view' : "documents"
}))
return HttpResponse(html)
......@@ -499,9 +565,10 @@ def graph(request, project_id, corpus_id):
results[i.id]["corpuses"].append(info)
# print("\t",j.id , j.name)
import pprint
pprint.pprint(results)
# import pprint
# pprint.pprint(results)
graphurl = "corpus/"+str(corpus_id)+"/node_link.json"
html = t.render(Context({\
'debug': settings.DEBUG,
'user' : user,\
......@@ -509,7 +576,7 @@ def graph(request, project_id, corpus_id):
'corpus' : corpus,\
'project' : project,\
'corpusinfo' : results,\
'graphfile' : "hola_mundo",\
'graphfile' : graphurl,\
}))
return HttpResponse(html)
......@@ -626,14 +693,14 @@ def node_link(request, corpus_id):
data = []
corpus = session.query(Node).filter(Node.id==corpus_id).first()
filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id)
print("file exists?:",os.path.isfile(filename))
if os.path.isfile(filename):
json_data = open(filename,"r")
data = json.load(json_data)
json_data.close()
else:
data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
# filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id)
# print("file exists?:",os.path.isfile(filename))
# if os.path.isfile(filename):
# json_data = open(filename,"r")
# data = json.load(json_data)
# json_data.close()
# else:
data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
return JsonHttpResponse(data)
def adjacency(request, corpus_id):
......
parsing/FileParsers/ISTex.py
View file @ 25584756
......@@ -19,7 +19,7 @@ class ISTex(FileParser):
"source" : 'corpusName',
"title" : 'title',
"genre" : "genre",
# "language_iso3" : 'MedlineCitation/Article/Language',
"language_iso3" : 'language',
"doi" : 'doi',
"host" : 'host',
"publication_date" : 'pubdate',
......@@ -27,18 +27,23 @@ class ISTex(FileParser):
"authorsRAW" : 'author',
"keywords" : "keywords"
}
suma = 0
for json_doc in json_docs:
hyperdata = {}
for key, path in hyperdata_path.items():
try:
# print(path," ==> ",len(json_doc[path]))
hyperdata[key] = json_doc[path]
except: pass
except:
pass
# print("|",hyperdata["publication_date"])
# print("|",hyperdata["language_iso3"])
if "doi" in hyperdata: hyperdata["doi"] = hyperdata["doi"][0]
if "doi" in hyperdata:
hyperdata["doi"] = hyperdata["doi"][0]
keywords = []
if "keywords" in hyperdata:
......@@ -75,34 +80,53 @@ class ISTex(FileParser):
if "genre" in hyperdata:
if len(hyperdata["genre"])==0:
hyperdata.pop("genre")
if "language_iso3" in hyperdata:
if len(hyperdata["language_iso3"])>0:
hyperdata["language_iso3"] = hyperdata["language_iso3"][0]
else:
hyperdata["language_iso3"] = "eng"
RealDate = hyperdata["publication_date"]
if "publication_date" in hyperdata: hyperdata.pop("publication_date")
if "publication_date" in hyperdata:
RealDate = hyperdata["publication_date"]
if "publication_date" in hyperdata:
hyperdata.pop("publication_date")
Decision=""
if len(RealDate)>4:
if len(RealDate)>8:
try: Decision = datetime.strptime(RealDate, '%Y-%b-%d').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
if isinstance(RealDate, list):
RealDate = RealDate[0]
# print( RealDate ," | length:",len(RealDate))
Decision=""
if len(RealDate)>4:
if len(RealDate)>8:
try: Decision = datetime.strptime(RealDate, '%Y-%b-%d').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
if Decision!=False:
hyperdata["publication_year"] = str(Decision.year)
hyperdata["publication_month"] = str(Decision.month)
hyperdata["publication_day"] = str(Decision.day)
hyperdata_list.append(hyperdata)
# print("\t||",hyperdata["title"])
# print("\t\t",Decision)
# print("=============================")
try: Decision = datetime.strptime(RealDate, '%Y').date()
except: Decision=False
if Decision!=False:
hyperdata["publication_year"] = str(Decision.year)
hyperdata["publication_month"] = str(Decision.month)
hyperdata["publication_day"] = str(Decision.day)
hyperdata_list.append(hyperdata)
# print("\t||",hyperdata["title"])
# print("\t\t",Decision)
# print("=============================")
# else:
# suma+=1
# if "pubdate" in json_doc:
# print ("\tfail pubdate:",json_doc["pubdate"])
# print ("nb_hits:",len(json_docs))
# print("\t - nb_fails:",suma)
# print(" -- - - - - - -- - -")
return hyperdata_list
scrappers/scrap_pubmed/views.py
View file @ 25584756
......@@ -66,7 +66,7 @@ def getGlobalStatsISTEXT(request ):
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
query_string = query.replace(" ","+")
url = "http://api.istex.fr/document/?q="+query_string
url = "http://api.istex.fr/document/?q="+query_string+"&output=id,title,abstract,pubdate,corpusName,authors,language"
tasks = MedlineFetcher()
......
static/js/Docs_dyna_chart_and_table.js
View file @ 25584756
This diff is collapsed.
static/js/Journals_dyna_chart_and_table.js
View file @ 25584756
......@@ -471,11 +471,17 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').process();
// // // $("#score_column_id").children()[0].text = FirstScore
// // // // MyTable.data('dynatable').process();
......
static/js/NGrams_dyna_chart_and_table.js
View file @ 25584756
......@@ -370,15 +370,18 @@ $("#Clean_All").click(function(){
});
$("#Save_All").click(function(){
console.log("click in save all 01")
var sum__selected_elems = 0;
var poubelle = []
for(var i in FlagsBuffer)
if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
for(var i in FlagsBuffer) {
if (Object.keys(FlagsBuffer[i]).length==0)
poubelle.push(i)
sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
}
console.log("click in save all 02")
for(var i in poubelle)
delete FlagsBuffer[poubelle[i]];
console.log("click in save all 03, sum:"+sum__selected_elems)
if ( sum__selected_elems>0 ) {
console.log("")
console.log("Do the ajax conexion with API and send this array to be processed:")
......@@ -498,11 +501,6 @@ function Main_test( data , initial) {
oldest = Number(min_occ);
latest = Number(max_occ);
var ndx = false;
ndx = crossfilter();
ndx.add(DistributionList);
......@@ -624,6 +622,9 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear();
......@@ -661,7 +662,7 @@ $.ajax({
// Building the Score-Selector
var FirstScore = data.scores.initial
var possible_scores = Object.keys( data.ngrams[0].scores );
var scores_div = '<select class="span1" id="scores_selector">'+"\n";
var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
for( var i in possible_scores ) {
if(possible_scores[i]!=FirstScore) {
......
templates/corpus/bannerBottome.html deleted 100644 → 0
View file @ d6cc0e32
<!--<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>--!>
templates/corpus/documents.html
View file @ 25584756
......@@ -4,7 +4,7 @@
{% load staticfiles %}
<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
<!-- here goes import stylesheet js/bootstrap/bootstrap-multiselect.css, mais ca marche pas-->
<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
......@@ -72,6 +72,7 @@ th a {
<input type="hidden" id="list_id" value="{{ list_id }}"></input>
<div class="container">
<div class="jumbotron">
......@@ -87,18 +88,7 @@ th a {
</div>
<div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
<div class="panel-body">
<p align="right">
<table id="my-ajax-table" class="table table-bordered">
<thead>
<!-- <th data-dynatable-column="id">ID</th> -->
<th width="100px;" data-dynatable-column="date">Date</th>
<th data-dynatable-column="name">Title</th>
<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>
</thead>
<tbody>
</tbody>
</table>
</p>
<div id="div-table"></div>
<p align="right">
<button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button>
</p>
......@@ -111,24 +101,26 @@ th a {
</div>
</div>
<div id="supmofos">
<select data-width="100px" dir="ltr" class="selectpicker">
<option selected>All</option>
<option>Title</option>
<option>Date</option>
<optgroup label="Duplicates">
<option>by DOI</option>
<option>by Title</option>
</optgroup>
</select>
<div id="filter_search" style="visibility:hidden">
<select id="example-single-optgroups" onchange="SearchFilters(this);">
<!-- <optgroup label=""> -->
<option id="filter_all" value="filter_all">All</option>
<!-- <option id="filter_title" value="filter_title">Title</option> -->
<!-- <option id="filter_date" value="filter_date">Date</option> -->
<!-- </optgroup> -->
<!-- <optgroup label="Duplicates"> -->
<!-- <option value="filter_doi">By DOI</option> -->
<option id="filter_dupl-titles" value="filter_dupl-titles">Duplicates by Title</option>
<!-- </optgroup> -->
</select>
</div>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
<!-- here goes import script js/bootstrap/bootstrap-multiselect.js, mais ca marche pas-->
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js -->
......
templates/corpus/menu.html
View file @ 25584756
{% extends "menu.html" %}
{% load staticfiles %}
{% block corpusBannerTop %}
......@@ -10,7 +11,7 @@
<div class="col-md-4">
{% if project %}
<h1>{{ project.name }}</h1>
<h1><a href="/project/{{project.id}}">{{ project.name }}</a></h1>
<h2>{{ corpus.name }}</h2>
{% endif %}
<!--<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
......@@ -32,9 +33,9 @@
<div class="col-md-4">
<br>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{corpus.id}}/{{view}}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Download</a>
<a type="button" class="btn btn-default btn-lg" data-container="body" data-toggle="popover" data-placement="bottom"
data-content='
<ul>
......@@ -62,9 +63,12 @@
<div class="btn-group btn-group-justified">
<center>
<a type="button" class="btn btn-default {% if view == "Documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/">{{number}} Documents</a>
<a type="button" class="btn btn-default {% if view == "Journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a>
<a type="button" class="btn btn-default {% if view == "Terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/ngrams">Terms</a>
<a type="button" class="btn btn-default {% if view == "documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/documents">{{number}} Documents</a>
<a type="button" class="btn btn-default {% if view == "journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a>
{% if processing == 0 %}
<a type="button" class="btn btn-default {% if view == "terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/terms">Terms</a>
{% endif %}
</center>
</div>
......@@ -76,7 +80,7 @@
<div class="container">
<div class="row">
<div class="col-md-4">
<div class="col-md-6">
<div class="jumbotron">
<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
<ol>
......@@ -88,23 +92,11 @@
</div>
</div>
<div class="col-md-4">
<div class="jumbotron">
<!-- <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3> -->
<h3>Matrix (soon)</h3>
<ol>
<li>Sort</li>
<li>Group</li>
<li>Cluster</li>
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
<div class="col-md-4">
<div class="col-md-6">
<div class="jumbotron">
{% if processing > 0 %}
<h3> Graph (later)</h3>
<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
{% else %}
<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
{% endif %}
......
templates/explorer.html
View file @ 25584756
......@@ -60,7 +60,7 @@
<li>
<a>
<div id="graphid" style="visibility: hidden;">/corpus/{{ corpus.id }}/node_link.json</div>
<div id="graphid" style="visibility: hidden;">{{graphfile}}</div>
</a>
</li>
......@@ -177,9 +177,7 @@
<div style="visibility: hidden;" id="sigma-othergraph"></div>
<div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
</div>
<div id="semLoader"></div>
<ul id="ctlzoom">
......
templates/project.html
View file @ 25584756
......@@ -87,7 +87,7 @@
{% ifequal corpus.processing 1 %}
{{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
{% else %}
<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} </a> , {{ corpus.count }} Documents
<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} , {{ corpus.count }} Documents </a>
{% endifequal %}
<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
data-content='
......
test-list-management.py
View file @ 25584756
......@@ -6,8 +6,8 @@ from ngram.lists import *
#from cooccurrences import *
#from gargantext_web.views import empty_trash
#empty_trash()
from gargantext_web.views import empty_trash
empty_trash()
#user = session.query(User).all()[0]
......@@ -77,6 +77,7 @@ print('Stem Node.id is', stem_id)
# print(n, listNgramIds(list_id=n[0][0])[:3])
#
stop_list_id = listIds(user_id=user.id,
corpus_id=corpus.id,
typeList='StopList')[0][0]
......@@ -86,13 +87,13 @@ miam_list_id = listIds(user_id=user.id,
typeList='MiamList')[0][0]
print('Stop List', stop_list_id)
print('Miam List', miam_list_id)
print('StopList', stop_list_id)
print('MiamList', miam_list_id)
ngrams2miam(user_id=user.id, corpus_id=corpus.id)
print(session.query(Node.id).filter(Node.parent_id==corpus.id, Node.type_id==cache.NodeType['WhiteList'].id).first())
#ngrams2miam(user_id=user.id, corpus_id=corpus.id)
print(listNgramIds(list_id=miam_list_id, user_id=user.id, corpus_id=corpus.id))
#print(listNgramIds(list_id=stop_list_id, user_id=user.id, corpus_id=corpus.id))
......
tests/ngramstable/views.py
View file @ 25584756
......@@ -100,7 +100,7 @@ def get_ngrams(request , project_id , corpus_id ):
except ValueError:
raise Http404()
t = get_template('tests/ngrams.html')
t = get_template('corpus/terms.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
......@@ -173,7 +173,7 @@ def get_journals(request , project_id , corpus_id ):
except ValueError:
raise Http404()
t = get_template('tests/journals.html')
t = get_template('corpus/journals.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment