Skip to content

gargantext
gargantext
Commit 64ea6b3d authored by delanoe's avatar delanoe
Browse files
Options

Revert "Merge remote-tracking branch 'origin/romain-testing' into testing" 

This reverts commit d9d93ae0, reversing
changes made to 4ef20306.
parent d9d93ae0
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 83 additions and 504 deletions
annotations/static/annotations/document.js
View file @ 64ea6b3d
......@@ -15,7 +15,7 @@
{'docId': $rootScope.docId},
function(data, responseHeaders) {
$scope.authors = data.authors;
$scope.source = data.source;
$scope.journal = data.journal;
$scope.publication_date = data.publication_date;
//$scope.current_page_number = data.current_page_number;
//$scope.last_page_number = data.last_page_number;
......@@ -23,34 +23,25 @@
$rootScope.docId = data.id;
$rootScope.full_text = data.full_text;
$rootScope.abstract_text = data.abstract_text;
$rootScope.workflow_finished = data.corpus_status['complete'] ;
console.log("workflow status", $rootScope.workflow_finished)
if ($scope.workflow_finished) {
console.log("annotations.document.DocController.getannotations")
// GET the annotationss
NgramListHttpService.get(
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
// eg id => 'MAPLIST'
$rootScope.lists = data[$rootScope.corpusId.toString()].lists;
// inverted 'MAPLIST' => id
$rootScope.listIds = _.invert($rootScope.lists)
$scope.dataLoading = false ;
},
function(data) {
console.error("unable to get the list of ngrams");
}
);
}
else {
$scope.dataLoading = false ;
}
console.log("annotations.document.DocController.getannotations")
// GET the annotationss
NgramListHttpService.get(
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
// eg id => 'MAPLIST'
$rootScope.lists = data[$rootScope.corpusId.toString()].lists;
// inverted 'MAPLIST' => id
$rootScope.listIds = _.invert($rootScope.lists)
$scope.dataLoading = false ;
},
function(data) {
console.error("unable to get the list of ngrams");
}
);
});
......
annotations/templates/annotations/main.html
View file @ 64ea6b3d
......@@ -24,7 +24,7 @@
<div id="annotationsApp" ng-cloak>
<div class="container-fluid">
<div class="row-fluid main-panel" ng-controller="NGramHighlightController">
<div ng-if="workflow_finished" class="col-md-4 col-xs-4 tabbable words-panel">
<div class="col-md-4 col-xs-4 tabbable words-panel">
<div class="list-selector">
<h5>Select highlighted list(s)
<select class="selectpicker" multiple ng-change="activeListsChange()" ng-model="lists" ng-controller="ActiveListsController">
......@@ -89,7 +89,7 @@
</div>
<div class="row-fluid">
<ul class="list-group clearfix">
<li class="list-group-item small"><span class="badge">source</span>{[{source}]}</li>
<li class="list-group-item small"><span class="badge">journal</span>{[{journal}]}</li>
<li class="list-group-item small"><span class="badge">authors</span>{[{authors}]}</li>
<li class="list-group-item small"><span class="badge">date</span>{[{publication_date}]}</li>
</ul>
......@@ -108,14 +108,12 @@
<span class="badge">abstract</span>
</div>
<p id="abstract-text" class="text-container">
{[{abstract_text}]}
<div ng-if="abstract_text == null" class="alert alert-info small" role="alert">Empty abstract text</div>
</p>
<div ng-if="full_text != null">
<span class="badge">full article</span>
</div>
<p id="full-text" class="text-container">
{[{full_text}]}
<div ng-if="full_text == null" class="alert alert-info small" role="alert">Empty full text</div>
</p>
</div>
......
annotations/views.py
View file @ 64ea6b3d
......@@ -172,9 +172,8 @@ class Document(APIView):
def get(self, request, doc_id):
"""Document by ID"""
# implicit global session
node = session.query(Node).filter(Node.id == doc_id).first()
corpus = session.query(Node).filter(Node.id == node.parent_id).first()
corpus_workflow_status = corpus.hyperdata['statuses'][0]
if node is None:
raise APIException('This node does not exist', 404)
......@@ -186,10 +185,9 @@ class Document(APIView):
pub_date = node.hyperdata.get('publication_date')
data = {
'corpus_status': corpus_workflow_status,
'title': node.hyperdata.get('title'),
'authors': node.hyperdata.get('authors'),
'source': node.hyperdata.get('source'),
'journal': node.hyperdata.get('journal'),
'publication_date': pub_date,
'full_text': node.hyperdata.get('full_text'),
'abstract_text': node.hyperdata.get('abstract'),
......
gargantext/constants.py
View file @ 64ea6b3d
......@@ -382,9 +382,6 @@ BATCH_NGRAMSEXTRACTION_SIZE = 3000 # how many new node-ngram relations before
QUERY_SIZE_N_MAX = 1000
QUERY_SIZE_N_DEFAULT = 1000
# Refresh corpora workflow status for project view's progressbar
PROJECT_VIEW_REFRESH_INTERVAL = 3000 # 1st refresh in ms (then increasing arithmetically)
PROJECT_VIEW_MAX_REFRESH_ATTEMPTS = 10 # how many times before we give up
# ------------------------------------------------------------------------------
# Graph <=> nodes API parameters
......
gargantext/util/db_cache.py
View file @ 64ea6b3d
......@@ -46,9 +46,6 @@ class ModelCache(dict):
class Cache:
def __getattr__(self, key):
'''
lazy init of new modelcaches: self.Node, self.User...
'''
try:
model = getattr(models, key)
except AttributeError:
......@@ -57,15 +54,4 @@ class Cache:
setattr(self, key, modelcache)
return modelcache
def clean_all(self):
'''
re-init any existing modelcaches
'''
for modelname in self.__dict__:
old_modelcache = getattr(cache, modelname)
new_modelcache = ModelCache(old_modelcache._model)
del old_modelcache
setattr(cache, modelname, new_modelcache)
cache = Cache()
gargantext/util/http.py
View file @ 64ea6b3d
......@@ -32,10 +32,6 @@ def requires_auth(func):
from gargantext.util.db import session
session.rollback()
print("=== session rollback ok!")
# re init the global cache (it must still have detached instances)
from gargantext.util.db_cache import cache
cache.clean_all()
print("=== cache reinit ok!")
# and relogin for safety
url = '/auth/login/?next=%s' % urlencode(request.path)
return redirect(url)
......
gargantext/util/toolchain/ngrams_addition.py
View file @ 64ea6b3d
......@@ -23,8 +23,6 @@ from gargantext.util.db import bulk_insert_ifnotexists # £TODO debug
from sqlalchemy import distinct
from re import findall, IGNORECASE
from gargantext.util.toolchain.main import t # timer
# TODO from gargantext.constants import LIST_OF_KEYS_TO_INDEX = title, abstract
def index_new_ngrams(ngram_ids, corpus, keys=('title', 'abstract', )):
......@@ -42,8 +40,6 @@ def index_new_ngrams(ngram_ids, corpus, keys=('title', 'abstract', )):
@param corpus: the CORPUS node
@param keys: the hyperdata fields to index
# FIXME too slow: index_new_ngrams should be faster via tsvector on DB
"""
# retrieve *all* the ngrams from our list
......@@ -60,11 +56,7 @@ def index_new_ngrams(ngram_ids, corpus, keys=('title', 'abstract', )):
node_ngram_to_write = {}
# loop throught the docs and their text fields
for (i, doc) in enumerate(corpus.children('DOCUMENT')):
if (i % 100 == 0):
print('CORPUS #%d: [%s] ngrams_addition: doc %i' % (corpus.id, t(), i))
print()
for doc in corpus.children('DOCUMENT'):
# a new empty counting subdict
node_ngram_to_write[doc.id] = {}
......
gargantext/views/api/ngramlists.py
View file @ 64ea6b3d
......@@ -373,10 +373,6 @@ class ListChange(APIView):
def put(self, request):
"""
Adds one or more ngrams to a list.
NB: we assume ngram_ids don't contain subforms !!
(this assumption is not checked here because it would be
slow: if you want to add a subform, send the mainform's id)
"""
# union of items ----------------------------
new_list = self.base_list + self.change_list
......
gargantext/views/api/nodes.py
View file @ 64ea6b3d
......@@ -17,11 +17,7 @@ _node_available_fields = ['id', 'parent_id', 'name', 'typename', 'hyperdata', 'n
_node_default_fields = ['id', 'parent_id', 'name', 'typename']
_node_available_types = NODETYPES
_hyperdata_available_fields = ['title', 'source', 'abstract', 'statuses',
'language_name', 'language_iso3','language_iso2','language_id',
'publication_date',
'publication_year','publication_month', 'publication_day',
'publication_hour','publication_minute','publication_second']
#_hyperdata_available_fields = ['title', 'resourcetype']
#_node_available_formats = ['json', 'csv', 'bibex']
......@@ -36,38 +32,24 @@ def _query_nodes(request, node_id=None):
# parameters validation
# fixme: this validation does not allow custom keys in url (eg '?name=' for rename action)
parameters = get_parameters(request)
parameters = validate(parameters, {'type': dict, 'items': {
'formated': {'type': str, 'required' : False, 'default': 'json'},
# 'hyperdata': {'type': list, 'default' : _hyperdata_available_fields, 'items': {
# 'type': str, 'range' : _node_available_fields,
# }},
'pagination_limit': {'type': int, 'default': 10},
'pagination_offset': {'type': int, 'default': 0},
'fields': {'type': list, 'default': _node_default_fields, 'items': {
'type': str, 'range': _node_available_fields,
}},
# choice of hyperdata fields
'hyperdata_filter': {'type': list, 'required':False,
'items': {
'type': str, 'range': _hyperdata_available_fields,
}},
# optional filtering parameters
'types': {'type': list, 'required': False, 'items': {
'type': str, 'range': _node_available_types,
}},
'parent_id': {'type': int, 'required': False},
}})
# debug
# print('PARAMS', parameters)
# additional validation for hyperdata_filter
if (('hyperdata_filter' in parameters)
and (not ('hyperdata' in parameters['fields']))):
raise ValidationException("Using the hyperdata_filter filter requires fields[]=hyperdata")
# start the query
query = user.nodes()
# filter by id
if node_id is not None:
query = query.filter(Node.id == node_id)
......@@ -81,7 +63,6 @@ def _query_nodes(request, node_id=None):
count = query.count()
# order
query = query.order_by(Node.hyperdata['publication_date'], Node.id)
# paginate the query
if parameters['pagination_limit'] == -1:
query = query[parameters['pagination_offset']:]
......@@ -91,54 +72,8 @@ def _query_nodes(request, node_id=None):
parameters['pagination_limit']
]
# return the result!
# (the receiver function does the filtering of fields and hyperdata_filter)
return parameters, query, count
def _filter_node_fields(node, parameters):
"""
Filters the properties of a Node object before sending them to response
@parameters: a dict comming from get_parameters
that must only contain a 'fields' key
Usually the dict looks like this :
{'fields': ['parent_id', 'id', 'name', 'typename', 'hyperdata'],
'hyperdata_filter': ['title'], 'parent_id': '55054',
'types': ['DOCUMENT'], 'pagination_limit': '15'}
History:
1) this used to be single line:
res = {field: getattr(node, field) for field in parameters['fields']}
2) it was in both NodeResource.get() and NodeListResource.get()
3) it's now expanded to add support for parameters['hyperdata_filter']
- if absent, entire hyperdata is considered as one field
(as before)
- if present, the hyperdata subfields are picked
(new)
"""
# FIXME all this filtering
# could be done in rawsql
# (in _query_nodes)
result = {}
for field in parameters['fields']:
# normal field or entire hyperdata
if field != 'hyperdata' or (not 'hyperdata_filter' in parameters):
result[field] = getattr(node,field)
# hyperdata if needs to be filtered
else:
this_filtered_hyp = {}
for hfield in parameters['hyperdata_filter']:
if hfield in node.hyperdata:
this_filtered_hyp[hfield] = node.hyperdata[hfield]
result['hyperdata'] = this_filtered_hyp
return result
class Status(APIView):
'''API endpoint that represent the current status of the node'''
renderer_classes = (JSONRenderer, BrowsableAPIRenderer)
......@@ -149,27 +84,17 @@ class Status(APIView):
return HttpResponse('Unauthorized', status=401)
user = cache.User[request.user.id]
# check_rights(request, node_id)
# I commented check_rights because filter on user_id below does the job
check_rights(request, node_id)
node = session.query(Node).filter(Node.id == node_id, Node.user_id== user.id).first()
if node is None:
return Response({"detail":"Node not Found for this user"}, status=HTTP_404_NOT_FOUND)
else:
# FIXME using the more generic strategy ---------------------------
# context = format_response(node, [n for n in node.children()])
# or perhaps ? context = format_response(None, [node])
# -----------------------------------------------------------------
# using a more direct strategy
context = {}
context = format_response(node, [n for n in node.children()])
try:
context["statuses"] = node.hyperdata["statuses"]
context["status"] = node.hyperdata["statuses"]
except KeyError:
context["statuses"] = None
context["status"] = None
return Response(context)
def post(self, request, data):
'''create a new status for node'''
if not request.user.is_authenticated():
......@@ -177,17 +102,17 @@ class Status(APIView):
return HttpResponse('Unauthorized', status=401)
raise NotImplementedError
def put(self, request, data):
'''update status for node'''
if not request.user.is_authenticated():
# can't use @requires_auth because of positional 'self' within class
return HttpResponse('Unauthorized', status=401)
user = cache.User[request.user.id]
# check_rights(request, node_id)
node = session.query(Node).filter(Node.id == node_id, Node.user_id== user.id).first()
check_rights(request, node_id)
node = session.query(Node).filter(Node.id == node_id).first()
raise NotImplementedError
......@@ -201,8 +126,8 @@ class Status(APIView):
return HttpResponse('Unauthorized', status=401)
user = cache.User[request.user.id]
# check_rights(request, node_id)
node = session.query(Node).filter(Node.id == node_id, Node.user_id == user.id).first()
check_rights(request, node_id)
node = session.query(Node).filter(Node.id == node_id).first()
if node is None:
return Response({"detail":"Node not Found"}, status=HTTP_404_NOT_FOUND)
node.hyperdata["status"] = []
......@@ -225,22 +150,16 @@ class NodeListResource(APIView):
parameters, query, count = _query_nodes(request)
if parameters['formated'] == 'json':
records_array = []
add_record = records_array.append
# FIXME filter in rawsql in _query_nodes
for node in query:
add_record(_filter_node_fields(node, parameters))
return JsonHttpResponse({
'parameters': parameters,
'count': count,
'records': records_array
'records': [
{ field: getattr(node, field) for field in parameters['fields'] }
for node in query
]
})
elif parameters['formated'] == 'csv':
# TODO add support for fields and hyperdata_filter
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="Gargantext_Corpus.csv"'
......@@ -386,8 +305,9 @@ class NodeResource(APIView):
if not len(query):
raise Http404()
node = query[0]
return JsonHttpResponse(_filter_node_fields(node, parameters))
return JsonHttpResponse({
field: getattr(node, field) for field in parameters['fields']
})
# contains a check on user.id (within _query_nodes)
def delete(self, request, node_id):
......@@ -495,11 +415,11 @@ class CorpusFavorites(APIView):
(will test if docs 53 and 54 are among the favorites of corpus 2)
(returns the intersection of fav docs with [53,54])
"""
if not request.user.is_authenticated():
# can't use @requires_auth because of positional 'self' within class
return HttpResponse('Unauthorized', status=401)
fav_node = self._get_fav_node(corpus_id)
req_params = validate(
......
gargantext/views/pages/projects.py
View file @ 64ea6b3d
......@@ -54,9 +54,6 @@ def overview(request):
# projects owned by the user's contacts
'common_users': (contact for contact, projects in contacts_projects),
'common_projects': sum((projects for contact, projects in contacts_projects), []),
# status refreshing params (when active workflows)
'status_refresh_initial_interval': PROJECT_VIEW_REFRESH_INTERVAL,
'status_refresh_max_attempts': PROJECT_VIEW_MAX_REFRESH_ATTEMPTS,
},
)
......
static/lib/gargantext/Docs_dyna_chart_and_table.js
View file @ 64ea6b3d
......@@ -592,7 +592,7 @@ function Main_test(Data) {
// by default we always decide to search in the title
matchInTexts = [record.rawtitle]
// if box is checked we'll also search in the abstracts (todo: via ajax)
// if box is checked we'll also search in the abstracts
if (doAbstractsSearch) {
if (typeof record.hyperdata.abstract !== 'undefined') {
matchInTexts.push(record.hyperdata.abstract)
......@@ -630,23 +630,9 @@ function Main_test(Data) {
MyTable.data('dynatable').sorts.functions["signatureSort"] = makeAlphaSortFunctionOnProperty('signature')
MyTable.data('dynatable').sorts.functions["sourceSort"] = function sourceSort (rec1,rec2, attr, direction) {
// like rawtitle but nested property
if (rec1.hyperdata && rec1.hyperdata.source
&& rec2.hyperdata && rec2.hyperdata.source) {
// the alphabetic sort
if (direction == 1) return rec1.hyperdata.source.localeCompare(rec2.hyperdata.source)
else return rec2.hyperdata.source.localeCompare(rec1.hyperdata.source)
}
else if (rec1.hyperdata && rec1.hyperdata.source) {
cmp = direction
}
else if (rec2.hyperdata && rec2.hyperdata.source) {
cmp = -direction
}
else {
cmp = 0
}
if (cmp == 0) cmp = RecDict[rec1.id] < RecDict[rec2.id] ? -1 : 1
}
if (direction == 1) return rec1.hyperdata.source.localeCompare(rec2.hyperdata.source)
else return rec2.hyperdata.source.localeCompare(rec1.hyperdata.source)
}
// hook on page change
MyTable.bind('dynatable:page:set', tidyAfterPageSet)
......@@ -750,20 +736,9 @@ function makeAlphaSortFunctionOnProperty(property) {
return function (rec1,rec2, attr, direction) {
var cmp = null
if (rec1[property] && rec2[property]) {
// the alphabetic sort
if (direction == 1) cmp = rec1[property].localeCompare(rec2[property])
else cmp = rec2[property].localeCompare(rec1[property])
}
else if (rec1[property]) {
cmp = direction
}
else if (rec2[property]) {
cmp = -direction
}
else {
cmp = 0
}
// the alphabetic sort
if (direction == 1) cmp = rec1[property].localeCompare(rec2[property])
else cmp = rec2[property].localeCompare(rec1[property])
// second level sorting on key = id in records array
// (this one volontarily not reversable by direction
......@@ -793,10 +768,7 @@ function tidyAfterPageSet() {
$.ajax({
url: '/api/nodes?types[]=DOCUMENT&pagination_limit=-1&parent_id='
+ corpus_id
+'&fields[]=parent_id&fields[]=id&fields[]=name&fields[]=typename&fields[]=hyperdata'
// +'&hyperdata_filter[]=title&hyperdata_filter[]=source&hyperdata_filter[]=language_iso2'
+'&hyperdata_filter[]=title&hyperdata_filter[]=source&hyperdata_filter[]=language_iso2&hyperdata_filter[]=abstract'
+'&hyperdata_filter[]=publication_year&hyperdata_filter[]=publication_month&hyperdata_filter[]=publication_day',
+'&fields[]=parent_id&fields[]=id&fields[]=name&fields[]=typename&fields[]=hyperdata',
success: function(maindata){
// unfortunately favorites info is in a separate request (other nodes)
$.ajax({
......@@ -866,10 +838,6 @@ $.ajax({
rec.hyperdata.publication_day
)
// and a bool property for remote search results
// (will be updated by ajax)
rec.matched_remote_search = false // TODO use it
}
AjaxRecords = maindata.records; // backup-ing in global variable!
......
templates/pages/corpora/terms.html
View file @ 64ea6b3d
......@@ -131,7 +131,7 @@ em {
&nbsp;
<!-- save/import button -->
<button id="ImportListOrSaveAll" class="btn btn-warning" style="font-size:120%"
onclick="$('#csvimport').modal('show'); document.getElementById('importsubmit').disabled = false ;">
onclick="$('#csvimport').modal('show');">
<b>Import a Termlist</b>
</button>
</div>
......@@ -437,22 +437,13 @@ function listmergeCsvPost(theFile){
// reload after 3s
setTimeout("location.reload(true)", 3000);
},
error: function(result, t) {
if (t != 'timeout') {
my_html = '<h3 style="color:red">Error</h3>'
my_html += "<p class='note'>please correct your CSV file and retry</p>"
my_html += "<p>"+ result.responseJSON['err']+"</p>"
}
else {
my_html = '<h3 style="color:red">Timeout</h3>'
my_html += "<p>The CSV import timed out.</p>"
my_html += "<p>(This bug is currently being fixed. <br/>The import and indexation are now continuing in background on our servers. Results will show in a few minutes.)</p>"
document.getElementById('importsubmit').disabled = true
}
error: function(result) {
my_html = '<h3 style="color:red">Error</h3>'
my_html += "<p class='note'>please correct your CSV file and retry</p>"
my_html += "<p>"+ result.responseJSON['err']+"</p>"
$('#formanswer').html(my_html);
console.error(result);
},
timeout: 15000 // 15s
});
}
};
......
templates/pages/corpora/titles.html
View file @ 64ea6b3d
......@@ -21,12 +21,12 @@
<div class="col-md-3 col-md-offset-2">
<div id="monthly-move-chart">
<center>
Select a time range in the chart with blue bars to zoom in
<p align="center">
Select a time range in the chart with blue bars to zoom in
<p align="center">
<!--<a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>-->
<a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a>
</p>
<div class="clearfix"></div>
<a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
<div class="clearfix"></div>
</center>
</div>
......@@ -68,14 +68,10 @@
<span style="font-size:70%;">
<span class="glyphicon glyphicon-filter" aria-hidden="true"></span>
<!-- Used by the #doubleSearch associated function -->
<input title="Search in Titles" id="searchTI" name="searchTI" type="checkbox" checked onclick="return false">
TI&nbsp;
</input>
<span class="glyphicon glyphicon-filter" aria-hidden="true"></span>
<input title="Search in Abstracts" id="searchAB" name="searchAB" type="checkbox">
AB&nbsp;&nbsp;
</input>
</span>
<input title="Search in Titles" id="searchTI" name="searchTI" type="checkbox" checked onclick="return false">TI&nbsp;
<span class="glyphicon glyphicon-filter" aria-hidden="true"></span>
<input title="Search in Abstracts" id="searchAB" name="searchAB" type="checkbox">AB
</span>&nbsp;&nbsp;
<span class="glyphicon glyphicon-filter" aria-hidden="true"></span>
<select id="docFilter" name="docFilter">
<option value="filter_all">All</option>
......
templates/pages/projects/project.html
View file @ 64ea6b3d
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment