Skip to content

gargantext
gargantext
Commit f4599719 authored by c24b's avatar c24b
Browse files
Options

Merge branch 'refactoring' of ssh://delanoe.org:1979/gargantext into c24b-cern 

[Merge OK]
parents 3ba747fa 5e7c5603
Hide whitespace changes
Inline Side-by-side
Showing with 114 additions and 106 deletions
annotations/static/annotations/activelists.js
View file @ f4599719
...@@ -10,10 +10,14 @@ ...@@ -10,10 +10,14 @@
$scope.activeListsChange = function() { $scope.activeListsChange = function() {
var selected = $('.selectpicker option:selected').val(); var selected = $('.selectpicker option:selected').val();
var newActive = {}; var newActive = {};
$('.selectpicker option:selected').each(function(item, value) { $('.selectpicker option:selected').each(function(item, opt) {
var id = value.id.split("---", 2)[1]; // ex opt:
newActive[id] = value.value; // <option id="list---748" value="MAINLIST">MAINLIST</option>
var id = opt.id.split("---", 2)[1];
newActive[id] = opt.value;
}); });
// ex: {745: "MAINLIST", 748: "MAPLIST"}
$rootScope.activeLists = newActive; $rootScope.activeLists = newActive;
}; };
......
annotations/static/annotations/app.css
View file @ f4599719
...@@ -9,7 +9,8 @@ ...@@ -9,7 +9,8 @@
.MAPLIST { .MAPLIST {
color: black; color: black;
/* green */ /* green */
background-color: rgba(60, 118, 61, .7); background-color: rgba(23, 255, 189, .7);
/* background-color: rgba(60, 118, 61, .7); */
cursor: pointer; cursor: pointer;
} }
......
annotations/static/annotations/highlight.js
View file @ f4599719
...@@ -441,15 +441,12 @@ ...@@ -441,15 +441,12 @@
var middlePattern = " "; var middlePattern = " ";
var endPattern = "(?:<\/span>)*)\\b"; var endPattern = "(?:<\/span>)*)\\b";
console.log("highlight annotations length: " + annotations.length) // hash of flags filled in first pass loop : (== did annotation i match ?)
var isDisplayedIntraText = {};
var sortedSizeAnnotations = lengthSort(annotations, "text"), console.log("highlight annotations length: " + annotations.length)
extraNgramList = angular.copy($rootScope.extraNgramList);
// reinitialize an empty list var sortedSizeAnnotations = lengthSort(annotations, "text")
extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = [];
});
// rl: £dbg counters // rl: £dbg counters
var i = 0 ; var i = 0 ;
...@@ -506,6 +503,9 @@ ...@@ -506,6 +503,9 @@
var nbMatches = matches ? eltLongtext.match(myPattern).length : 0 var nbMatches = matches ? eltLongtext.match(myPattern).length : 0
if (nbMatches > 0) { if (nbMatches > 0) {
k += nbMatches ; k += nbMatches ;
// remember that this annotation.text matched
isDisplayedIntraText[annotation.uuid] = annotation
l ++ ; l ++ ;
// ------------------------------------------------------------ // ------------------------------------------------------------
// ICI we update each time // ICI we update each time
...@@ -549,81 +549,72 @@ ...@@ -549,81 +549,72 @@
// highlight anchors as html spans // highlight anchors as html spans
// ------------------------------- // -------------------------------
angular.forEach(textMapping, function(text, eltId) { angular.forEach(textMapping, function(textContent, eltId) {
// console.log(anchorPattern) // console.log(anchorPattern)
if(text) { if(textContent) {
textMapping[eltId] = replaceAnchorByTemplate( textMapping[eltId] = replaceAnchorByTemplate(
text, textContent,
annotation, annotation,
template, template,
anchorPattern); anchorPattern);
} }
}); });
// rloth: for now let's show *all* ngrams of the active list
// in the left side
extraNgramList[annotation.list_id] = extraNgramList[annotation.list_id].concat(annotation);
}); });
// let's show just the ngrams that matched
// in the left side
var sortedDisplayedKeys = Object.keys(isDisplayedIntraText).sort()
// sorts on ngram_id
// new update ngramsInPanel
angular.forEach(sortedDisplayedKeys, function(id) {
var the_annot = isDisplayedIntraText[id] ;
var the_list_id = the_annot.list_id ;
$rootScope.ngramsInPanel[the_list_id].push(the_annot)
});
// debug
//console.warn("$rootScope.ngramsInPanel :")
//console.warn($rootScope.ngramsInPanel)
// update extraNgramList
$rootScope.extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = lengthSort(extraNgramList[id], 'text');
});
// return the object of element ID with the corresponding HTML // return the object of element ID with the corresponding HTML
return textMapping; return textMapping;
} }
/*
* main refresh
*/
$rootScope.refreshDisplay = function() { $rootScope.refreshDisplay = function() {
console.log("annotations.highlight.refreshDisplay()") console.log("annotations.highlight.refreshDisplay()")
if ($rootScope.annotations === undefined) return; if ($rootScope.annotations === undefined) return;
if ($rootScope.activeLists === undefined) return; if ($rootScope.activeLists === undefined) return;
if (_.keys($rootScope.activeLists).length === 0) return; if (_.keys($rootScope.activeLists).length === 0) return;
// initialize extraNgramList // initialize ngramsInPanel
var extraNgramList = {}; // ------------------------
$rootScope.extraNgramList = angular.forEach($rootScope.activeLists, function(name, id) { // $rootScope.ngramsInPanel = {
this[id] = []; // activelist1_id : [
}, extraNgramList); // annotation_a,
$rootScope.extraNgramList = extraNgramList; // annotation_b,
// annotation_c
// ] ,
// activelist2_id : [
// annotation_x,
// annotation_y,
// annotation_z
// ] ,
// ....
// }
//
var ngramsInPanel = {};
$rootScope.ngramsInPanel = angular.forEach($rootScope.activeLists, function(name, list_id) {
this[list_id] = [];
}, ngramsInPanel);
$rootScope.ngramsInPanel = ngramsInPanel;
/* /*
* Transform text into HTML with higlighted ngrams * Transform text into HTML with higlighted ngrams via compileNgramsHtml
*/ */
var result = compileNgramsHtml( var result = compileNgramsHtml(
$rootScope.annotations, $rootScope.annotations,
...@@ -643,6 +634,8 @@ ...@@ -643,6 +634,8 @@
angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true))); angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true)));
}); });
} }
/* /*
* Listen changes on the ngram data * Listen changes on the ngram data
*/ */
......
annotations/static/annotations/ngramlist.js
View file @ f4599719
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
// (then used for left-side flatlist AND inline annots) // (then used for left-side flatlist AND inline annots)
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()]; $rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
// TODO £NEW : lookup obj[list_id][term_text] = {terminfo} // TODO £NEW : lookup obj[list_id][term_text] = {terminfo}
// $rootScope.lookup = // $rootScope.lookup =
$rootScope.refreshDisplay(); $rootScope.refreshDisplay();
}, },
function(data) { function(data) {
...@@ -49,7 +49,7 @@ ...@@ -49,7 +49,7 @@
annotationsAppNgramList.controller('NgramListPaginationController', annotationsAppNgramList.controller('NgramListPaginationController',
['$scope', '$rootScope', function ($scope, $rootScope) { ['$scope', '$rootScope', function ($scope, $rootScope) {
$rootScope.$watchCollection('extraNgramList', function (newValue, oldValue) { $rootScope.$watchCollection('ngramsInPanel', function (newValue, oldValue) {
$scope.currentListPage = 0; $scope.currentListPage = 0;
$scope.pageSize = 15; $scope.pageSize = 15;
...@@ -62,8 +62,8 @@ ...@@ -62,8 +62,8 @@
}; };
$scope.totalListPages = function(listId) { $scope.totalListPages = function(listId) {
if ($rootScope.extraNgramList[listId] === undefined) return 0; if ($rootScope.ngramsInPanel[listId] === undefined) return 0;
return Math.ceil($rootScope.extraNgramList[listId].length / $scope.pageSize); return Math.ceil($rootScope.ngramsInPanel[listId].length / $scope.pageSize);
}; };
}); });
}]); }]);
...@@ -93,12 +93,12 @@ ...@@ -93,12 +93,12 @@
var value = angular.element(inputEltId).val().trim(); var value = angular.element(inputEltId).val().trim();
if (value === "") return; if (value === "") return;
// £TEST locally check if already in annotations NodeNgrams ------ // £TEST locally check if already in annotations NodeNgrams ------
// $rootScope.annotations = array of ngram objects like: // $rootScope.annotations = array of ngram objects like:
// {"list_id":805,"occurrences":2,"uuid":9386,"text":"petit échantillon"} // {"list_id":805,"occurrences":2,"uuid":9386,"text":"petit échantillon"}
console.log('looking for "' + value + '" in list:' + listId) console.log('looking for "' + value + '" in list:' + listId)
var already_in_list = false ; var already_in_list = false ;
angular.forEach($rootScope.annotations, function(annot,i) { angular.forEach($rootScope.annotations, function(annot,i) {
...@@ -112,7 +112,7 @@ ...@@ -112,7 +112,7 @@
); );
if (already_in_list) { return ; } if (already_in_list) { return ; }
// --------------------------------------------------------------- // ---------------------------------------------------------------
// will check if there's a preexisting ngramId for this value // will check if there's a preexisting ngramId for this value
// TODO: if maplist => also add to miam // TODO: if maplist => also add to miam
NgramHttpService.post( NgramHttpService.post(
...@@ -136,11 +136,11 @@ ...@@ -136,11 +136,11 @@
}, },
function(data) { function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()]; $rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
// TODO £NEW : lookup obj[list_id][term_text] = {terminfo} // TODO £NEW : lookup obj[list_id][term_text] = {terminfo}
// $rootScope.lookup = // $rootScope.lookup =
$rootScope.refreshDisplay(); $rootScope.refreshDisplay();
}, },
function(data) { function(data) {
......
annotations/templates/annotations/main.html
View file @ f4599719
...@@ -32,12 +32,12 @@ ...@@ -32,12 +32,12 @@
</ul> </ul>
<div class="tab-content"> <div class="tab-content">
<div ng-controller="NgramListPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}"> <div ng-controller="NgramListPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}">
<div ng-if="extraNgramList[listId].length == 0" class="alert alert-info" role="alert"> <div ng-if="ngramsInPanel[listId].length == 0" class="alert alert-info" role="alert">
Input any keyword you want to link to this article and the list named '{[{listName}]}' Input any keyword you want to link to this article and the list named '{[{listName}]}'
</div> </div>
<ul class="list-group words-list clearfix"> <ul class="list-group words-list clearfix">
<li ng-repeat="keyword in extraNgramList[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="keyword-group-item"> <li ng-repeat="keyword in ngramsInPanel[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="keyword-group-item">
<div ng-controller="NgramController" keyword-template class="keyword-container"></div> <div ng-controller="NgramController" keyword-template class="keyword-container"></div>
</li> </li>
</ul> </ul>
......
annotations/views.py
View file @ f4599719
...@@ -14,7 +14,7 @@ from rest_framework.authentication import SessionAuthentication, BasicAuthentica ...@@ -14,7 +14,7 @@ from rest_framework.authentication import SessionAuthentication, BasicAuthentica
# 2016-03-24: refactoring, new paths # 2016-03-24: refactoring, new paths
from gargantext.models.ngrams import Node, NodeNgram, Ngram from gargantext.models.ngrams import Node, NodeNgram, Ngram
from gargantext.util.db import session from gargantext.util.db import session, aliased
from gargantext.util.db_cache import cache from gargantext.util.db_cache import cache
from gargantext.util.http import requires_auth from gargantext.util.http import requires_auth
...@@ -47,16 +47,25 @@ class NgramList(APIView): ...@@ -47,16 +47,25 @@ class NgramList(APIView):
doc_ngram_list = [] doc_ngram_list = []
lists = {} lists = {}
corpus_nod = cache.Node[corpus_id]
doc_nod = cache.Node[doc_id]
scores_nod = corpus_nod.children(typename="OCCURRENCES").first()
for list_type in ['MAINLIST', 'MAPLIST', 'STOPLIST']: for list_type in ['MAINLIST', 'MAPLIST', 'STOPLIST']:
corpus_nod = cache.Node[corpus_id]
list_nod = corpus_nod.children(typename=list_type).first() list_nod = corpus_nod.children(typename=list_type).first()
list_id = list_nod.id list_id = list_nod.id
lists["%s" % list_id] = list_type lists["%s" % list_id] = list_type
ListsTable = aliased(NodeNgram)
# doc_nod.ngrams iff we just need the occurrences in the doc (otherwise do manually)
q = doc_nod.ngrams.join(ListsTable).filter(ListsTable.node_id == list_id)
# add to results # add to results
doc_ngram_list += [(obj.id, obj.terms, w, list_id) for (w,obj) in list_nod.ngrams.all()] doc_ngram_list += [(obj.id, obj.terms, w, list_id) for (w,obj) in q.all()]
print("annotations.views.NgramList.doc_ngram_list: ", doc_ngram_list) # debug
# print("annotations.views.NgramList.doc_ngram_list: ", doc_ngram_list)
data = { '%s' % corpus_id : { data = { '%s' % corpus_id : {
'%s' % doc_id : '%s' % doc_id :
[ [
......
gargantext/constants.py
View file @ f4599719
...@@ -212,17 +212,17 @@ RESOURCETYPES = [ ...@@ -212,17 +212,17 @@ RESOURCETYPES = [
# linguistic extraction parameters --------------------------------------------- # linguistic extraction parameters ---------------------------------------------
DEFAULT_TFIDF_CUTOFF_RATIO = .45 # MAINLIST maximum terms in % DEFAULT_TFIDF_CUTOFF_RATIO = .75 # MAINLIST maximum terms in %
DEFAULT_TFIDF_HARD_LIMIT = 750 # MAINLIST maximum terms abs DEFAULT_TFIDF_HARD_LIMIT = 5000 # MAINLIST maximum terms abs
# (makes COOCS larger ~ O(N²) /!\) # (makes COOCS larger ~ O(N²) /!\)
DEFAULT_COOC_THRESHOLD = 2 # inclusive minimum for COOCS coefs DEFAULT_COOC_THRESHOLD = 2 # inclusive minimum for COOCS coefs
# (makes COOCS more sparse) # (makes COOCS more sparse)
DEFAULT_MAPLIST_MAX = 300 # MAPLIST maximum terms DEFAULT_MAPLIST_MAX = 350 # MAPLIST maximum terms
DEFAULT_MAPLIST_MONOGRAMS_RATIO = .5 # part of monograms in MAPLIST DEFAULT_MAPLIST_MONOGRAMS_RATIO = .15 # part of monograms in MAPLIST
DEFAULT_MAX_NGRAM_LEN = 7 # limit used after POStagging rule DEFAULT_MAX_NGRAM_LEN = 7 # limit used after POStagging rule
# (initial ngrams number is a power law of this /!\) # (initial ngrams number is a power law of this /!\)
......
gargantext/util/parsers/CSV.py
View file @ f4599719
...@@ -124,7 +124,8 @@ class CSVParser(Parser): ...@@ -124,7 +124,8 @@ class CSVParser(Parser):
for columnum in range( Coords["column"],len(tokens) ): for columnum in range( Coords["column"],len(tokens) ):
data = tokens[columnum] data = tokens[columnum]
RecordDict[ Headers_Int2Str[columnum] ] = data RecordDict[ Headers_Int2Str[columnum] ] = data
hyperdata_list.append( RecordDict ) if len(RecordDict.keys())>0:
hyperdata_list.append( RecordDict )
# # = = = = [ / Reading the whole CSV and saving ] = = = = # # # = = = = [ / Reading the whole CSV and saving ] = = = = #
return hyperdata_list return hyperdata_list
gargantext/util/toolchain/list_stop.py
View file @ f4599719
...@@ -27,10 +27,10 @@ def is_stop_word(ngram, stop_words=None): ...@@ -27,10 +27,10 @@ def is_stop_word(ngram, stop_words=None):
# , "(.*)(\.)(.*)" trop fort (enlève les sigles !) # , "(.*)(\.)(.*)" trop fort (enlève les sigles !)
, "(.*)(\,)(.*)" , "(.*)(\,)(.*)"
, "(.*)(< ?/?p ?>)(.*)" # marques de paragraphes , "(.*)(< ?/?p ?>)(.*)" # marques de paragraphes
, "(.*)(study)(.*)" , "(.*)(study|elsevier)(.*)"
, "(.*)\b(xx|xi|xv)\b(.*)" , "(.*)\b(xx|xi|xv)\b(.*)"
, "(.*)(result)(.*)" , "(.*)(result)(.*)"
, "(.*)(année|nombre|moitié)(.*)" , "(.*)(year|année|nombre|moitié)(.*)"
, "(.*)(temps)(.*)" , "(.*)(temps)(.*)"
, "(.*)(%)(.*)" , "(.*)(%)(.*)"
, "(.*)(\{)(.*)" , "(.*)(\{)(.*)"
......
gargantext/views/api/nodes.py
View file @ f4599719
...@@ -84,9 +84,10 @@ class NodeListResource(APIView): ...@@ -84,9 +84,10 @@ class NodeListResource(APIView):
response = HttpResponse(content_type='text/csv') response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="Gargantext_Corpus.csv"' response['Content-Disposition'] = 'attachment; filename="Gargantext_Corpus.csv"'
writer = csv.writer(response) writer = csv.writer(response, delimiter='\t')
keys = [ 'title' , 'journal', 'publication_date' keys = [ 'title' , 'journal'
, 'publication_year', 'publication_month', 'publication_day'
, 'abstract', 'authors'] , 'abstract', 'authors']
writer.writerow(keys) writer.writerow(keys)
......
static/js/gargantext/Docs_dyna_chart_and_table.js
View file @ f4599719
...@@ -170,7 +170,7 @@ function toggleFavstatus (rec_id) { ...@@ -170,7 +170,7 @@ function toggleFavstatus (rec_id) {
var myHttpAction = statusBefore ? 'DELETE' : 'PUT' var myHttpAction = statusBefore ? 'DELETE' : 'PUT'
$.ajax({ $.ajax({
url: 'http://localhost:8000/api/nodes/'+corpus_id+'/favorites?docs='+doc_id, url: window.location.origin + '/api/nodes/'+corpus_id+'/favorites?docs='+doc_id,
type: myHttpAction, type: myHttpAction,
beforeSend: function(xhr) { beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken")); xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
...@@ -600,7 +600,7 @@ $.ajax({ ...@@ -600,7 +600,7 @@ $.ajax({
success: function(maindata){ success: function(maindata){
// unfortunately favorites info is in a separate request (other nodes) // unfortunately favorites info is in a separate request (other nodes)
$.ajax({ $.ajax({
url: 'http://localhost:8000/api/nodes/'+corpus_id+'/favorites', url: window.location.origin + '/api/nodes/'+corpus_id+'/favorites',
success: function(favdata){ success: function(favdata){
// initialize favs lookup // initialize favs lookup
for (var i in favdata['favdocs']) { for (var i in favdata['favdocs']) {
......
templates/pages/main/about.html
View file @ f4599719
...@@ -41,7 +41,7 @@ ...@@ -41,7 +41,7 @@
<li> <li>
Version 3.0.0 Version 3.0.0
<ul> <ul>
<li>[NAME] Blue Jasmine</li> <li>[NAME] Blue Jasmin</li>
<li>[CODE] Refactored</li> <li>[CODE] Refactored</li>
<li>[DATABASE] New schema</li> <li>[DATABASE] New schema</li>
</ul> </ul>
......
templates/pages/menu.html
View file @ f4599719
...@@ -151,15 +151,17 @@ ...@@ -151,15 +151,17 @@
<br> <br>
<br> <br>
<div class="row"> <div class="row">
<div class="col-md-3"> <h3>
<h3> <a href="/projects/{{project.id}}">
<a href="/projects/{{project.id}}"> <span class="glyphicon glyphicon-book" aria-hidden="true"></span>
<span class="glyphicon glyphicon-book" aria-hidden="true"></span> {{ project.name | truncatechars:50}}
{{ project.name | truncatechars:15}} </a>
</a> </h3>
</h3> </div>
<div class="row">
<div class="col-md-1">
</div> </div>
<div class="col-md-5"> <div class="col-md-6">
<h3> <h3>
<span class="glyphicon glyphicon-cd" aria-hidden="true"></span> <span class="glyphicon glyphicon-cd" aria-hidden="true"></span>
{{ resourcename | truncatechars:20 }} {{ resourcename | truncatechars:20 }}
...@@ -167,23 +169,20 @@ ...@@ -167,23 +169,20 @@
<h3> <h3>
<span class="glyphicon glyphicon-file" aria-hidden="true"></span> <span class="glyphicon glyphicon-file" aria-hidden="true"></span>
{{ corpus.name | truncatechars:20 }} {{ corpus.name | truncatechars:20 }}
<a class="btn btn-default" role="button" href="/api/nodes?parent_id={{corpus.id}}&types[]=DOCUMENT&pagination_limit=100000&formated=csv"> <a class="btn btn-primary" role="button" href="/api/nodes?parent_id={{corpus.id}}&types[]=DOCUMENT&pagination_limit=100000&formated=csv">
<span class="glyphicon glyphicon-download" aria-hidden="true"></span> <span class="glyphicon glyphicon-download" aria-hidden="true"></span>
</a> </a>
</h3> </h3>
</div> </div>
<div class="col-md-4"> <div class="col-md-5">
<h3> <h3>
<span class="glyphicon glyphicon-calendar" aria-hidden="true"></span> {{ corpus.date }} <span class="glyphicon glyphicon-calendar" aria-hidden="true"></span> {{ corpus.date }}
</h3> </h3>
<h3> <h3>
<span class="glyphicon glyphicon-user" aria-hidden="true"></span> <span class="glyphicon glyphicon-user" aria-hidden="true"></span>
Author(s): Author(s): {{ user.username | truncatechars:15}}
</h3> </h3>
<h4>
{{ user.username | truncatechars:15}}
</h4>
</div> </div>
</div> </div>
</div> </div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment