Skip to content

gargantext
gargantext
Commit 25584756 authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into unstable-mat

parents d6cc0e32 225cb308
Show whitespace changes
Inline Side-by-side
Showing with 830 additions and 614 deletions
analysis/functions.py
View file @ 25584756
...@@ -21,7 +21,10 @@ def create_synonymes(user, corpus): ...@@ -21,7 +21,10 @@ def create_synonymes(user, corpus):
size = 1000 size = 1000
def create_whitelist(user, corpus_id, size=size, count_min=2): def create_whitelist(user, corpus_id, size=size, count_min=2, miam_id=None):
if miam_id is None:
PrintException()
cursor = connection.cursor() cursor = connection.cursor()
whitelist_type_id = cache.NodeType['WhiteList'].id whitelist_type_id = cache.NodeType['WhiteList'].id
...@@ -53,10 +56,14 @@ def create_whitelist(user, corpus_id, size=size, count_min=2): ...@@ -53,10 +56,14 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
node_node_ngram AS nngX ON nngX.node_id = n.id node_node_ngram AS nngX ON nngX.node_id = n.id
INNER JOIN INNER JOIN
node_ngram AS ngX ON ngX.id = nngX.ngram_id node_ngram AS ngX ON ngX.id = nngX.ngram_id
INNER JOIN
node_node_ngram AS miam ON ngX.id = miam.ngram_id
WHERE WHERE
n.parent_id = %d n.parent_id = %d
AND AND
n.type_id = %d n.type_id = %d
AND
miam.node_id = %d
AND AND
ngX.n >= 2 ngX.n >= 2
AND AND
...@@ -72,7 +79,7 @@ def create_whitelist(user, corpus_id, size=size, count_min=2): ...@@ -72,7 +79,7 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
LIMIT LIMIT
%d %d
; ;
""" % (white_list.id, int(corpus_id), int(type_document_id), count_min, size) """ % (white_list.id, int(corpus_id), int(type_document_id), int(miam_id), count_min, size)
# print("PRINTING QYERY OF WHITELIST:") # print("PRINTING QYERY OF WHITELIST:")
# print(query_whitelist) # print(query_whitelist)
cursor.execute(query_whitelist) cursor.execute(query_whitelist)
...@@ -155,6 +162,7 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size= ...@@ -155,6 +162,7 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
from gargantext_web.api import JsonHttpResponse from gargantext_web.api import JsonHttpResponse
from analysis.louvain import best_partition from analysis.louvain import best_partition
from ngram.lists import listIds
#print(corpus_id, cooc_id) #print(corpus_id, cooc_id)
...@@ -168,7 +176,9 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size= ...@@ -168,7 +176,9 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
if session.query(Node).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first() is None: if session.query(Node).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first() is None:
print("Coocurrences do not exist yet, create it.") print("Coocurrences do not exist yet, create it.")
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size) miam_id = listIds(typeList='MiamList', user_id=request.user.id, corpus_id=corpus_id)[0][0]
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size, miam_id=miam_id)
cooccurrence_node_id = create_cooc(user=request.user, corpus_id=corpus_id, whitelist=whitelist, size=size) cooccurrence_node_id = create_cooc(user=request.user, corpus_id=corpus_id, whitelist=whitelist, size=size)
else: else:
cooccurrence_node_id = session.query(Node.id).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first() cooccurrence_node_id = session.query(Node.id).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first()
...@@ -211,8 +221,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size= ...@@ -211,8 +221,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
n = n.sort(inplace=False) n = n.sort(inplace=False)
m = m.sort(inplace=False) m = m.sort(inplace=False)
print(n) #print(n)
print(m) #print(m)
nodes_included = 300 #int(round(size/20,0)) nodes_included = 300 #int(round(size/20,0))
#nodes_excluded = int(round(size/10,0)) #nodes_excluded = int(round(size/10,0))
...@@ -237,25 +247,25 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size= ...@@ -237,25 +247,25 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
xxx = xx.values xxx = xx.values
threshold = min(xxx.max(axis=1)) threshold = min(xxx.max(axis=1))
matrix_filtered = np.where(xxx >= threshold, xxx, 0) matrix_filtered = np.where(xxx >= threshold, xxx, 0)
#matrix_filtered = matrix_filtered.resize((90,90)) #matrix_filtered = matrix_filtered.resize((90,90))
except: except:
PrintException() PrintException()
try: try:
G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph()) G = nx.from_numpy_matrix(np.matrix(matrix_filtered))
G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)]))) #G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph())
#print(G) G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
# Removing too connected nodes (find automatic way to do it) # Removing too connected nodes (find automatic way to do it)
#edges_to_remove = [ e for e in G.edges_iter() if #edges_to_remove = [ e for e in G.edges_iter() if
degree = G.degree() degree = G.degree()
nodes_to_remove = [n for n in degree if degree[n] <= 1] nodes_to_remove = [n for n in degree if degree[n] <= 1]
G.remove_nodes_from(nodes_to_remove) G.remove_nodes_from(nodes_to_remove)
uG = G.to_undirected()
partition = best_partition(G) partition = best_partition(uG)
except: except:
print("-" * 30)
PrintException() PrintException()
...@@ -271,7 +281,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size= ...@@ -271,7 +281,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
G.node[node]['group'] = partition[node] G.node[node]['group'] = partition[node]
# G.add_edge(node, "cluster " + str(partition[node]), weight=3) # G.add_edge(node, "cluster " + str(partition[node]), weight=3)
except Exception as error: except Exception as error:
print("error01: ",error) pass#PrintException()
#print("error01: ",error)
data = json_graph.node_link_data(G) data = json_graph.node_link_data(G)
......
annotations/static/annotations/app.css
View file @ 25584756
/* app css stylesheet */ /* app css stylesheet */
/*
* Class names corresponding to server-side list names
* To display another list name, please add a new class under this
*/
.MiamList {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.StopList {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.delete-keyword, .occurrences { .delete-keyword, .occurrences {
vertical-align: super; vertical-align: super;
font-size: 70%; font-size: 70%;
...@@ -27,27 +43,9 @@ ...@@ -27,27 +43,9 @@
border-bottom: none; border-bottom: none;
} }
.miamword {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.stopword {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.global-stopword {
color: black;
background-color: rgba(169, 68, 66, 0.05);
cursor: pointer;
}
.main-panel, .text-panel, .words-panel { .main-panel, .text-panel, .words-panel {
height: 800px; height: 800px;
margin: 10px 0px; margin: 10px 0;
} }
.text-panel { .text-panel {
...@@ -59,11 +57,7 @@ ...@@ -59,11 +57,7 @@
height: 250px; height: 250px;
} }
.keyword-container { .keyword-text {
/*display: inline-block;*/
}
.keyword {
word-break: break-all; word-break: break-all;
} }
......
annotations/static/annotations/app.js
View file @ 25584756
(function () { (function () {
'use strict'; 'use strict';
/*
* Django STATIC_URL given to angular to load async resources
*/
var S = window.STATIC_URL; var S = window.STATIC_URL;
window.annotationsApp = angular.module('annotationsApp', ['annotationsAppHttp']); window.annotationsApp = angular.module('annotationsApp', ['annotationsAppHttp']);
/*
* Angular Templates must not conflict with Django's
*/
window.annotationsApp.config(function($interpolateProvider) { window.annotationsApp.config(function($interpolateProvider) {
$interpolateProvider.startSymbol('{[{'); $interpolateProvider.startSymbol('{[{');
$interpolateProvider.endSymbol('}]}'); $interpolateProvider.endSymbol('}]}');
}); });
/*
* Template of the ngram element displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.directive('keywordTemplate', function () { window.annotationsApp.directive('keywordTemplate', function () {
return { return {
templateUrl: function ($element, $attributes) { templateUrl: function ($element, $attributes) {
...@@ -18,25 +27,36 @@ ...@@ -18,25 +27,36 @@
}; };
}); });
/*
* For ngram elements displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.controller('ExtraAnnotationController', window.annotationsApp.controller('ExtraAnnotationController',
['$scope', '$rootScope', '$element', 'NgramHttpService', ['$scope', '$rootScope', '$element', 'NgramHttpService',
function ($scope, $rootScope, $element, NgramHttpService) { function ($scope, $rootScope, $element, NgramHttpService) {
// TODO use the tooltip ? /*
* Click on the 'delete' cross button
*/
$scope.onDeleteClick = function () { $scope.onDeleteClick = function () {
NgramHttpService.delete({ NgramHttpService.delete({
'listId': $scope.keyword.list_id, 'listId': $scope.keyword.list_id,
'ngramId': $scope.keyword.uuid 'ngramId': $scope.keyword.uuid
}).$promise.then(function(data) { }, function(data) {
$.each($rootScope.annotations, function(index, element) { $.each($rootScope.annotations, function(index, element) {
if (element.list_id == $scope.keyword.list_id && element.uuid == $scope.keyword.uuid) { if (element.list_id == $scope.keyword.list_id && element.uuid == $scope.keyword.uuid) {
$rootScope.annotations.splice(index, 1); $rootScope.annotations.splice(index, 1);
return false; return false;
} }
}); });
}, function(data) {
console.log(data);
console.error("unable to delete the Ngram " + $scope.keyword.uuid);
}); });
} };
}]); }]);
/*
* For mouse selection on the text
*/
window.annotationsApp.controller('AnnotationController', window.annotationsApp.controller('AnnotationController',
['$scope', '$rootScope', '$element', ['$scope', '$rootScope', '$element',
function ($scope, $rootScope, $element) { function ($scope, $rootScope, $element) {
...@@ -57,14 +77,9 @@ ...@@ -57,14 +77,9 @@
}; };
}]); }]);
window.annotationsApp.directive('selectionTemplate', function () { /*
return { * Controller of the menu over the current mouse selection
templateUrl: function ($element, $attributes) { */
return S + 'annotations/selection_tpl.html';
}
};
});
window.annotationsApp.controller('AnnotationMenuController', window.annotationsApp.controller('AnnotationMenuController',
['$scope', '$rootScope', '$element', '$timeout', 'NgramHttpService', ['$scope', '$rootScope', '$element', '$timeout', 'NgramHttpService',
function ($scope, $rootScope, $element, $timeout, NgramHttpService) { function ($scope, $rootScope, $element, $timeout, NgramHttpService) {
...@@ -87,9 +102,12 @@ ...@@ -87,9 +102,12 @@
} }
return false; return false;
} }
// we only need one singleton at a time
var selection = getSelected(); var selection = getSelected();
/*
* When mouse selection is started, we highlight it
*/
function toggleSelectionHighlight(text) { function toggleSelectionHighlight(text) {
if (text.trim() !== "") { if (text.trim() !== "") {
$(".text-panel").addClass("selection"); $(".text-panel").addClass("selection");
...@@ -98,53 +116,68 @@ ...@@ -98,53 +116,68 @@
} }
} }
/*
* Dynamically construct the selection menu scope
*/
function toggleMenu(context, annotation) { function toggleMenu(context, annotation) {
$timeout(function() { $timeout(function() {
$scope.$apply(function() { $scope.$apply(function() {
var miamlist_id = _.invert($rootScope.activeLists).MiamList;
if (angular.isObject(annotation)) { var stoplist_id = _.invert($rootScope.activeLists).StopList;
$scope.level = angular.copy(annotation.level || 'global'); // variable used in onClick
$scope.category = $rootScope.lists[annotation.list_id].toLowerCase();
$scope.listId = angular.copy(annotation.list_id);
// used in onClick
$scope.selection_text = angular.copy(annotation); $scope.selection_text = angular.copy(annotation);
if ($scope.category == "miamlist") { if (angular.isObject(annotation)) {
$scope.local_miamlist = false; // existing ngram
$scope.global_stoplist = true; // Delete from the current list
$scope.local_stoplist = true; $scope.menuItems = [
} else if ($scope.category == "stoplist") { {
'action': 'delete',
if ($scope.level == "local") { 'listId': annotation.list_id,
$scope.local_stoplist = false; 'verb': 'Delete from',
$scope.global_stoplist = true; 'listName': $rootScope.lists[annotation.list_id]
} }
if ($scope.level == "global") { ];
$scope.global_stoplist = false; if ($rootScope.lists[annotation.list_id] == "MiamList") {
$scope.local_stoplist = true; // Add to the alternative list
} $scope.menuItems.push({
$scope.local_miamlist = true; 'action': 'post',
'listId': stoplist_id,
'verb': 'Add to',
'listName': $rootScope.lists[stoplist_id]
});
} else if ($rootScope.lists[annotation.list_id] == "StopList") {
// Add to the alternative list
$scope.menuItems.push({
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.lists[miamlist_id]
});
} }
// show menu // show the menu
$element.fadeIn(100); $element.fadeIn(100);
} else if (annotation.trim() !== "") {
// new ngram
$scope.menuItems = [
{
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.activeLists[miamlist_id]
} }
else if (annotation.trim() !== "") { ];
$scope.selection_text = angular.copy(annotation); // show the menu
$scope.level = "New Ngram from selection";
$scope.category = null;
$scope.local_miamlist = true;
$scope.local_stoplist = true;
$scope.global_stoplist = true;
// show menu
$element.fadeIn(100); $element.fadeIn(100);
} else { } else {
// close menu $scope.menuItems = [];
// close the menu
$element.fadeOut(100); $element.fadeOut(100);
} }
}); });
}); });
} }
var elt = $(".text-panel")[0];
var pos = $(".text-panel").position(); var pos = $(".text-panel").position();
function positionElement(context, x, y) { function positionElement(context, x, y) {
...@@ -157,18 +190,26 @@ ...@@ -157,18 +190,26 @@
positionElement(null, e.pageX, e.pageY); positionElement(null, e.pageX, e.pageY);
} }
// TODO is mousedown necessary ? /*
$(".text-panel").mousedown(function(){ * Dynamically position the menu
$(".text-panel").mousemove(positionMenu); */
$(".text-container").mousedown(function(){
$(".text-container").mousemove(positionMenu);
}); });
$(".text-panel").mouseup(function(){ /*
$(".text-panel").unbind("mousemove", positionMenu); * Finish positioning the menu then display the menu
*/
$(".text-container").mouseup(function(){
$(".text-container").unbind("mousemove", positionMenu);
toggleSelectionHighlight(selection.toString().trim()); toggleSelectionHighlight(selection.toString().trim());
toggleMenu(null, selection.toString().trim()); toggleMenu(null, selection.toString().trim());
}); });
$(".text-panel").delegate(':not("#selection")', "click", function(e) { /*
* Toggle the menu when clicking on an existing ngram keyword
*/
$(".text-container").delegate(':not("#selection")', "click", function(e) {
if ($(e.target).hasClass("keyword-inline")) return; if ($(e.target).hasClass("keyword-inline")) return;
positionMenu(e); positionMenu(e);
toggleSelectionHighlight(selection.toString().trim()); toggleSelectionHighlight(selection.toString().trim());
...@@ -178,81 +219,80 @@ ...@@ -178,81 +219,80 @@
$rootScope.$on("positionAnnotationMenu", positionElement); $rootScope.$on("positionAnnotationMenu", positionElement);
$rootScope.$on("toggleAnnotationMenu", toggleMenu); $rootScope.$on("toggleAnnotationMenu", toggleMenu);
$scope.onClick = function($event, action, listId, level) { /*
* Menu click action
*/
$scope.onMenuClick = function($event, action, listId) {
if (angular.isObject($scope.selection_text)) { if (angular.isObject($scope.selection_text)) {
// delete from the current list // action on an existing Ngram
NgramHttpService[action]({ NgramHttpService[action]({
'listId': listId, 'listId': listId,
'ngramId': $scope.selection_text.uuid 'ngramId': $scope.selection_text.uuid
}).$promise.then(function(data) { }, function(data) {
$.each($rootScope.annotations, function(index, element) { $.each($rootScope.annotations, function(index, element) {
if (element.list_id == listId && element.uuid == $scope.selection_text.uuid) { if (element.list_id == listId && element.uuid == $scope.selection_text.uuid) {
$rootScope.annotations.splice(index, 1); $rootScope.annotations.splice(index, 1);
return false; return false;
} }
}); });
}); }, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
} else if ($scope.selection_text.trim() !== "") { } else if ($scope.selection_text.trim() !== "") {
// new annotation from selection // new annotation from selection
NgramHttpService.post( NgramHttpService.post(
{ {
'listId': listId 'listId': listId,
'ngramId': 'new'
}, },
{'annotation' : {'text': $scope.selection_text.trim()}} {
).$promise.then(function(data) { 'annotation' : {'text': $scope.selection_text.trim()}
}, function(data) {
$rootScope.annotations.push(data); $rootScope.annotations.push(data);
}); }, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
} }
// hide selection highlighted text and the menu // hide the highlighted text the the menu
$(".text-panel").removeClass("selection"); $(".text-panel").removeClass("selection");
$element.fadeOut(100); $element.fadeOut(100);
}; };
} }
]); ]);
/*
* Text highlighting controller
*/
window.annotationsApp.controller('IntraTextController', window.annotationsApp.controller('IntraTextController',
['$scope', '$rootScope', '$compile', 'NgramHttpService', ['$scope', '$rootScope', '$compile', 'NgramHttpService',
function ($scope, $rootScope, $compile, NgramHttpService) { function ($scope, $rootScope, $compile, NgramHttpService) {
$scope.extra_stoplist = [];
$scope.extra_miamlist = [];
$scope.currentStopPage = 0;
$scope.currentMiamPage = 0;
$scope.pageSize = 15;
var counter = 0; var counter = 0;
/* /*
* Replace the text by and html template * Replace the text by an html template for ngram keywords
*/ */
function replaceTextByTemplate(text, annotation, template, pattern, lists) { function replaceTextByTemplate(text, ngram, template, pattern, lists) {
return text.replace(pattern, function(matched) { return text.replace(pattern, function(matched) {
var tpl = angular.element(template); var tpl = angular.element(template);
tpl.append(matched); tpl.append(matched);
tpl.attr('title', annotation.tooltip_content); tpl.attr('title', ngram.tooltip_content);
tpl.attr('uuid', annotation.uuid); tpl.attr('uuid', ngram.uuid);
/*
if ('MiamList' == lists[annotation.list_id]) tpl.addClass("miamword"); * Add CSS class depending on the list the ngram is into
if ('StopList' == lists[annotation.list_id]) tpl.addClass("stopword"); * FIXME Lists names and css classes are fixed, can do better
//if (annotation.category == 'stoplist' && annotation.level == 'global') tpl.addClass("global-stopword"); */
tpl.addClass(ngram.listName);
return tpl.get(0).outerHTML; return tpl.get(0).outerHTML;
}); });
} }
function compileText(annotations, fullText, abstractText, $rootScope) {
counter = 0;
var templateBegin = "<span ng-controller='AnnotationController' ng-click='onClick($event)' class='keyword-inline'>";
var templateBeginRegexp = "<span ng-controller='AnnotationController' ng-click='onClick\(\$event\)' class='keyword-inline'>";
var templateEnd = "</span>";
var template = templateBegin + templateEnd;
var startPattern = "\\b((?:"+templateBeginRegexp+")*";
var middlePattern = "(?:<\/span>)*\\s(?:"+templateBeginRegexp+")*";
var endPattern = "(?:<\/span>)*)\\b";
/* /*
* Sorts annotations on the number of words * Sorts annotations on the number of words
* Required for overlapping ngrams
*/ */
function lengthSort(listitems, valuekey) { function lengthSort(listitems, valuekey) {
listitems.sort(function(a, b) { listitems.sort(function(a, b) {
...@@ -262,86 +302,111 @@ ...@@ -262,86 +302,111 @@
}); });
return listitems; return listitems;
} }
/*
* Match and replace Ngram into the text
*/
function compileNgramsHtml(annotations, textMapping, $rootScope) {
// TODO remove this debug counter
counter = 0;
var templateBegin = "<span ng-controller='AnnotationController' ng-click='onClick($event)' class='keyword-inline'>";
var templateBeginRegexp = "<span ng-controller='AnnotationController' ng-click='onClick\(\$event\)' class='keyword-inline'>";
var sortedSizeAnnotations = lengthSort(annotations, "text"); var templateEnd = "</span>";
var extra_stoplist = [], var template = templateBegin + templateEnd;
extra_miamlist = [];
_.each(sortedSizeAnnotations, function (annotation) { var startPattern = "\\b((?:"+templateBeginRegexp+")*";
// TODO better split to manage two-words with minus sign var middlePattern = "(?:<\/span>)*\\s(?:"+templateBeginRegexp+")*";
annotation.category = $rootScope.lists[annotation.list_id].toLowerCase(); var endPattern = "(?:<\/span>)*)\\b";
var sortedSizeAnnotations = lengthSort(annotations, "text"),
extraNgramList = angular.copy($rootScope.extraNgramList);
// reinitialize an empty list
extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = [];
});
angular.forEach(sortedSizeAnnotations, function (annotation) {
// exclude ngrams that are into inactive lists
if ($rootScope.activeLists[annotation.list_id] === undefined) return;
// used to setup css class
annotation.listName = $rootScope.lists[annotation.list_id];
// regexps
var words = annotation.text.split(" "); var words = annotation.text.split(" ");
var pattern = new RegExp(startPattern + words.join(middlePattern) + endPattern, 'gmi'); var pattern = new RegExp(startPattern + words.join(middlePattern) + endPattern, 'gmi');
var textRegexp = new RegExp("\\b"+annotation.text+"\\b", 'igm'); var textRegexp = new RegExp("\\b"+annotation.text+"\\b", 'igm');
var isDisplayedIntraText = false;
if (pattern.test(fullText) === true) { // highlight text as html
fullText = replaceTextByTemplate(fullText, annotation, template, pattern, $rootScope.lists); angular.forEach(textMapping, function(text, eltId) {
if (pattern.test(text) === true) {
textMapping[eltId] = replaceTextByTemplate(text, annotation, template, pattern, $rootScope.lists);
// TODO remove debug // TODO remove debug
counter++; counter++;
} else if (pattern.test(abstractText) === true) { isDisplayedIntraText = true;
abstractText = replaceTextByTemplate(abstractText, annotation, template, pattern, $rootScope.lists); }
counter++; });
} else if (!textRegexp.test($rootScope.full_text) && !textRegexp.test($rootScope.abstract_text)) {
if (annotation.category == "stoplist") { if (!isDisplayedIntraText) {
// Deactivated stoplist for the moment // add extra-text ngrams that are not already displayed
// if ($.inArray(annotation.uuid, $scope.extra_stoplist.map(function (item) { if ($.inArray(annotation.uuid, extraNgramList[annotation.list_id].map(function (item) {
// return item.uuid;
// })) == -1) {
// extra_stoplist = lengthSort(extra_stoplist.concat(annotation), "text");
// }
} else if (annotation.category == "miamlist") {
if ($.inArray(annotation.uuid, $scope.extra_miamlist.map(function (item) {
return item.uuid; return item.uuid;
})) == -1) { })) == -1) {
extra_miamlist = lengthSort(extra_miamlist.concat(annotation), "text"); // push the ngram and sort
} extraNgramList[annotation.list_id] = extraNgramList[annotation.list_id].concat(annotation);
} }
} }
}); });
$scope.extra_stoplist = extra_stoplist; // update extraNgramList
$scope.extra_miamlist = extra_miamlist; $rootScope.extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = lengthSort(extraNgramList[id], 'text');
return { });
'fullTextHtml': fullText, // return the object of element ID with the corresponding HTML
'abstractTextHtml': abstractText return textMapping;
};
} }
/*
* Listen changes on the ngram data
*/
$rootScope.$watchCollection('annotations', function (newValue, oldValue) { $rootScope.$watchCollection('annotations', function (newValue, oldValue) {
if ($rootScope.annotations === undefined) return; if ($rootScope.annotations === undefined) return;
if (angular.equals(newValue, oldValue)) return; if (angular.equals(newValue, oldValue)) return;
$rootScope.miamListId = _.invert($rootScope.lists)['MiamList']; // initialize extraNgramList
$rootScope.stopListId = _.invert($rootScope.lists)['StopList']; var extraNgramList = {};
$rootScope.extraNgramList = angular.forEach($rootScope.activeLists, function(name, id) {
this[id] = [];
}, extraNgramList);
$rootScope.extraNgramList = extraNgramList;
$scope.extra_stoplist = []; /*
$scope.extra_miamlist = []; * Transform text into HTML with higlighted ngrams
*/
var result = compileText( var result = compileNgramsHtml(
$rootScope.annotations, $rootScope.annotations,
angular.copy($rootScope.full_text), {
angular.copy($rootScope.abstract_text), '#full-text': angular.copy($rootScope.full_text),
'#abstract-text': angular.copy($rootScope.abstract_text),
'#title': angular.copy($rootScope.title)
},
$rootScope $rootScope
); );
// inject highlighted HTML
$.each($rootScope.annotations, function(index, element) { angular.forEach(result, function(html, eltId) {
if (element.list_id == $rootScope.stopListId) { angular.element(eltId).html(html);
$scope.extra_stoplist.push(element);
} else if (element.list_id == $rootScope.miamListId) {
$scope.extra_miamlist.push(element);
}
}); });
// inject one Angular controller on every highlighted text element
angular.element('#full-text').html(result.fullTextHtml);
angular.element('#abstract-text').html(result.abstractTextHtml);
angular.element('.text-container').find('[ng-controller=AnnotationController]').each(function(idx, elt) { angular.element('.text-container').find('[ng-controller=AnnotationController]').each(function(idx, elt) {
angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true))); angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true)));
}); });
}); });
function submitNewAnnotation($event, inputEltId, listId) { /*
* Add a new NGram from the user input in the extra-text list
*/
$scope.onListSubmit = function ($event, listId) {
var inputEltId = "#"+ listId +"-input";
if ($event.keyCode !== undefined && $event.keyCode != 13) return; if ($event.keyCode !== undefined && $event.keyCode != 13) return;
var value = $(inputEltId).val().trim(); var value = $(inputEltId).val().trim();
if (value === "") return; if (value === "") return;
...@@ -350,47 +415,54 @@ ...@@ -350,47 +415,54 @@
'listId': listId, 'listId': listId,
'ngramId': 'new' 'ngramId': 'new'
}, },
{'annotation' : {'text': value}}, {
'annotation' : {'text': value}
},
function(data) { function(data) {
// on success // on success
if (data) { if (data) {
$rootScope.annotations.push(data); $rootScope.annotations.push(data);
$(inputEltId).val("");
} }
}); }, function(data) {
// on error
$(inputEltId).parent().addClass("has-error");
console.error("error adding Ngram "+ value);
}
);
};
$(inputEltId).val("");
} }
]);
$scope.onMiamlistSubmit = function ($event) { /*
submitNewAnnotation($event, "#miamlist-input", _.invert($rootScope.lists)['MiamList']); * Controller for one List Tab displaying extra-text ngram
}; */
// TODO refactor window.annotationsApp.controller('ExtraTextPaginationController',
$scope.onStoplistSubmit = function ($event) { ['$scope', '$rootScope', function ($scope, $rootScope) {
submitNewAnnotation($event, "#stoplist-input", _.invert($rootScope.lists)['MiamList']);
}; $rootScope.$watchCollection('extraNgramList', function (newValue, oldValue) {
$scope.numStopPages = function () { $scope.currentListPage = 0;
if ($scope.extra_stoplist === undefined) return 0; $scope.pageSize = 15;
return Math.ceil($scope.extra_stoplist.length / $scope.pageSize);
}; $scope.nextListPage = function() {
$scope.numMiamPages = function () { $scope.currentListPage = $scope.currentListPage + 1;
if ($scope.extra_miamlist === undefined) return 0;
return Math.ceil($scope.extra_miamlist.length / $scope.pageSize);
};
$scope.nextMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage + 1;
};
$scope.previousMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage - 1;
}; };
$scope.nextStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage + 1; $scope.previousListPage = function() {
$scope.currentListPage = $scope.currentListPage - 1;
}; };
$scope.previousStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage - 1; $scope.totalListPages = function (listId) {
if ($rootScope.extraNgramList[listId] === undefined) return 0;
return Math.ceil($rootScope.extraNgramList[listId].length / $scope.pageSize);
}; };
} });
]); }]);
/*
* Filter used in Ngram flat lists pagination (extra-text panel)
*/
window.annotationsApp.filter('startFrom', function () { window.annotationsApp.filter('startFrom', function () {
return function (input, start) { return function (input, start) {
if (input === undefined) return; if (input === undefined) return;
...@@ -405,28 +477,32 @@ ...@@ -405,28 +477,32 @@
$rootScope.documentResource = DocumentHttpService.get( $rootScope.documentResource = DocumentHttpService.get(
{'docId': $rootScope.docId}, {'docId': $rootScope.docId},
function(data, responseHeaders) { function(data, responseHeaders) {
$scope.title = data.title;
$scope.authors = data.authors; $scope.authors = data.authors;
$scope.journal = data.journal; $scope.journal = data.journal;
$scope.publication_date = data.publication_date; $scope.publication_date = data.publication_date;
// TODO this data have to be deleted
//$scope.current_page_number = data.current_page_number; //$scope.current_page_number = data.current_page_number;
//$scope.last_page_number = data.last_page_number; //$scope.last_page_number = data.last_page_number;
// put in rootScope because used by many components $rootScope.title = data.title;
$rootScope.docId = data.id; $rootScope.docId = data.id;
$rootScope.full_text = data.full_text; $rootScope.full_text = data.full_text;
$rootScope.abstract_text = data.abstract_text; $rootScope.abstract_text = data.abstract_text;
// GET the annotations // GET the annotationss
// TODO
$rootScope.annotationsResource = NgramListHttpService.get( $rootScope.annotationsResource = NgramListHttpService.get(
{'corpusId': $rootScope.corpusId, 'docId': $rootScope.docId} {
).$promise.then(function(data) { 'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()]; $rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.lists = data[$rootScope.corpusId.toString()]['lists']; $rootScope.lists = data[$rootScope.corpusId.toString()].lists;
}); // TODO active list selection controller
$rootScope.activeLists = angular.copy($rootScope.lists);
$rootScope.mainListId = _.invert($rootScope.activeLists).MiamList;
}
);
}); });
// TODO setup pagination client-side // TODO setup article pagination
$scope.onPreviousClick = function () { $scope.onPreviousClick = function () {
DocumentHttpService.get($scope.docId - 1); DocumentHttpService.get($scope.docId - 1);
}; };
...@@ -435,8 +511,11 @@ ...@@ -435,8 +511,11 @@
}; };
}]); }]);
/*
* Main function
* GET the document node and all its ngrams
*/
window.annotationsApp.run(function ($rootScope) { window.annotationsApp.run(function ($rootScope) {
/* GET the document node and all the annotations in the list associated */
var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//); var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//);
$rootScope.projectId = path[1]; $rootScope.projectId = path[1];
$rootScope.corpusId = path[2]; $rootScope.corpusId = path[2];
......
annotations/static/annotations/http.js
View file @ 25584756
...@@ -57,11 +57,11 @@ ...@@ -57,11 +57,11 @@
{ {
post: { post: {
method: 'POST', method: 'POST',
params: {'listId': '@listId', 'ngramId': ''} params: {'listId': '@listId', 'ngramId': '@ngramId'}
}, },
delete: { delete: {
method: 'DELETE', method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@id'} params: {'listId': '@listId', 'ngramId': '@ngramId'}
} }
} }
); );
......
annotations/static/annotations/keyword_tpl.html
View file @ 25584756
<span ng-if="keyword.category == 'miamlist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="miamlist">×</span> <span ng-click='onDeleteClick()' class="delete-keyword">×</span>
<span ng-if="keyword.category == 'miamlist'" data-toggle="tooltip" class="keyword miamword">{[{keyword.text}]}</span> <span data-toggle="tooltip" class="keyword-text {[{keyword.listName}]}">{[{keyword.text}]}</span>
<span ng-if="keyword.category == 'stoplist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="stoplist">×</span>
<span ng-if="keyword.category == 'stoplist'" data-toggle="tooltip" class="keyword stopword">{[{keyword.text}]}</span>
<span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span> <span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span>
annotations/static/annotations/selection_tpl.html deleted 100644 → 0
View file @ d6cc0e32
<ul class="noselection">
<li class="miamword" ng-if="local_miamlist === true" ng-click="onClick($event, 'post', miamListId, 'local')">add to miam-list</li>
<li class="miamword" ng-if="local_miamlist === false" ng-click="onClick($event, 'delete', miamListId, 'local')">remove from miam-list</li>
<li class="stopword" ng-if="local_stoplist === true" ng-click="onClick($event, 'post', stopListId, 'local')">add to stop-list</li>
<li class="stopword" ng-if="local_stoplist === false" ng-click="onClick($event, 'delete', stopListId, 'local')">remove from stop-list</li>
<!--<li class="stopword" ng-if="global_stoplist === true" ng-click="onClick($event, 'post', 'stoplist', 'global')">add to global stop-list</li>
<li class="stopword" ng-if="global_stoplist === false" ng-click="onClick($event, 'delete', 'stoplist', 'global')">remove from global stop-list</li>-->
</ul>
annotations/templates/annotations/main.html
View file @ 25584756
...@@ -9,8 +9,8 @@ ...@@ -9,8 +9,8 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>Gargantext corpus annotations editor</title> <title>Gargantext article editor</title>
<meta name="description" content=""> <meta name="description" content="Gargantext">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}"> <link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}"> <link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}">
...@@ -18,51 +18,38 @@ ...@@ -18,51 +18,38 @@
<script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script> <script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script>
</head> </head>
<body> <body>
<!-- TODO integrate this later into the corpus.html django template --> <!-- TODO integrate this later into the any other django template -->
<div id="annotationsApp"> <div id="annotationsApp">
<div class="container-fluid"> <div class="container-fluid">
<div class="row-fluid main-panel" ng-controller="IntraTextController"> <div class="row-fluid main-panel" ng-controller="IntraTextController">
<div class="col-md-4 col-xs-4 tabbable words-panel"> <div class="col-md-4 col-xs-4 tabbable words-panel">
<ul class="nav nav-pills nav-justified"> <ul class="nav nav-pills nav-justified">
<li class="active"><a href="#tab1" data-toggle="tab">Miamwords</a></li> <li ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}">
<li><a href="#tab2" data-toggle="tab">Stopwords</a></li> <a href="#tab-{[{listId}]}" data-toggle="tab">{[{listName}]}</a>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="tab1">
<div ng-if="extra_miamlist.length == 0" class="alert alert-info" role="alert">No extra text miam-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_miamlist | startFrom:currentMiamPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li> </li>
</ul> </ul>
<nav ng-class="{invisible: numMiamPages() - 1 == 0}" class="clearfix"> <div class="tab-content">
<ul class="pagination pagination-s pull-right words-pagination"> <div ng-controller="ExtraTextPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}">
<li ng-class="{disabled: currentMiamPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li> <div ng-if="extraNgramList[listId].length == 0" class="alert alert-info" role="alert">
<li ng-class="{disabled: currentMiamPage >= numMiamPages()-1}"><a ng-click="nextMiamPage()" class="glyphicon glyphicon-forward"></a></li> Input any keyword you want to link to this article and the list named '{[{listName}]}'
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="miamlist-input" ng-keypress="onMiamlistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onMiamlistSubmit($event)">Add</button>
</div>
</div> </div>
<div class="tab-pane" id="tab2">
<div ng-if="extra_stoplist.length == 0" class="alert alert-info" role="alert">No extra text stop-word yet</div> <ul class="list-group words-list clearfix">
<ul class="list-group words-list"> <li ng-repeat="keyword in extraNgramList[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="list-group-item">
<li ng-repeat="keyword in extra_stoplist | startFrom:currentStopPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div> <div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li> </li>
</ul> </ul>
<nav ng-class="{invisible: numStopPages() - 1 == 0}" class="clearfix">
<nav ng-class="{invisible: totalListPages - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination"> <ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentStopPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li> <li ng-class="{disabled: currentListPage == 0}"><a ng-click="previousListPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentStopPage >= numStopPages()-1}"><a ng-click="nextStopPage()" class="glyphicon glyphicon-forward"></a></li> <li ng-class="{disabled: currentListPage >= totalListPages - 1}"><a ng-click="nextListPage()" class="glyphicon glyphicon-forward"></a></li>
</ul> </ul>
</nav> </nav>
<div class="form-group"> <div class="form-group">
<input type="text" class="form-control" id="stoplist-input" ng-keypress="onStoplistSubmit($event)"> <input type="text" class="form-control" id="{[{ listId }]}-input" ng-keypress="onListSubmit($event, listId)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onStoplistSubmit($event)">Add</button> <button type="submit" class="form-control btn btn-default btn-primary" ng-click="onListSubmit($event, listId)">Add to {[{listName}]}</button>
</div> </div>
</div> </div>
</div> </div>
...@@ -70,7 +57,7 @@ ...@@ -70,7 +57,7 @@
<div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document"> <div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document">
<div class="row-fluid clearfix"> <div class="row-fluid clearfix">
<div class="col-md-7 col-xs-7"> <div class="col-md-7 col-xs-7">
<h3>{[{title}]}</h3> <h3 class="text-container" id="title">{[{title}]}</h3>
</div> </div>
<div class="col-md-5 col-xs-5"> <div class="col-md-5 col-xs-5">
<nav> <nav>
...@@ -90,17 +77,21 @@ ...@@ -90,17 +77,21 @@
</div> </div>
<h4 ng-if="abstract_text != null">Abstract</h4> <h4 ng-if="abstract_text != null">Abstract</h4>
<p id="abstract-text" class="text-container"> <p id="abstract-text" class="text-container">
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">No abstract text</div> <div ng-if="abstract_text == null" class="alert alert-info" role="alert">Empty abstract text</div>
</p> </p>
<h4 ng-if="full_text != null">Full Article</h4> <h4 ng-if="full_text != null">Full Article</h4>
<p id="full-text" class="text-container"> <p id="full-text" class="text-container">
<div ng-if="full_text == null" class="alert alert-info" role="alert">No full text</div> <div ng-if="full_text == null" class="alert alert-info" role="alert">Empty full text</div>
</p> </p>
</div> </div>
</div> <!-- end of the main row --> </div> <!-- end of the main row -->
</div> </div>
<!-- this menu is over the text --> <!-- this menu is over the text on mouse selection -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu" selection-template></div> <div ng-controller="AnnotationMenuController" id="selection" class="selection-menu">
<ul class="noselection">
<li ng-repeat="item in menuItems" class="{[{item.listName}]}" ng-click="onMenuClick($event, item.action, item.listId)">{[{item.verb}]} {[{item.listName}]}</li>
</ul>
</div>
</div> </div>
<!--[if lt IE 7]> <!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p> <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
......
annotations/views.py
View file @ 25584756
...@@ -40,15 +40,16 @@ class NgramList(APIView): ...@@ -40,15 +40,16 @@ class NgramList(APIView):
"""Get All for a doc id""" """Get All for a doc id"""
corpus_id = int(corpus_id) corpus_id = int(corpus_id)
doc_id = int(doc_id) doc_id = int(doc_id)
lists = dict() lists = {}
for list_type in ['MiamList', 'StopList']: for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type) list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type lists["%s" % list_id[0][0]] = list_type
# ngrams of list_id of corpus_id: # ngrams of list_id of corpus_id:
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id) doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id)
#doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)] doc_ngram_list = [(i, 'miam', i, 1931) for i in range(500)]
doc_ngram_list += [(i, 'stop', i, 1932) for i in range(501, 600)]
# doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
data = { '%s' % corpus_id : { data = { '%s' % corpus_id : {
'%s' % doc_id : [ '%s' % doc_id : [
......
gargantext.ini
View file @ 25584756
...@@ -39,6 +39,10 @@ vacuum = true ...@@ -39,6 +39,10 @@ vacuum = true
pidfile = /tmp/gargantext.pid pidfile = /tmp/gargantext.pid
# touch /tmp/gargantext.reload to reload configuration (after git pull for instance)
touch-reload = /tmp/gargantext.reload
# respawn processes taking more than 20 seconds # respawn processes taking more than 20 seconds
harakiri = 120 harakiri = 120
......
gargantext_web/about.py
View file @ 25584756
...@@ -29,6 +29,11 @@ def get_team(): ...@@ -29,6 +29,11 @@ def get_team():
'picture' : 'david.jpg', 'picture' : 'david.jpg',
'role':'principal investigator'}, 'role':'principal investigator'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
{ 'first_name' : 'Mathieu', 'last_name' : 'Rodic', { 'first_name' : 'Mathieu', 'last_name' : 'Rodic',
'mail' : '', 'mail' : '',
'website' : 'http://rodic.fr', 'website' : 'http://rodic.fr',
...@@ -41,11 +46,6 @@ def get_team(): ...@@ -41,11 +46,6 @@ def get_team():
'picture' : 'samuel.jpg', 'picture' : 'samuel.jpg',
'role' : 'developer'}, 'role' : 'developer'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
#{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''}, #{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''},
# copy paste the line above and write your informations please # copy paste the line above and write your informations please
] ]
......
gargantext_web/corpus_views.py
View file @ 25584756
...@@ -43,7 +43,9 @@ from scrappers.scrap_pubmed.admin import Logger ...@@ -43,7 +43,9 @@ from scrappers.scrap_pubmed.admin import Logger
from gargantext_web.db import * from gargantext_web.db import *
from sqlalchemy import or_, func from sqlalchemy.sql import func
from sqlalchemy import desc, asc, or_, and_, Date, cast, select
from gargantext_web import about from gargantext_web import about
...@@ -128,7 +130,7 @@ def get_ngrams(request , project_id , corpus_id ): ...@@ -128,7 +130,7 @@ def get_ngrams(request , project_id , corpus_id ):
'processing' : processing, 'processing' : processing,
'number' : number, 'number' : number,
'list_id': list_id[0][0], 'list_id': list_id[0][0],
'view' : "Terms", 'view' : "terms",
})) }))
return HttpResponse(html) return HttpResponse(html)
...@@ -195,7 +197,7 @@ def get_journals(request , project_id , corpus_id ): ...@@ -195,7 +197,7 @@ def get_journals(request , project_id , corpus_id ):
'corpus' : corpus, 'corpus' : corpus,
'processing' : processing, 'processing' : processing,
'number' : number, 'number' : number,
'view' : "Journals", 'view' : "journals",
})) }))
return HttpResponse(html) return HttpResponse(html)
...@@ -205,16 +207,18 @@ def test_journals(request , project_id, corpus_id ): ...@@ -205,16 +207,18 @@ def test_journals(request , project_id, corpus_id ):
JournalsDict = {} JournalsDict = {}
user_id = request.user.id
document_type_id = cache.NodeType['Document'].id document_type_id = cache.NodeType['Document'].id
documents = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
for doc in documents: journal_count = (session.query(Node.hyperdata['journal'].label("journal"), func.count().label("count"))
if "journal" in doc.hyperdata: .filter(Node.parent_id == corpus_id, Node.type_id==cache.NodeType['Document'].id)
journal = doc.hyperdata["journal"] .group_by("journal")
if journal not in JournalsDict: .order_by(desc("count"))
JournalsDict [journal] = 0 .all())
JournalsDict[journal] += 1 jc = dict()
return JsonHttpResponse(JournalsDict) for journal,count in journal_count:
jc[journal] = count
return JsonHttpResponse(jc)
def test_ngrams(request , project_id, corpus_id ): def test_ngrams(request , project_id, corpus_id ):
results = ["hola" , "mundo"] results = ["hola" , "mundo"]
...@@ -341,7 +345,11 @@ def test_ngrams(request , project_id, corpus_id ): ...@@ -341,7 +345,11 @@ def test_ngrams(request , project_id, corpus_id ):
ngrams_ids = Ngrams_Scores.keys() ngrams_ids = Ngrams_Scores.keys()
import math import math
if len(ngrams_ids) != 0:
occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) ) occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) )
else:
occs_threshold = 10
Metrics = { Metrics = {
"ngrams":[], "ngrams":[],
......
gargantext_web/urls.py
View file @ 25584756
...@@ -48,16 +48,20 @@ urlpatterns = patterns('', ...@@ -48,16 +48,20 @@ urlpatterns = patterns('',
# Corpus management # Corpus management
# Document view (main) # Document view (main)
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus), url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
url(r'^project/(\d+)/corpus/(\d+)/documents/?$', views.corpus),
# Journals view # Journals view
url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals), url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals$', corpus_views.get_journals), url(r'^project/(\d+)/corpus/(\d+)/journals', corpus_views.get_journals),
# Terms view # # Terms view
url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', corpus_views.test_ngrams), url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', corpus_views.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/ngrams$', corpus_views.get_ngrams), url(r'^project/(\d+)/corpus/(\d+)/terms/?$', corpus_views.get_ngrams),
############################################################################
# Update corpus
url(r'^project/(\d+)/corpus/(\d+)/(\w+)/update$', views.update_nodes),
############################################################################
# annotations App # annotations App
url(r'^project/(\d+)/corpus/(\d+)/document/(\d+)/$', annotations_main_view), url(r'^project/(\d+)/corpus/(\d+)/document/(\d+)/$', annotations_main_view),
url(r'^annotations/', include(annotations_urls)), url(r'^annotations/', include(annotations_urls)),
...@@ -104,7 +108,9 @@ urlpatterns = patterns('', ...@@ -104,7 +108,9 @@ urlpatterns = patterns('',
url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX), url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX),
url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON), url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
url(r'^tests/move2trash/$' , views.move_to_trash_multiple ), url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test) url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test),
url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', samtest.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/terms', samtest.get_ngrams)
) )
......
gargantext_web/views.py
View file @ 25584756
from django.shortcuts import redirect from admin.utils import PrintException
from django.shortcuts import render from django.shortcuts import redirect, render, render_to_response
from django.db import transaction from django.db import transaction
from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden
from django.template.loader import get_template from django.template.loader import get_template
from django.template import Context from django.template import Context, RequestContext
# remove this # remove this
from node import models from node import models
...@@ -43,7 +44,7 @@ from sqlalchemy import or_, func ...@@ -43,7 +44,7 @@ from sqlalchemy import or_, func
from gargantext_web import about from gargantext_web import about
from gargantext_web.db import NodeNgram, NodeNgramNgram
def login_user(request): def login_user(request):
logout(request) logout(request)
...@@ -251,6 +252,71 @@ def projects(request): ...@@ -251,6 +252,71 @@ def projects(request):
}) })
def update_nodes(request, project_id, corpus_id, view):
'''
update function:
- remove previous computations (temporary lists and coocurrences)
- permanent deletion of Trash
'''
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = int(project_id)
offset = int(corpus_id)
offset = str(view)
except ValueError:
raise Http404()
try:
print(corpus_id)
whites = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['WhiteList'].id)
.all()
)
print(whites)
for white in whites:
session.query(NodeNgram).filter(NodeNgram.node_id==white.id).delete()
session.delete(white)
cooc = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['Cooccurrence'].id)
.first()
)
session.query(NodeNgramNgram).filter(NodeNgramNgram.node_id==cooc.id).delete()
session.delete(cooc)
session.commit()
except :
PrintException()
nodes = models.Node.objects.filter(type_id=cache.NodeType['Trash'].id, user_id=request.user.id).all()
if nodes is not None:
with transaction.atomic():
for node in nodes:
try:
node.children.delete()
except Exception as error:
print(error)
node.delete()
#return redirect(request.path.replace('update', ''))
return redirect('/project/%s/corpus/%s/%s' % (project_id, corpus_id, view))
#
# return render_to_response(
# request.path,
# { 'title': 'Corpus view' },
# context_instance=RequestContext(request)
# )
#
def corpus(request, project_id, corpus_id): def corpus(request, project_id, corpus_id):
if not request.user.is_authenticated(): if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path) return redirect('/login/?next=%s' % request.path)
...@@ -287,7 +353,7 @@ def corpus(request, project_id, corpus_id): ...@@ -287,7 +353,7 @@ def corpus(request, project_id, corpus_id):
'processing' : processing, 'processing' : processing,
# 'documents': documents,\ # 'documents': documents,\
'number' : number, 'number' : number,
'view' : "Documents" 'view' : "documents"
})) }))
return HttpResponse(html) return HttpResponse(html)
...@@ -499,9 +565,10 @@ def graph(request, project_id, corpus_id): ...@@ -499,9 +565,10 @@ def graph(request, project_id, corpus_id):
results[i.id]["corpuses"].append(info) results[i.id]["corpuses"].append(info)
# print("\t",j.id , j.name) # print("\t",j.id , j.name)
import pprint # import pprint
pprint.pprint(results) # pprint.pprint(results)
graphurl = "corpus/"+str(corpus_id)+"/node_link.json"
html = t.render(Context({\ html = t.render(Context({\
'debug': settings.DEBUG, 'debug': settings.DEBUG,
'user' : user,\ 'user' : user,\
...@@ -509,7 +576,7 @@ def graph(request, project_id, corpus_id): ...@@ -509,7 +576,7 @@ def graph(request, project_id, corpus_id):
'corpus' : corpus,\ 'corpus' : corpus,\
'project' : project,\ 'project' : project,\
'corpusinfo' : results,\ 'corpusinfo' : results,\
'graphfile' : "hola_mundo",\ 'graphfile' : graphurl,\
})) }))
return HttpResponse(html) return HttpResponse(html)
...@@ -626,13 +693,13 @@ def node_link(request, corpus_id): ...@@ -626,13 +693,13 @@ def node_link(request, corpus_id):
data = [] data = []
corpus = session.query(Node).filter(Node.id==corpus_id).first() corpus = session.query(Node).filter(Node.id==corpus_id).first()
filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id) # filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id)
print("file exists?:",os.path.isfile(filename)) # print("file exists?:",os.path.isfile(filename))
if os.path.isfile(filename): # if os.path.isfile(filename):
json_data = open(filename,"r") # json_data = open(filename,"r")
data = json.load(json_data) # data = json.load(json_data)
json_data.close() # json_data.close()
else: # else:
data = get_cooc(request=request, corpus_id=corpus_id, type="node_link") data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
return JsonHttpResponse(data) return JsonHttpResponse(data)
......
parsing/FileParsers/ISTex.py
View file @ 25584756
...@@ -19,7 +19,7 @@ class ISTex(FileParser): ...@@ -19,7 +19,7 @@ class ISTex(FileParser):
"source" : 'corpusName', "source" : 'corpusName',
"title" : 'title', "title" : 'title',
"genre" : "genre", "genre" : "genre",
# "language_iso3" : 'MedlineCitation/Article/Language', "language_iso3" : 'language',
"doi" : 'doi', "doi" : 'doi',
"host" : 'host', "host" : 'host',
"publication_date" : 'pubdate', "publication_date" : 'pubdate',
...@@ -28,17 +28,22 @@ class ISTex(FileParser): ...@@ -28,17 +28,22 @@ class ISTex(FileParser):
"keywords" : "keywords" "keywords" : "keywords"
} }
suma = 0
for json_doc in json_docs: for json_doc in json_docs:
hyperdata = {} hyperdata = {}
for key, path in hyperdata_path.items(): for key, path in hyperdata_path.items():
try: try:
# print(path," ==> ",len(json_doc[path])) # print(path," ==> ",len(json_doc[path]))
hyperdata[key] = json_doc[path] hyperdata[key] = json_doc[path]
except: pass except:
pass
# print("|",hyperdata["publication_date"]) # print("|",hyperdata["language_iso3"])
if "doi" in hyperdata: hyperdata["doi"] = hyperdata["doi"][0] if "doi" in hyperdata:
hyperdata["doi"] = hyperdata["doi"][0]
keywords = [] keywords = []
if "keywords" in hyperdata: if "keywords" in hyperdata:
...@@ -75,11 +80,21 @@ class ISTex(FileParser): ...@@ -75,11 +80,21 @@ class ISTex(FileParser):
if "genre" in hyperdata: if "genre" in hyperdata:
if len(hyperdata["genre"])==0: if len(hyperdata["genre"])==0:
hyperdata.pop("genre") hyperdata.pop("genre")
if "language_iso3" in hyperdata:
if len(hyperdata["language_iso3"])>0:
hyperdata["language_iso3"] = hyperdata["language_iso3"][0]
else:
hyperdata["language_iso3"] = "eng"
if "publication_date" in hyperdata:
RealDate = hyperdata["publication_date"] RealDate = hyperdata["publication_date"]
if "publication_date" in hyperdata: hyperdata.pop("publication_date") if "publication_date" in hyperdata:
hyperdata.pop("publication_date")
if isinstance(RealDate, list):
RealDate = RealDate[0]
# print( RealDate ," | length:",len(RealDate))
Decision="" Decision=""
if len(RealDate)>4: if len(RealDate)>4:
if len(RealDate)>8: if len(RealDate)>8:
...@@ -90,10 +105,10 @@ class ISTex(FileParser): ...@@ -90,10 +105,10 @@ class ISTex(FileParser):
else: else:
try: Decision = datetime.strptime(RealDate, '%Y-%b').date() try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
except: except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date() try: Decision = datetime.strptime(RealDate, '%Y-%m').date()
except: Decision=False except: Decision=False
else: else:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date() try: Decision = datetime.strptime(RealDate, '%Y').date()
except: Decision=False except: Decision=False
if Decision!=False: if Decision!=False:
...@@ -104,5 +119,14 @@ class ISTex(FileParser): ...@@ -104,5 +119,14 @@ class ISTex(FileParser):
# print("\t||",hyperdata["title"]) # print("\t||",hyperdata["title"])
# print("\t\t",Decision) # print("\t\t",Decision)
# print("=============================") # print("=============================")
# else:
# suma+=1
# if "pubdate" in json_doc:
# print ("\tfail pubdate:",json_doc["pubdate"])
# print ("nb_hits:",len(json_docs))
# print("\t - nb_fails:",suma)
# print(" -- - - - - - -- - -")
return hyperdata_list return hyperdata_list
scrappers/scrap_pubmed/views.py
View file @ 25584756
...@@ -66,7 +66,7 @@ def getGlobalStatsISTEXT(request ): ...@@ -66,7 +66,7 @@ def getGlobalStatsISTEXT(request ):
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query ) print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N ) print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
query_string = query.replace(" ","+") query_string = query.replace(" ","+")
url = "http://api.istex.fr/document/?q="+query_string url = "http://api.istex.fr/document/?q="+query_string+"&output=id,title,abstract,pubdate,corpusName,authors,language"
tasks = MedlineFetcher() tasks = MedlineFetcher()
......
static/js/Docs_dyna_chart_and_table.js
View file @ 25584756
...@@ -127,19 +127,9 @@ for(var i=0; i<url_elems.length; i++) { ...@@ -127,19 +127,9 @@ for(var i=0; i<url_elems.length; i++) {
url_mainIDs[url_elems[i-1]] = Number(url_elems[i]); url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
} }
} }
var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
// $.ajax({
// url: theurl,
// success: function(data) {
// bisarray = data.data
// for(var i in bisarray) {
// untitlebis = bisarray[i].values
// BIS_dict[untitlebis[0]] = [bisarray[i].count , 0];// [ total amount , removed ]
// }
// pr(BIS_dict)
// if(Object.keys(BIS_dict).length>0) $("#delAll").css("visibility", "visible"); $("#delAll").show();
// }
// });
...@@ -168,7 +158,7 @@ function getRecords() { ...@@ -168,7 +158,7 @@ function getRecords() {
function transformContent2(rec_id) { function transformContent2(rec_id) {
// pr("\t\ttransformContent2: "+rec_id) // pr("\t\ttransformContent2: "+rec_id)
var elem = AjaxRecords[rec_id]; var elem = AjaxRecords[rec_id];
// pr("\t\t\t"+elem.date) // pr("\t"+elem.title)
var result = {} var result = {}
if (elem["del"]) { if (elem["del"]) {
result["id"] = elem["id"] result["id"] = elem["id"]
...@@ -261,46 +251,39 @@ function ulWriter(rowIndex, record, columns, cellWriter) { ...@@ -261,46 +251,39 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
} }
function Main_test( Data , SearchFilter ) {
// var div__filter_for_search = '' var DistributionDict = {}
// div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple data-max-options="1">'; for(var i in DistributionDict)
// div__filter_for_search += ' <optgroup label="All" data-max-options="1" selected>'; delete DistributionDict[i];
// div__filter_for_search += ' <option>Title</option>'; delete DistributionDict;
// div__filter_for_search += ' <option>Date</option>'; DistributionDict = {}
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Category" data-max-options="1">';
// div__filter_for_search += ' <option>Title</option>';
// div__filter_for_search += ' <option>Date</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Duplicates" data-max-options="1">';
// div__filter_for_search += ' <option>by DOI</option>';
// div__filter_for_search += ' <option>by Title</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += '</select>';
// $("#supmofos").html(div__filter_for_search)
$("#div-table").html("")
// (3) Get records and hyperdata for paginator $("#div-table").empty();
$.ajax({ var div_table = '<p align="right">'+"\n"
url: '/tests/paginator/corpus/'+url_mainIDs["corpus"], div_table += '<table id="my-ajax-table" class="table table-bordered">'+"\n"
success: function(data){ div_table += "\t"+'<thead>'+"\n"
console.log(data) div_table += "\t"+"\t"+'<th width="100px;" data-dynatable-column="date">Date</th>'+"\n"
div_table += "\t"+"\t"+'<th data-dynatable-column="name">Title</th>'+"\n"
div_table += "\t"+"\t"+'<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>'+"\n"
div_table += "\t"+"\t"+'</th>'+"\n"
div_table += "\t"+'</thead>'+"\n"
div_table += "\t"+'<tbody>'+"\n"
div_table += "\t"+'</tbody>'+"\n"
div_table += '</table>'+"\n"
div_table += '</p>';
$("#div-table").html(div_table)
var justdates = {} var justdates = {}
for(var i in data.records) { for(var i in Data) {
var orig_id = parseInt(data.records[i].id) var date = Data[i]["date"];
var arr_id = parseInt(i)
RecDict[orig_id] = arr_id;
data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["del"] = false
var date = data.records[i]["date"];
if ( ! justdates[date] ) justdates[date] = 0; if ( ! justdates[date] ) justdates[date] = 0;
justdates[date]++; justdates[date]++;
// console.log(data.records[i]["date"]+" : originalRecords["+arr_id+"] <- "+orig_id+" | "+data.records[i]["name"]) // console.log(Data[i]["date"]+" : originalRecords["+arr_id+"] <- "+orig_id+" | "+Data[i]["name"])
} }
AjaxRecords = data.records; // backup!!
// $("#move2trash").prop('disabled', true); // $("#move2trash").prop('disabled', true);
...@@ -316,8 +299,8 @@ function ulWriter(rowIndex, record, columns, cellWriter) { ...@@ -316,8 +299,8 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
var arrayd3 = [] var arrayd3 = []
for(var e in data.records) { for(var e in Data) {
var date = data.records[e]["date"]; var date = Data[e]["date"];
if(justdates[date]!=false) { if(justdates[date]!=false) {
var info = {} var info = {}
info.date = date info.date = date
...@@ -438,13 +421,14 @@ function ulWriter(rowIndex, record, columns, cellWriter) { ...@@ -438,13 +421,14 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
dc.renderAll(); dc.renderAll();
MyTable = []
MyTable = $('#my-ajax-table').dynatable({ MyTable = $('#my-ajax-table').dynatable({
dataset: { dataset: {
records: data.records records: Data
}, },
features: { features: {
pushState: false, pushState: false,
sort: false //i need to fix the sorting function... the current one just sucks // sort: false //i need to fix the sorting function... the current one just sucks
}, },
writers: { writers: {
_rowWriter: ulWriter _rowWriter: ulWriter
...@@ -452,17 +436,87 @@ function ulWriter(rowIndex, record, columns, cellWriter) { ...@@ -452,17 +436,87 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
} }
}); });
MyTable.data('dynatable').paginationPage.set(1);
MyTable.data('dynatable').process();
if ( $(".imadiv").length>0 ) return 1; if ( $(".imadiv").length>0 ) return 1;
$('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page") $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
$(".dynatable-record-count").insertAfter(".imadiv") $(".dynatable-record-count").insertAfter(".imadiv")
$(".dynatable-pagination-links").insertAfter(".imadiv") $(".dynatable-pagination-links").insertAfter(".imadiv")
var the_content = $("#supmofos").html(); $("#filter_search").html( $("#filter_search").html().replace('selected="selected"') );
$("#"+SearchFilter).attr( "selected" , "selected" )
var the_content = $("#filter_search").html();
$(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table") $(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
$("#supmofos").remove()
// .insertAfter("#dynatable-query-search-my-ajax-table") // .insertAfter("#dynatable-query-search-my-ajax-table")
return "OK"
}
function SearchFilters( elem ) {
var MODE = elem.value;
if( MODE == "filter_all") {
var result = Main_test(AjaxRecords , MODE)
console.log( result )
}
if( MODE == "filter_dupl-titles") {
var getDupl_API = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
$.ajax({
url: getDupl_API,
success: function(data) {
bisarray = data.data
for(var i in bisarray) {
titlebis = bisarray[i].values
BIS_dict[titlebis[0]] = true;
}
var Duplicates = []
for(var r in AjaxRecords) {
if ( BIS_dict[AjaxRecords[r].title] )
Duplicates.push( AjaxRecords[r] )
}
var result = Main_test(Duplicates , MODE)
console.log( result )
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('title', 1) // 1=ASCENDING,
MyTable.data('dynatable').process();
} }
}); });
}
}
// FIRST portion of code to be EXECUTED:
// (3) Get records and hyperdata for paginator
$.ajax({
url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
success: function(data){
for(var i in data.records) {
var orig_id = parseInt(data.records[i].id)
var arr_id = parseInt(i)
RecDict[orig_id] = arr_id;
data.records[i]["title"] = data.records[i]["name"];
data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["del"] = false
}
AjaxRecords = data.records; // backup-ing in global variable!
var result = Main_test(data.records , "filter_all")
console.log( result )
},
});
static/js/Journals_dyna_chart_and_table.js
View file @ 25584756
...@@ -471,11 +471,17 @@ function Main_test( data , initial) { ...@@ -471,11 +471,17 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = [] // MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords; // MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1); MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process(); // MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear(); // MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').process(); MyTable.data('dynatable').process();
// // // $("#score_column_id").children()[0].text = FirstScore // // // $("#score_column_id").children()[0].text = FirstScore
// // // // MyTable.data('dynatable').process(); // // // // MyTable.data('dynatable').process();
......
static/js/NGrams_dyna_chart_and_table.js
View file @ 25584756
...@@ -370,15 +370,18 @@ $("#Clean_All").click(function(){ ...@@ -370,15 +370,18 @@ $("#Clean_All").click(function(){
}); });
$("#Save_All").click(function(){ $("#Save_All").click(function(){
console.log("click in save all 01")
var sum__selected_elems = 0; var sum__selected_elems = 0;
var poubelle = [] var poubelle = []
for(var i in FlagsBuffer) for(var i in FlagsBuffer) {
if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i) if (Object.keys(FlagsBuffer[i]).length==0)
poubelle.push(i)
sum__selected_elems += Object.keys(FlagsBuffer[i]).length; sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
}
console.log("click in save all 02")
for(var i in poubelle) for(var i in poubelle)
delete FlagsBuffer[poubelle[i]]; delete FlagsBuffer[poubelle[i]];
console.log("click in save all 03, sum:"+sum__selected_elems)
if ( sum__selected_elems>0 ) { if ( sum__selected_elems>0 ) {
console.log("") console.log("")
console.log("Do the ajax conexion with API and send this array to be processed:") console.log("Do the ajax conexion with API and send this array to be processed:")
...@@ -498,11 +501,6 @@ function Main_test( data , initial) { ...@@ -498,11 +501,6 @@ function Main_test( data , initial) {
oldest = Number(min_occ); oldest = Number(min_occ);
latest = Number(max_occ); latest = Number(max_occ);
var ndx = false; var ndx = false;
ndx = crossfilter(); ndx = crossfilter();
ndx.add(DistributionList); ndx.add(DistributionList);
...@@ -624,6 +622,9 @@ function Main_test( data , initial) { ...@@ -624,6 +622,9 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = [] // MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords; // MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1); MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process(); // MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear(); // MyTable.data('dynatable').sorts.clear();
...@@ -661,7 +662,7 @@ $.ajax({ ...@@ -661,7 +662,7 @@ $.ajax({
// Building the Score-Selector // Building the Score-Selector
var FirstScore = data.scores.initial var FirstScore = data.scores.initial
var possible_scores = Object.keys( data.ngrams[0].scores ); var possible_scores = Object.keys( data.ngrams[0].scores );
var scores_div = '<select class="span1" id="scores_selector">'+"\n"; var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n" scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
for( var i in possible_scores ) { for( var i in possible_scores ) {
if(possible_scores[i]!=FirstScore) { if(possible_scores[i]!=FirstScore) {
......
templates/corpus/bannerBottome.html deleted 100644 → 0
View file @ d6cc0e32
<!--<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>--!>
templates/corpus/documents.html
View file @ 25584756
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
{% load staticfiles %} {% load staticfiles %}
<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}"> <link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}"> <!-- here goes import stylesheet js/bootstrap/bootstrap-multiselect.css, mais ca marche pas-->
<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}"> <link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}"> <link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
...@@ -72,6 +72,7 @@ th a { ...@@ -72,6 +72,7 @@ th a {
<input type="hidden" id="list_id" value="{{ list_id }}"></input>
<div class="container"> <div class="container">
<div class="jumbotron"> <div class="jumbotron">
...@@ -87,18 +88,7 @@ th a { ...@@ -87,18 +88,7 @@ th a {
</div> </div>
<div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel"> <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
<div class="panel-body"> <div class="panel-body">
<p align="right"> <div id="div-table"></div>
<table id="my-ajax-table" class="table table-bordered">
<thead>
<!-- <th data-dynatable-column="id">ID</th> -->
<th width="100px;" data-dynatable-column="date">Date</th>
<th data-dynatable-column="name">Title</th>
<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>
</thead>
<tbody>
</tbody>
</table>
</p>
<p align="right"> <p align="right">
<button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button> <button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button>
</p> </p>
...@@ -111,24 +101,26 @@ th a { ...@@ -111,24 +101,26 @@ th a {
</div> </div>
</div> </div>
<div id="supmofos">
<select data-width="100px" dir="ltr" class="selectpicker"> <div id="filter_search" style="visibility:hidden">
<option selected>All</option> <select id="example-single-optgroups" onchange="SearchFilters(this);">
<option>Title</option> <!-- <optgroup label=""> -->
<option>Date</option> <option id="filter_all" value="filter_all">All</option>
<optgroup label="Duplicates"> <!-- <option id="filter_title" value="filter_title">Title</option> -->
<option>by DOI</option> <!-- <option id="filter_date" value="filter_date">Date</option> -->
<option>by Title</option> <!-- </optgroup> -->
</optgroup> <!-- <optgroup label="Duplicates"> -->
</select> <!-- <option value="filter_doi">By DOI</option> -->
<option id="filter_dupl-titles" value="filter_dupl-titles">Duplicates by Title</option>
<!-- </optgroup> -->
</select>
</div> </div>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script> <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script src="{% static "js/charts/bootstrap.min.js" %}"></script> <script src="{% static "js/charts/bootstrap.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script> <!-- here goes import script js/bootstrap/bootstrap-multiselect.js, mais ca marche pas-->
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script> <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js --> <!-- custom-lib for dynatable.js and dc.js -->
......
templates/corpus/menu.html
View file @ 25584756
{% extends "menu.html" %} {% extends "menu.html" %}
{% load staticfiles %}
{% block corpusBannerTop %} {% block corpusBannerTop %}
...@@ -10,7 +11,7 @@ ...@@ -10,7 +11,7 @@
<div class="col-md-4"> <div class="col-md-4">
{% if project %} {% if project %}
<h1>{{ project.name }}</h1> <h1><a href="/project/{{project.id}}">{{ project.name }}</a></h1>
<h2>{{ corpus.name }}</h2> <h2>{{ corpus.name }}</h2>
{% endif %} {% endif %}
<!--<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> --> <!--<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
...@@ -32,9 +33,9 @@ ...@@ -32,9 +33,9 @@
<div class="col-md-4"> <div class="col-md-4">
<br> <br>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/update">Update</a> <a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{corpus.id}}/{{view}}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save</a> <a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Download</a>
<a type="button" class="btn btn-default btn-lg" data-container="body" data-toggle="popover" data-placement="bottom" <a type="button" class="btn btn-default btn-lg" data-container="body" data-toggle="popover" data-placement="bottom"
data-content=' data-content='
<ul> <ul>
...@@ -62,9 +63,12 @@ ...@@ -62,9 +63,12 @@
<div class="btn-group btn-group-justified"> <div class="btn-group btn-group-justified">
<center> <center>
<a type="button" class="btn btn-default {% if view == "Documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/">{{number}} Documents</a> <a type="button" class="btn btn-default {% if view == "documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/documents">{{number}} Documents</a>
<a type="button" class="btn btn-default {% if view == "Journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a> <a type="button" class="btn btn-default {% if view == "journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a>
<a type="button" class="btn btn-default {% if view == "Terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/ngrams">Terms</a>
{% if processing == 0 %}
<a type="button" class="btn btn-default {% if view == "terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/terms">Terms</a>
{% endif %}
</center> </center>
</div> </div>
...@@ -76,7 +80,7 @@ ...@@ -76,7 +80,7 @@
<div class="container"> <div class="container">
<div class="row"> <div class="row">
<div class="col-md-4"> <div class="col-md-6">
<div class="jumbotron"> <div class="jumbotron">
<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3> <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
<ol> <ol>
...@@ -88,23 +92,11 @@ ...@@ -88,23 +92,11 @@
</div> </div>
</div> </div>
<div class="col-md-4">
<div class="jumbotron">
<!-- <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3> -->
<h3>Matrix (soon)</h3>
<ol>
<li>Sort</li>
<li>Group</li>
<li>Cluster</li>
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
<div class="col-md-4"> <div class="col-md-6">
<div class="jumbotron"> <div class="jumbotron">
{% if processing > 0 %} {% if processing > 0 %}
<h3> Graph (later)</h3> <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
{% else %} {% else %}
<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3> <h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
{% endif %} {% endif %}
......
templates/explorer.html
View file @ 25584756
...@@ -60,7 +60,7 @@ ...@@ -60,7 +60,7 @@
<li> <li>
<a> <a>
<div id="graphid" style="visibility: hidden;">/corpus/{{ corpus.id }}/node_link.json</div> <div id="graphid" style="visibility: hidden;">{{graphfile}}</div>
</a> </a>
</li> </li>
...@@ -177,9 +177,7 @@ ...@@ -177,9 +177,7 @@
<div style="visibility: hidden;" id="sigma-othergraph"></div> <div style="visibility: hidden;" id="sigma-othergraph"></div>
<div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;"> <div id="semLoader"></div>
<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
</div>
<ul id="ctlzoom"> <ul id="ctlzoom">
......
templates/project.html
View file @ 25584756
...@@ -87,7 +87,7 @@ ...@@ -87,7 +87,7 @@
{% ifequal corpus.processing 1 %} {% ifequal corpus.processing 1 %}
{{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :) {{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
{% else %} {% else %}
<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} </a> , {{ corpus.count }} Documents <a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} , {{ corpus.count }} Documents </a>
{% endifequal %} {% endifequal %}
<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" <button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
data-content=' data-content='
......
test-list-management.py
View file @ 25584756
...@@ -6,8 +6,8 @@ from ngram.lists import * ...@@ -6,8 +6,8 @@ from ngram.lists import *
#from cooccurrences import * #from cooccurrences import *
#from gargantext_web.views import empty_trash from gargantext_web.views import empty_trash
#empty_trash() empty_trash()
#user = session.query(User).all()[0] #user = session.query(User).all()[0]
...@@ -77,6 +77,7 @@ print('Stem Node.id is', stem_id) ...@@ -77,6 +77,7 @@ print('Stem Node.id is', stem_id)
# print(n, listNgramIds(list_id=n[0][0])[:3]) # print(n, listNgramIds(list_id=n[0][0])[:3])
# #
stop_list_id = listIds(user_id=user.id, stop_list_id = listIds(user_id=user.id,
corpus_id=corpus.id, corpus_id=corpus.id,
typeList='StopList')[0][0] typeList='StopList')[0][0]
...@@ -86,13 +87,13 @@ miam_list_id = listIds(user_id=user.id, ...@@ -86,13 +87,13 @@ miam_list_id = listIds(user_id=user.id,
typeList='MiamList')[0][0] typeList='MiamList')[0][0]
print('Stop List', stop_list_id) print('StopList', stop_list_id)
print('Miam List', miam_list_id) print('MiamList', miam_list_id)
ngrams2miam(user_id=user.id, corpus_id=corpus.id) print(session.query(Node.id).filter(Node.parent_id==corpus.id, Node.type_id==cache.NodeType['WhiteList'].id).first())
#ngrams2miam(user_id=user.id, corpus_id=corpus.id)
print(listNgramIds(list_id=miam_list_id, user_id=user.id, corpus_id=corpus.id)) #print(listNgramIds(list_id=stop_list_id, user_id=user.id, corpus_id=corpus.id))
......
tests/ngramstable/views.py
View file @ 25584756
...@@ -100,7 +100,7 @@ def get_ngrams(request , project_id , corpus_id ): ...@@ -100,7 +100,7 @@ def get_ngrams(request , project_id , corpus_id ):
except ValueError: except ValueError:
raise Http404() raise Http404()
t = get_template('tests/ngrams.html') t = get_template('corpus/terms.html')
user = cache.User[request.user.username].id user = cache.User[request.user.username].id
date = datetime.datetime.now() date = datetime.datetime.now()
...@@ -173,7 +173,7 @@ def get_journals(request , project_id , corpus_id ): ...@@ -173,7 +173,7 @@ def get_journals(request , project_id , corpus_id ):
except ValueError: except ValueError:
raise Http404() raise Http404()
t = get_template('tests/journals.html') t = get_template('corpus/journals.html')
user = cache.User[request.user.username].id user = cache.User[request.user.username].id
date = datetime.datetime.now() date = datetime.datetime.now()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment