Skip to content

gargantext
gargantext
Commit 25584756 authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into unstable-mat

parents d6cc0e32 225cb308
Hide whitespace changes
Inline Side-by-side
Showing with 830 additions and 614 deletions
analysis/functions.py
View file @ 25584756
......@@ -21,7 +21,10 @@ def create_synonymes(user, corpus):
size = 1000
def create_whitelist(user, corpus_id, size=size, count_min=2):
def create_whitelist(user, corpus_id, size=size, count_min=2, miam_id=None):
if miam_id is None:
PrintException()
cursor = connection.cursor()
whitelist_type_id = cache.NodeType['WhiteList'].id
......@@ -53,10 +56,14 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
node_node_ngram AS nngX ON nngX.node_id = n.id
INNER JOIN
node_ngram AS ngX ON ngX.id = nngX.ngram_id
INNER JOIN
node_node_ngram AS miam ON ngX.id = miam.ngram_id
WHERE
n.parent_id = %d
AND
n.type_id = %d
AND
miam.node_id = %d
AND
ngX.n >= 2
AND
......@@ -72,7 +79,7 @@ def create_whitelist(user, corpus_id, size=size, count_min=2):
LIMIT
%d
;
""" % (white_list.id, int(corpus_id), int(type_document_id), count_min, size)
""" % (white_list.id, int(corpus_id), int(type_document_id), int(miam_id), count_min, size)
# print("PRINTING QYERY OF WHITELIST:")
# print(query_whitelist)
cursor.execute(query_whitelist)
......@@ -155,6 +162,7 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
from gargantext_web.api import JsonHttpResponse
from analysis.louvain import best_partition
from ngram.lists import listIds
#print(corpus_id, cooc_id)
......@@ -168,7 +176,9 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
if session.query(Node).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first() is None:
print("Coocurrences do not exist yet, create it.")
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size)
miam_id = listIds(typeList='MiamList', user_id=request.user.id, corpus_id=corpus_id)[0][0]
whitelist = create_whitelist(request.user, corpus_id=corpus_id, size=size, miam_id=miam_id)
cooccurrence_node_id = create_cooc(user=request.user, corpus_id=corpus_id, whitelist=whitelist, size=size)
else:
cooccurrence_node_id = session.query(Node.id).filter(Node.type_id==type_cooc_id, Node.parent_id==corpus_id).first()
......@@ -211,8 +221,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
n = n.sort(inplace=False)
m = m.sort(inplace=False)
print(n)
print(m)
#print(n)
#print(m)
nodes_included = 300 #int(round(size/20,0))
#nodes_excluded = int(round(size/10,0))
......@@ -237,25 +247,25 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
xxx = xx.values
threshold = min(xxx.max(axis=1))
matrix_filtered = np.where(xxx >= threshold, xxx, 0)
#matrix_filtered = matrix_filtered.resize((90,90))
except:
PrintException()
try:
G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph())
G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
G = nx.from_numpy_matrix(np.matrix(matrix_filtered))
#G = nx.from_numpy_matrix(matrix_filtered, create_using=nx.MultiDiGraph())
#print(G)
G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
# Removing too connected nodes (find automatic way to do it)
#edges_to_remove = [ e for e in G.edges_iter() if
degree = G.degree()
nodes_to_remove = [n for n in degree if degree[n] <= 1]
G.remove_nodes_from(nodes_to_remove)
partition = best_partition(G)
uG = G.to_undirected()
partition = best_partition(uG)
except:
print("-" * 30)
PrintException()
......@@ -271,7 +281,8 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
G.node[node]['group'] = partition[node]
# G.add_edge(node, "cluster " + str(partition[node]), weight=3)
except Exception as error:
print("error01: ",error)
pass#PrintException()
#print("error01: ",error)
data = json_graph.node_link_data(G)
......
annotations/static/annotations/app.css
View file @ 25584756
/* app css stylesheet */
/*
* Class names corresponding to server-side list names
* To display another list name, please add a new class under this
*/
.MiamList {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.StopList {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.delete-keyword, .occurrences {
vertical-align: super;
font-size: 70%;
......@@ -27,27 +43,9 @@
border-bottom: none;
}
.miamword {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.stopword {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.global-stopword {
color: black;
background-color: rgba(169, 68, 66, 0.05);
cursor: pointer;
}
.main-panel, .text-panel, .words-panel {
height: 800px;
margin: 10px 0px;
margin: 10px 0;
}
.text-panel {
......@@ -59,11 +57,7 @@
height: 250px;
}
.keyword-container {
/*display: inline-block;*/
}
.keyword {
.keyword-text {
word-break: break-all;
}
......
annotations/static/annotations/app.js
View file @ 25584756
(function () {
'use strict';
/*
* Django STATIC_URL given to angular to load async resources
*/
var S = window.STATIC_URL;
window.annotationsApp = angular.module('annotationsApp', ['annotationsAppHttp']);
/*
* Angular Templates must not conflict with Django's
*/
window.annotationsApp.config(function($interpolateProvider) {
$interpolateProvider.startSymbol('{[{');
$interpolateProvider.endSymbol('}]}');
});
/*
* Template of the ngram element displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.directive('keywordTemplate', function () {
return {
templateUrl: function ($element, $attributes) {
......@@ -18,25 +27,36 @@
};
});
/*
* For ngram elements displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.controller('ExtraAnnotationController',
['$scope', '$rootScope', '$element', 'NgramHttpService',
function ($scope, $rootScope, $element, NgramHttpService) {
// TODO use the tooltip ?
/*
* Click on the 'delete' cross button
*/
$scope.onDeleteClick = function () {
NgramHttpService.delete({
'listId': $scope.keyword.list_id,
'ngramId': $scope.keyword.uuid
}).$promise.then(function(data) {
}, function(data) {
$.each($rootScope.annotations, function(index, element) {
if (element.list_id == $scope.keyword.list_id && element.uuid == $scope.keyword.uuid) {
$rootScope.annotations.splice(index, 1);
return false;
}
});
}, function(data) {
console.log(data);
console.error("unable to delete the Ngram " + $scope.keyword.uuid);
});
}
};
}]);
/*
* For mouse selection on the text
*/
window.annotationsApp.controller('AnnotationController',
['$scope', '$rootScope', '$element',
function ($scope, $rootScope, $element) {
......@@ -57,14 +77,9 @@
};
}]);
window.annotationsApp.directive('selectionTemplate', function () {
return {
templateUrl: function ($element, $attributes) {
return S + 'annotations/selection_tpl.html';
}
};
});
/*
* Controller of the menu over the current mouse selection
*/
window.annotationsApp.controller('AnnotationMenuController',
['$scope', '$rootScope', '$element', '$timeout', 'NgramHttpService',
function ($scope, $rootScope, $element, $timeout, NgramHttpService) {
......@@ -87,9 +102,12 @@
}
return false;
}
// we only need one singleton at a time
var selection = getSelected();
/*
* When mouse selection is started, we highlight it
*/
function toggleSelectionHighlight(text) {
if (text.trim() !== "") {
$(".text-panel").addClass("selection");
......@@ -98,53 +116,68 @@
}
}
/*
* Dynamically construct the selection menu scope
*/
function toggleMenu(context, annotation) {
$timeout(function() {
$scope.$apply(function() {
var miamlist_id = _.invert($rootScope.activeLists).MiamList;
var stoplist_id = _.invert($rootScope.activeLists).StopList;
// variable used in onClick
$scope.selection_text = angular.copy(annotation);
if (angular.isObject(annotation)) {
$scope.level = angular.copy(annotation.level || 'global');
$scope.category = $rootScope.lists[annotation.list_id].toLowerCase();
$scope.listId = angular.copy(annotation.list_id);
// used in onClick
$scope.selection_text = angular.copy(annotation);
if ($scope.category == "miamlist") {
$scope.local_miamlist = false;
$scope.global_stoplist = true;
$scope.local_stoplist = true;
} else if ($scope.category == "stoplist") {
if ($scope.level == "local") {
$scope.local_stoplist = false;
$scope.global_stoplist = true;
// existing ngram
// Delete from the current list
$scope.menuItems = [
{
'action': 'delete',
'listId': annotation.list_id,
'verb': 'Delete from',
'listName': $rootScope.lists[annotation.list_id]
}
if ($scope.level == "global") {
$scope.global_stoplist = false;
$scope.local_stoplist = true;
}
$scope.local_miamlist = true;
];
if ($rootScope.lists[annotation.list_id] == "MiamList") {
// Add to the alternative list
$scope.menuItems.push({
'action': 'post',
'listId': stoplist_id,
'verb': 'Add to',
'listName': $rootScope.lists[stoplist_id]
});
} else if ($rootScope.lists[annotation.list_id] == "StopList") {
// Add to the alternative list
$scope.menuItems.push({
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.lists[miamlist_id]
});
}
// show menu
// show the menu
$element.fadeIn(100);
}
else if (annotation.trim() !== "") {
$scope.selection_text = angular.copy(annotation);
$scope.level = "New Ngram from selection";
$scope.category = null;
$scope.local_miamlist = true;
$scope.local_stoplist = true;
$scope.global_stoplist = true;
// show menu
} else if (annotation.trim() !== "") {
// new ngram
$scope.menuItems = [
{
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.activeLists[miamlist_id]
}
];
// show the menu
$element.fadeIn(100);
} else {
// close menu
$scope.menuItems = [];
// close the menu
$element.fadeOut(100);
}
});
});
}
var elt = $(".text-panel")[0];
var pos = $(".text-panel").position();
function positionElement(context, x, y) {
......@@ -157,18 +190,26 @@
positionElement(null, e.pageX, e.pageY);
}
// TODO is mousedown necessary ?
$(".text-panel").mousedown(function(){
$(".text-panel").mousemove(positionMenu);
/*
* Dynamically position the menu
*/
$(".text-container").mousedown(function(){
$(".text-container").mousemove(positionMenu);
});
$(".text-panel").mouseup(function(){
$(".text-panel").unbind("mousemove", positionMenu);
/*
* Finish positioning the menu then display the menu
*/
$(".text-container").mouseup(function(){
$(".text-container").unbind("mousemove", positionMenu);
toggleSelectionHighlight(selection.toString().trim());
toggleMenu(null, selection.toString().trim());
});
$(".text-panel").delegate(':not("#selection")', "click", function(e) {
/*
* Toggle the menu when clicking on an existing ngram keyword
*/
$(".text-container").delegate(':not("#selection")', "click", function(e) {
if ($(e.target).hasClass("keyword-inline")) return;
positionMenu(e);
toggleSelectionHighlight(selection.toString().trim());
......@@ -178,69 +219,94 @@
$rootScope.$on("positionAnnotationMenu", positionElement);
$rootScope.$on("toggleAnnotationMenu", toggleMenu);
$scope.onClick = function($event, action, listId, level) {
/*
* Menu click action
*/
$scope.onMenuClick = function($event, action, listId) {
if (angular.isObject($scope.selection_text)) {
// delete from the current list
// action on an existing Ngram
NgramHttpService[action]({
'listId': listId,
'ngramId': $scope.selection_text.uuid
}).$promise.then(function(data) {
}, function(data) {
$.each($rootScope.annotations, function(index, element) {
if (element.list_id == listId && element.uuid == $scope.selection_text.uuid) {
$rootScope.annotations.splice(index, 1);
return false;
}
});
});
}, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
} else if ($scope.selection_text.trim() !== "") {
// new annotation from selection
NgramHttpService.post(
{
'listId': listId
'listId': listId,
'ngramId': 'new'
},
{'annotation' : {'text': $scope.selection_text.trim()}}
).$promise.then(function(data) {
$rootScope.annotations.push(data);
});
{
'annotation' : {'text': $scope.selection_text.trim()}
}, function(data) {
$rootScope.annotations.push(data);
}, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
}
// hide selection highlighted text and the menu
// hide the highlighted text the the menu
$(".text-panel").removeClass("selection");
$element.fadeOut(100);
};
}
]);
/*
* Text highlighting controller
*/
window.annotationsApp.controller('IntraTextController',
['$scope', '$rootScope', '$compile', 'NgramHttpService',
function ($scope, $rootScope, $compile, NgramHttpService) {
$scope.extra_stoplist = [];
$scope.extra_miamlist = [];
$scope.currentStopPage = 0;
$scope.currentMiamPage = 0;
$scope.pageSize = 15;
var counter = 0;
/*
* Replace the text by and html template
* Replace the text by an html template for ngram keywords
*/
function replaceTextByTemplate(text, annotation, template, pattern, lists) {
function replaceTextByTemplate(text, ngram, template, pattern, lists) {
return text.replace(pattern, function(matched) {
var tpl = angular.element(template);
tpl.append(matched);
tpl.attr('title', annotation.tooltip_content);
tpl.attr('uuid', annotation.uuid);
if ('MiamList' == lists[annotation.list_id]) tpl.addClass("miamword");
if ('StopList' == lists[annotation.list_id]) tpl.addClass("stopword");
//if (annotation.category == 'stoplist' && annotation.level == 'global') tpl.addClass("global-stopword");
tpl.attr('title', ngram.tooltip_content);
tpl.attr('uuid', ngram.uuid);
/*
* Add CSS class depending on the list the ngram is into
* FIXME Lists names and css classes are fixed, can do better
*/
tpl.addClass(ngram.listName);
return tpl.get(0).outerHTML;
});
}
function compileText(annotations, fullText, abstractText, $rootScope) {
/*
* Sorts annotations on the number of words
* Required for overlapping ngrams
*/
function lengthSort(listitems, valuekey) {
listitems.sort(function(a, b) {
var compA = a[valuekey].split(" ").length;
var compB = b[valuekey].split(" ").length;
return (compA > compB) ? -1 : (compA <= compB) ? 1 : 0;
});
return listitems;
}
/*
* Match and replace Ngram into the text
*/
function compileNgramsHtml(annotations, textMapping, $rootScope) {
// TODO remove this debug counter
counter = 0;
var templateBegin = "<span ng-controller='AnnotationController' ng-click='onClick($event)' class='keyword-inline'>";
var templateBeginRegexp = "<span ng-controller='AnnotationController' ng-click='onClick\(\$event\)' class='keyword-inline'>";
......@@ -251,97 +317,96 @@
var startPattern = "\\b((?:"+templateBeginRegexp+")*";
var middlePattern = "(?:<\/span>)*\\s(?:"+templateBeginRegexp+")*";
var endPattern = "(?:<\/span>)*)\\b";
/*
* Sorts annotations on the number of words
*/
function lengthSort(listitems, valuekey) {
listitems.sort(function(a, b) {
var compA = a[valuekey].split(" ").length;
var compB = b[valuekey].split(" ").length;
return (compA > compB) ? -1 : (compA <= compB) ? 1 : 0;
});
return listitems;
}
var sortedSizeAnnotations = lengthSort(annotations, "text");
var extra_stoplist = [],
extra_miamlist = [];
var sortedSizeAnnotations = lengthSort(annotations, "text"),
extraNgramList = angular.copy($rootScope.extraNgramList);
_.each(sortedSizeAnnotations, function (annotation) {
// TODO better split to manage two-words with minus sign
annotation.category = $rootScope.lists[annotation.list_id].toLowerCase();
// reinitialize an empty list
extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = [];
});
angular.forEach(sortedSizeAnnotations, function (annotation) {
// exclude ngrams that are into inactive lists
if ($rootScope.activeLists[annotation.list_id] === undefined) return;
// used to setup css class
annotation.listName = $rootScope.lists[annotation.list_id];
// regexps
var words = annotation.text.split(" ");
var pattern = new RegExp(startPattern + words.join(middlePattern) + endPattern, 'gmi');
var textRegexp = new RegExp("\\b"+annotation.text+"\\b", 'igm');
var isDisplayedIntraText = false;
// highlight text as html
angular.forEach(textMapping, function(text, eltId) {
if (pattern.test(text) === true) {
textMapping[eltId] = replaceTextByTemplate(text, annotation, template, pattern, $rootScope.lists);
// TODO remove debug
counter++;
isDisplayedIntraText = true;
}
});
if (pattern.test(fullText) === true) {
fullText = replaceTextByTemplate(fullText, annotation, template, pattern, $rootScope.lists);
// TODO remove debug
counter++;
} else if (pattern.test(abstractText) === true) {
abstractText = replaceTextByTemplate(abstractText, annotation, template, pattern, $rootScope.lists);
counter++;
} else if (!textRegexp.test($rootScope.full_text) && !textRegexp.test($rootScope.abstract_text)) {
if (annotation.category == "stoplist") {
// Deactivated stoplist for the moment
// if ($.inArray(annotation.uuid, $scope.extra_stoplist.map(function (item) {
// return item.uuid;
// })) == -1) {
// extra_stoplist = lengthSort(extra_stoplist.concat(annotation), "text");
// }
} else if (annotation.category == "miamlist") {
if ($.inArray(annotation.uuid, $scope.extra_miamlist.map(function (item) {
return item.uuid;
})) == -1) {
extra_miamlist = lengthSort(extra_miamlist.concat(annotation), "text");
}
if (!isDisplayedIntraText) {
// add extra-text ngrams that are not already displayed
if ($.inArray(annotation.uuid, extraNgramList[annotation.list_id].map(function (item) {
return item.uuid;
})) == -1) {
// push the ngram and sort
extraNgramList[annotation.list_id] = extraNgramList[annotation.list_id].concat(annotation);
}
}
});
$scope.extra_stoplist = extra_stoplist;
$scope.extra_miamlist = extra_miamlist;
return {
'fullTextHtml': fullText,
'abstractTextHtml': abstractText
};
// update extraNgramList
$rootScope.extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = lengthSort(extraNgramList[id], 'text');
});
// return the object of element ID with the corresponding HTML
return textMapping;
}
/*
* Listen changes on the ngram data
*/
$rootScope.$watchCollection('annotations', function (newValue, oldValue) {
if ($rootScope.annotations === undefined) return;
if (angular.equals(newValue, oldValue)) return;
$rootScope.miamListId = _.invert($rootScope.lists)['MiamList'];
$rootScope.stopListId = _.invert($rootScope.lists)['StopList'];
// initialize extraNgramList
var extraNgramList = {};
$rootScope.extraNgramList = angular.forEach($rootScope.activeLists, function(name, id) {
this[id] = [];
}, extraNgramList);
$rootScope.extraNgramList = extraNgramList;
$scope.extra_stoplist = [];
$scope.extra_miamlist = [];
var result = compileText(
/*
* Transform text into HTML with higlighted ngrams
*/
var result = compileNgramsHtml(
$rootScope.annotations,
angular.copy($rootScope.full_text),
angular.copy($rootScope.abstract_text),
{
'#full-text': angular.copy($rootScope.full_text),
'#abstract-text': angular.copy($rootScope.abstract_text),
'#title': angular.copy($rootScope.title)
},
$rootScope
);
$.each($rootScope.annotations, function(index, element) {
if (element.list_id == $rootScope.stopListId) {
$scope.extra_stoplist.push(element);
} else if (element.list_id == $rootScope.miamListId) {
$scope.extra_miamlist.push(element);
}
// inject highlighted HTML
angular.forEach(result, function(html, eltId) {
angular.element(eltId).html(html);
});
angular.element('#full-text').html(result.fullTextHtml);
angular.element('#abstract-text').html(result.abstractTextHtml);
// inject one Angular controller on every highlighted text element
angular.element('.text-container').find('[ng-controller=AnnotationController]').each(function(idx, elt) {
angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true)));
});
});
function submitNewAnnotation($event, inputEltId, listId) {
/*
* Add a new NGram from the user input in the extra-text list
*/
$scope.onListSubmit = function ($event, listId) {
var inputEltId = "#"+ listId +"-input";
if ($event.keyCode !== undefined && $event.keyCode != 13) return;
var value = $(inputEltId).val().trim();
if (value === "") return;
......@@ -350,47 +415,54 @@
'listId': listId,
'ngramId': 'new'
},
{'annotation' : {'text': value}},
{
'annotation' : {'text': value}
},
function(data) {
// on success
if (data) {
$rootScope.annotations.push(data);
$(inputEltId).val("");
}
});
}, function(data) {
// on error
$(inputEltId).parent().addClass("has-error");
console.error("error adding Ngram "+ value);
}
);
};
$(inputEltId).val("");
}
}
]);
$scope.onMiamlistSubmit = function ($event) {
submitNewAnnotation($event, "#miamlist-input", _.invert($rootScope.lists)['MiamList']);
};
// TODO refactor
$scope.onStoplistSubmit = function ($event) {
submitNewAnnotation($event, "#stoplist-input", _.invert($rootScope.lists)['MiamList']);
};
$scope.numStopPages = function () {
if ($scope.extra_stoplist === undefined) return 0;
return Math.ceil($scope.extra_stoplist.length / $scope.pageSize);
};
$scope.numMiamPages = function () {
if ($scope.extra_miamlist === undefined) return 0;
return Math.ceil($scope.extra_miamlist.length / $scope.pageSize);
};
$scope.nextMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage + 1;
};
$scope.previousMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage - 1;
/*
* Controller for one List Tab displaying extra-text ngram
*/
window.annotationsApp.controller('ExtraTextPaginationController',
['$scope', '$rootScope', function ($scope, $rootScope) {
$rootScope.$watchCollection('extraNgramList', function (newValue, oldValue) {
$scope.currentListPage = 0;
$scope.pageSize = 15;
$scope.nextListPage = function() {
$scope.currentListPage = $scope.currentListPage + 1;
};
$scope.nextStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage + 1;
$scope.previousListPage = function() {
$scope.currentListPage = $scope.currentListPage - 1;
};
$scope.previousStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage - 1;
$scope.totalListPages = function (listId) {
if ($rootScope.extraNgramList[listId] === undefined) return 0;
return Math.ceil($rootScope.extraNgramList[listId].length / $scope.pageSize);
};
}
]);
});
}]);
/*
* Filter used in Ngram flat lists pagination (extra-text panel)
*/
window.annotationsApp.filter('startFrom', function () {
return function (input, start) {
if (input === undefined) return;
......@@ -405,28 +477,32 @@
$rootScope.documentResource = DocumentHttpService.get(
{'docId': $rootScope.docId},
function(data, responseHeaders) {
$scope.title = data.title;
$scope.authors = data.authors;
$scope.journal = data.journal;
$scope.publication_date = data.publication_date;
// TODO this data have to be deleted
//$scope.current_page_number = data.current_page_number;
//$scope.last_page_number = data.last_page_number;
// put in rootScope because used by many components
$rootScope.title = data.title;
$rootScope.docId = data.id;
$rootScope.full_text = data.full_text;
$rootScope.abstract_text = data.abstract_text;
// GET the annotations
// TODO
// GET the annotationss
$rootScope.annotationsResource = NgramListHttpService.get(
{'corpusId': $rootScope.corpusId, 'docId': $rootScope.docId}
).$promise.then(function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.lists = data[$rootScope.corpusId.toString()]['lists'];
});
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.lists = data[$rootScope.corpusId.toString()].lists;
// TODO active list selection controller
$rootScope.activeLists = angular.copy($rootScope.lists);
$rootScope.mainListId = _.invert($rootScope.activeLists).MiamList;
}
);
});
// TODO setup pagination client-side
// TODO setup article pagination
$scope.onPreviousClick = function () {
DocumentHttpService.get($scope.docId - 1);
};
......@@ -435,8 +511,11 @@
};
}]);
/*
* Main function
* GET the document node and all its ngrams
*/
window.annotationsApp.run(function ($rootScope) {
/* GET the document node and all the annotations in the list associated */
var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//);
$rootScope.projectId = path[1];
$rootScope.corpusId = path[2];
......
annotations/static/annotations/http.js
View file @ 25584756
......@@ -57,11 +57,11 @@
{
post: {
method: 'POST',
params: {'listId': '@listId', 'ngramId': ''}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
},
delete: {
method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@id'}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
}
}
);
......
annotations/static/annotations/keyword_tpl.html
View file @ 25584756
<span ng-if="keyword.category == 'miamlist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="miamlist">×</span>
<span ng-if="keyword.category == 'miamlist'" data-toggle="tooltip" class="keyword miamword">{[{keyword.text}]}</span>
<span ng-if="keyword.category == 'stoplist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="stoplist">×</span>
<span ng-if="keyword.category == 'stoplist'" data-toggle="tooltip" class="keyword stopword">{[{keyword.text}]}</span>
<span ng-click='onDeleteClick()' class="delete-keyword">×</span>
<span data-toggle="tooltip" class="keyword-text {[{keyword.listName}]}">{[{keyword.text}]}</span>
<span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span>
annotations/static/annotations/selection_tpl.html deleted 100644 → 0
View file @ d6cc0e32
<ul class="noselection">
<li class="miamword" ng-if="local_miamlist === true" ng-click="onClick($event, 'post', miamListId, 'local')">add to miam-list</li>
<li class="miamword" ng-if="local_miamlist === false" ng-click="onClick($event, 'delete', miamListId, 'local')">remove from miam-list</li>
<li class="stopword" ng-if="local_stoplist === true" ng-click="onClick($event, 'post', stopListId, 'local')">add to stop-list</li>
<li class="stopword" ng-if="local_stoplist === false" ng-click="onClick($event, 'delete', stopListId, 'local')">remove from stop-list</li>
<!--<li class="stopword" ng-if="global_stoplist === true" ng-click="onClick($event, 'post', 'stoplist', 'global')">add to global stop-list</li>
<li class="stopword" ng-if="global_stoplist === false" ng-click="onClick($event, 'delete', 'stoplist', 'global')">remove from global stop-list</li>-->
</ul>
annotations/templates/annotations/main.html
View file @ 25584756
......@@ -9,8 +9,8 @@
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>Gargantext corpus annotations editor</title>
<meta name="description" content="">
<title>Gargantext article editor</title>
<meta name="description" content="Gargantext">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}">
......@@ -18,51 +18,38 @@
<script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script>
</head>
<body>
<!-- TODO integrate this later into the corpus.html django template -->
<!-- TODO integrate this later into the any other django template -->
<div id="annotationsApp">
<div class="container-fluid">
<div class="row-fluid main-panel" ng-controller="IntraTextController">
<div class="col-md-4 col-xs-4 tabbable words-panel">
<ul class="nav nav-pills nav-justified">
<li class="active"><a href="#tab1" data-toggle="tab">Miamwords</a></li>
<li><a href="#tab2" data-toggle="tab">Stopwords</a></li>
<li ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}">
<a href="#tab-{[{listId}]}" data-toggle="tab">{[{listName}]}</a>
</li>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="tab1">
<div ng-if="extra_miamlist.length == 0" class="alert alert-info" role="alert">No extra text miam-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_miamlist | startFrom:currentMiamPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li>
</ul>
<nav ng-class="{invisible: numMiamPages() - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentMiamPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentMiamPage >= numMiamPages()-1}"><a ng-click="nextMiamPage()" class="glyphicon glyphicon-forward"></a></li>
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="miamlist-input" ng-keypress="onMiamlistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onMiamlistSubmit($event)">Add</button>
<div ng-controller="ExtraTextPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}">
<div ng-if="extraNgramList[listId].length == 0" class="alert alert-info" role="alert">
Input any keyword you want to link to this article and the list named '{[{listName}]}'
</div>
</div>
<div class="tab-pane" id="tab2">
<div ng-if="extra_stoplist.length == 0" class="alert alert-info" role="alert">No extra text stop-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_stoplist | startFrom:currentStopPage*pageSize | limitTo:pageSize" class="list-group-item">
<ul class="list-group words-list clearfix">
<li ng-repeat="keyword in extraNgramList[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li>
</ul>
<nav ng-class="{invisible: numStopPages() - 1 == 0}" class="clearfix">
<nav ng-class="{invisible: totalListPages - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentStopPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentStopPage >= numStopPages()-1}"><a ng-click="nextStopPage()" class="glyphicon glyphicon-forward"></a></li>
<li ng-class="{disabled: currentListPage == 0}"><a ng-click="previousListPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentListPage >= totalListPages - 1}"><a ng-click="nextListPage()" class="glyphicon glyphicon-forward"></a></li>
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="stoplist-input" ng-keypress="onStoplistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onStoplistSubmit($event)">Add</button>
<input type="text" class="form-control" id="{[{ listId }]}-input" ng-keypress="onListSubmit($event, listId)">
<button type="submit" class="form-control btn btn-default btn-primary" ng-click="onListSubmit($event, listId)">Add to {[{listName}]}</button>
</div>
</div>
</div>
......@@ -70,7 +57,7 @@
<div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document">
<div class="row-fluid clearfix">
<div class="col-md-7 col-xs-7">
<h3>{[{title}]}</h3>
<h3 class="text-container" id="title">{[{title}]}</h3>
</div>
<div class="col-md-5 col-xs-5">
<nav>
......@@ -90,17 +77,21 @@
</div>
<h4 ng-if="abstract_text != null">Abstract</h4>
<p id="abstract-text" class="text-container">
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">No abstract text</div>
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">Empty abstract text</div>
</p>
<h4 ng-if="full_text != null">Full Article</h4>
<p id="full-text" class="text-container">
<div ng-if="full_text == null" class="alert alert-info" role="alert">No full text</div>
<div ng-if="full_text == null" class="alert alert-info" role="alert">Empty full text</div>
</p>
</div>
</div> <!-- end of the main row -->
</div>
<!-- this menu is over the text -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu" selection-template></div>
<!-- this menu is over the text on mouse selection -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu">
<ul class="noselection">
<li ng-repeat="item in menuItems" class="{[{item.listName}]}" ng-click="onMenuClick($event, item.action, item.listId)">{[{item.verb}]} {[{item.listName}]}</li>
</ul>
</div>
</div>
<!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
......
annotations/views.py
View file @ 25584756
......@@ -40,15 +40,16 @@ class NgramList(APIView):
"""Get All for a doc id"""
corpus_id = int(corpus_id)
doc_id = int(doc_id)
lists = dict()
lists = {}
for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
# ngrams of list_id of corpus_id:
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id)
#doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
doc_ngram_list = [(i, 'miam', i, 1931) for i in range(500)]
doc_ngram_list += [(i, 'stop', i, 1932) for i in range(501, 600)]
# doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
data = { '%s' % corpus_id : {
'%s' % doc_id : [
......
gargantext.ini
View file @ 25584756
......@@ -39,6 +39,10 @@ vacuum = true
pidfile = /tmp/gargantext.pid
# touch /tmp/gargantext.reload to reload configuration (after git pull for instance)
touch-reload = /tmp/gargantext.reload
# respawn processes taking more than 20 seconds
harakiri = 120
......
gargantext_web/about.py
View file @ 25584756
......@@ -29,6 +29,11 @@ def get_team():
'picture' : 'david.jpg',
'role':'principal investigator'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
{ 'first_name' : 'Mathieu', 'last_name' : 'Rodic',
'mail' : '',
'website' : 'http://rodic.fr',
......@@ -41,11 +46,6 @@ def get_team():
'picture' : 'samuel.jpg',
'role' : 'developer'},
{ 'first_name' : 'Elias', 'last_name' : 'Showk',
'mail' : '',
'website' : 'https://github.com/elishowk',
'picture' : '', 'role' : 'developer'},
#{ 'first_name' : '', 'name' : '', 'mail' : '', 'website' : '', 'picture' : ''},
# copy paste the line above and write your informations please
]
......
gargantext_web/corpus_views.py
View file @ 25584756
......@@ -43,7 +43,9 @@ from scrappers.scrap_pubmed.admin import Logger
from gargantext_web.db import *
from sqlalchemy import or_, func
from sqlalchemy.sql import func
from sqlalchemy import desc, asc, or_, and_, Date, cast, select
from gargantext_web import about
......@@ -128,7 +130,7 @@ def get_ngrams(request , project_id , corpus_id ):
'processing' : processing,
'number' : number,
'list_id': list_id[0][0],
'view' : "Terms",
'view' : "terms",
}))
return HttpResponse(html)
......@@ -195,7 +197,7 @@ def get_journals(request , project_id , corpus_id ):
'corpus' : corpus,
'processing' : processing,
'number' : number,
'view' : "Journals",
'view' : "journals",
}))
return HttpResponse(html)
......@@ -205,16 +207,18 @@ def test_journals(request , project_id, corpus_id ):
JournalsDict = {}
user_id = request.user.id
document_type_id = cache.NodeType['Document'].id
documents = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
for doc in documents:
if "journal" in doc.hyperdata:
journal = doc.hyperdata["journal"]
if journal not in JournalsDict:
JournalsDict [journal] = 0
JournalsDict[journal] += 1
return JsonHttpResponse(JournalsDict)
journal_count = (session.query(Node.hyperdata['journal'].label("journal"), func.count().label("count"))
.filter(Node.parent_id == corpus_id, Node.type_id==cache.NodeType['Document'].id)
.group_by("journal")
.order_by(desc("count"))
.all())
jc = dict()
for journal,count in journal_count:
jc[journal] = count
return JsonHttpResponse(jc)
def test_ngrams(request , project_id, corpus_id ):
results = ["hola" , "mundo"]
......@@ -341,7 +345,11 @@ def test_ngrams(request , project_id, corpus_id ):
ngrams_ids = Ngrams_Scores.keys()
import math
occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) )
if len(ngrams_ids) != 0:
occs_threshold = min ( 10 , math.sqrt(Sum / len(ngrams_ids)) )
else:
occs_threshold = 10
Metrics = {
"ngrams":[],
......
gargantext_web/urls.py
View file @ 25584756
......@@ -48,16 +48,20 @@ urlpatterns = patterns('',
# Corpus management
# Document view (main)
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
url(r'^project/(\d+)/corpus/(\d+)/documents/?$', views.corpus),
# Journals view
url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals$', corpus_views.get_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals', corpus_views.get_journals),
# Terms view
url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', corpus_views.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/ngrams$', corpus_views.get_ngrams),
############################################################################
# # Terms view
url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', corpus_views.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/terms/?$', corpus_views.get_ngrams),
# Update corpus
url(r'^project/(\d+)/corpus/(\d+)/(\w+)/update$', views.update_nodes),
############################################################################
# annotations App
url(r'^project/(\d+)/corpus/(\d+)/document/(\d+)/$', annotations_main_view),
url(r'^annotations/', include(annotations_urls)),
......@@ -104,7 +108,9 @@ urlpatterns = patterns('',
url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX),
url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test),
url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', samtest.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/terms', samtest.get_ngrams)
)
......
gargantext_web/views.py
View file @ 25584756
from django.shortcuts import redirect
from django.shortcuts import render
from admin.utils import PrintException
from django.shortcuts import redirect, render, render_to_response
from django.db import transaction
from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden
from django.template.loader import get_template
from django.template import Context
from django.template import Context, RequestContext
# remove this
from node import models
......@@ -43,7 +44,7 @@ from sqlalchemy import or_, func
from gargantext_web import about
from gargantext_web.db import NodeNgram, NodeNgramNgram
def login_user(request):
logout(request)
......@@ -251,6 +252,71 @@ def projects(request):
})
def update_nodes(request, project_id, corpus_id, view):
'''
update function:
- remove previous computations (temporary lists and coocurrences)
- permanent deletion of Trash
'''
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = int(project_id)
offset = int(corpus_id)
offset = str(view)
except ValueError:
raise Http404()
try:
print(corpus_id)
whites = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['WhiteList'].id)
.all()
)
print(whites)
for white in whites:
session.query(NodeNgram).filter(NodeNgram.node_id==white.id).delete()
session.delete(white)
cooc = (session.query(Node)
.filter(Node.parent_id==corpus_id)
.filter(Node.type_id == cache.NodeType['Cooccurrence'].id)
.first()
)
session.query(NodeNgramNgram).filter(NodeNgramNgram.node_id==cooc.id).delete()
session.delete(cooc)
session.commit()
except :
PrintException()
nodes = models.Node.objects.filter(type_id=cache.NodeType['Trash'].id, user_id=request.user.id).all()
if nodes is not None:
with transaction.atomic():
for node in nodes:
try:
node.children.delete()
except Exception as error:
print(error)
node.delete()
#return redirect(request.path.replace('update', ''))
return redirect('/project/%s/corpus/%s/%s' % (project_id, corpus_id, view))
#
# return render_to_response(
# request.path,
# { 'title': 'Corpus view' },
# context_instance=RequestContext(request)
# )
#
def corpus(request, project_id, corpus_id):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
......@@ -287,7 +353,7 @@ def corpus(request, project_id, corpus_id):
'processing' : processing,
# 'documents': documents,\
'number' : number,
'view' : "Documents"
'view' : "documents"
}))
return HttpResponse(html)
......@@ -499,9 +565,10 @@ def graph(request, project_id, corpus_id):
results[i.id]["corpuses"].append(info)
# print("\t",j.id , j.name)
import pprint
pprint.pprint(results)
# import pprint
# pprint.pprint(results)
graphurl = "corpus/"+str(corpus_id)+"/node_link.json"
html = t.render(Context({\
'debug': settings.DEBUG,
'user' : user,\
......@@ -509,7 +576,7 @@ def graph(request, project_id, corpus_id):
'corpus' : corpus,\
'project' : project,\
'corpusinfo' : results,\
'graphfile' : "hola_mundo",\
'graphfile' : graphurl,\
}))
return HttpResponse(html)
......@@ -626,14 +693,14 @@ def node_link(request, corpus_id):
data = []
corpus = session.query(Node).filter(Node.id==corpus_id).first()
filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id)
print("file exists?:",os.path.isfile(filename))
if os.path.isfile(filename):
json_data = open(filename,"r")
data = json.load(json_data)
json_data.close()
else:
data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
# filename = settings.MEDIA_ROOT + '/corpora/%s/%s_%s.json' % (request.user , corpus.parent_id, corpus_id)
# print("file exists?:",os.path.isfile(filename))
# if os.path.isfile(filename):
# json_data = open(filename,"r")
# data = json.load(json_data)
# json_data.close()
# else:
data = get_cooc(request=request, corpus_id=corpus_id, type="node_link")
return JsonHttpResponse(data)
def adjacency(request, corpus_id):
......
parsing/FileParsers/ISTex.py
View file @ 25584756
......@@ -19,7 +19,7 @@ class ISTex(FileParser):
"source" : 'corpusName',
"title" : 'title',
"genre" : "genre",
# "language_iso3" : 'MedlineCitation/Article/Language',
"language_iso3" : 'language',
"doi" : 'doi',
"host" : 'host',
"publication_date" : 'pubdate',
......@@ -27,18 +27,23 @@ class ISTex(FileParser):
"authorsRAW" : 'author',
"keywords" : "keywords"
}
suma = 0
for json_doc in json_docs:
hyperdata = {}
for key, path in hyperdata_path.items():
try:
# print(path," ==> ",len(json_doc[path]))
hyperdata[key] = json_doc[path]
except: pass
except:
pass
# print("|",hyperdata["publication_date"])
# print("|",hyperdata["language_iso3"])
if "doi" in hyperdata: hyperdata["doi"] = hyperdata["doi"][0]
if "doi" in hyperdata:
hyperdata["doi"] = hyperdata["doi"][0]
keywords = []
if "keywords" in hyperdata:
......@@ -75,34 +80,53 @@ class ISTex(FileParser):
if "genre" in hyperdata:
if len(hyperdata["genre"])==0:
hyperdata.pop("genre")
if "language_iso3" in hyperdata:
if len(hyperdata["language_iso3"])>0:
hyperdata["language_iso3"] = hyperdata["language_iso3"][0]
else:
hyperdata["language_iso3"] = "eng"
RealDate = hyperdata["publication_date"]
if "publication_date" in hyperdata: hyperdata.pop("publication_date")
if "publication_date" in hyperdata:
RealDate = hyperdata["publication_date"]
if "publication_date" in hyperdata:
hyperdata.pop("publication_date")
Decision=""
if len(RealDate)>4:
if len(RealDate)>8:
try: Decision = datetime.strptime(RealDate, '%Y-%b-%d').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
if isinstance(RealDate, list):
RealDate = RealDate[0]
# print( RealDate ," | length:",len(RealDate))
Decision=""
if len(RealDate)>4:
if len(RealDate)>8:
try: Decision = datetime.strptime(RealDate, '%Y-%b-%d').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%b').date()
except:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
else:
try: Decision = datetime.strptime(RealDate, '%Y-%m-%d').date()
except: Decision=False
if Decision!=False:
hyperdata["publication_year"] = str(Decision.year)
hyperdata["publication_month"] = str(Decision.month)
hyperdata["publication_day"] = str(Decision.day)
hyperdata_list.append(hyperdata)
# print("\t||",hyperdata["title"])
# print("\t\t",Decision)
# print("=============================")
try: Decision = datetime.strptime(RealDate, '%Y').date()
except: Decision=False
if Decision!=False:
hyperdata["publication_year"] = str(Decision.year)
hyperdata["publication_month"] = str(Decision.month)
hyperdata["publication_day"] = str(Decision.day)
hyperdata_list.append(hyperdata)
# print("\t||",hyperdata["title"])
# print("\t\t",Decision)
# print("=============================")
# else:
# suma+=1
# if "pubdate" in json_doc:
# print ("\tfail pubdate:",json_doc["pubdate"])
# print ("nb_hits:",len(json_docs))
# print("\t - nb_fails:",suma)
# print(" -- - - - - - -- - -")
return hyperdata_list
scrappers/scrap_pubmed/views.py
View file @ 25584756
......@@ -66,7 +66,7 @@ def getGlobalStatsISTEXT(request ):
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
query_string = query.replace(" ","+")
url = "http://api.istex.fr/document/?q="+query_string
url = "http://api.istex.fr/document/?q="+query_string+"&output=id,title,abstract,pubdate,corpusName,authors,language"
tasks = MedlineFetcher()
......
static/js/Docs_dyna_chart_and_table.js
View file @ 25584756
......@@ -127,19 +127,9 @@ for(var i=0; i<url_elems.length; i++) {
url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
}
}
var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
// $.ajax({
// url: theurl,
// success: function(data) {
// bisarray = data.data
// for(var i in bisarray) {
// untitlebis = bisarray[i].values
// BIS_dict[untitlebis[0]] = [bisarray[i].count , 0];// [ total amount , removed ]
// }
// pr(BIS_dict)
// if(Object.keys(BIS_dict).length>0) $("#delAll").css("visibility", "visible"); $("#delAll").show();
// }
// });
......@@ -168,7 +158,7 @@ function getRecords() {
function transformContent2(rec_id) {
// pr("\t\ttransformContent2: "+rec_id)
var elem = AjaxRecords[rec_id];
// pr("\t\t\t"+elem.date)
// pr("\t"+elem.title)
var result = {}
if (elem["del"]) {
result["id"] = elem["id"]
......@@ -261,208 +251,272 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
}
function Main_test( Data , SearchFilter ) {
var DistributionDict = {}
for(var i in DistributionDict)
delete DistributionDict[i];
delete DistributionDict;
DistributionDict = {}
$("#div-table").html("")
$("#div-table").empty();
var div_table = '<p align="right">'+"\n"
div_table += '<table id="my-ajax-table" class="table table-bordered">'+"\n"
div_table += "\t"+'<thead>'+"\n"
div_table += "\t"+"\t"+'<th width="100px;" data-dynatable-column="date">Date</th>'+"\n"
div_table += "\t"+"\t"+'<th data-dynatable-column="name">Title</th>'+"\n"
div_table += "\t"+"\t"+'<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>'+"\n"
div_table += "\t"+"\t"+'</th>'+"\n"
div_table += "\t"+'</thead>'+"\n"
div_table += "\t"+'<tbody>'+"\n"
div_table += "\t"+'</tbody>'+"\n"
div_table += '</table>'+"\n"
div_table += '</p>';
$("#div-table").html(div_table)
var justdates = {}
for(var i in Data) {
var date = Data[i]["date"];
if ( ! justdates[date] ) justdates[date] = 0;
justdates[date]++;
// console.log(Data[i]["date"]+" : originalRecords["+arr_id+"] <- "+orig_id+" | "+Data[i]["name"])
}
// var div__filter_for_search = ''
// div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple data-max-options="1">';
// div__filter_for_search += ' <optgroup label="All" data-max-options="1" selected>';
// div__filter_for_search += ' <option>Title</option>';
// div__filter_for_search += ' <option>Date</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Category" data-max-options="1">';
// div__filter_for_search += ' <option>Title</option>';
// div__filter_for_search += ' <option>Date</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Duplicates" data-max-options="1">';
// div__filter_for_search += ' <option>by DOI</option>';
// div__filter_for_search += ' <option>by Title</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += '</select>';
// $("#supmofos").html(div__filter_for_search)
// $("#move2trash").prop('disabled', true);
// (3) Get records and hyperdata for paginator
$.ajax({
url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
success: function(data){
console.log(data)
var justdates = {}
for(var i in data.records) {
var orig_id = parseInt(data.records[i].id)
var arr_id = parseInt(i)
RecDict[orig_id] = arr_id;
data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["del"] = false
var date = data.records[i]["date"];
if ( ! justdates[date] ) justdates[date] = 0;
justdates[date]++;
// console.log(data.records[i]["date"]+" : originalRecords["+arr_id+"] <- "+orig_id+" | "+data.records[i]["name"])
}
AjaxRecords = data.records; // backup!!
// $("#move2trash").prop('disabled', true);
var t0 = AjaxRecords[0].date.split("-").map(Number)
var t1 = AjaxRecords.slice(-1)[0].date.split("-").map(Number)
oldest = t0;
latest = t1;
TheBuffer = [new Date(t0[0],(t0[1]-1),t0[2]), new Date(t1[0],(t1[1]-1),t1[2])];
var t0 = AjaxRecords[0].date.split("-").map(Number)
var t1 = AjaxRecords.slice(-1)[0].date.split("-").map(Number)
oldest = t0;
latest = t1;
var arrayd3 = []
for(var e in Data) {
var date = Data[e]["date"];
if(justdates[date]!=false) {
var info = {}
info.date = date
info.dd = dateFormat.parse(date)
info.month = d3.time.month(info.dd)
info.volume = justdates[date]
arrayd3.push(info)
justdates[date] = false;
}
}
for(var i in justdates)
delete justdates[i];
delete justdates;
var ndx = crossfilter(arrayd3);
var all = ndx.groupAll();
//volumeChart:(1)
//moveChart:(1)
// monthly index avg fluctuation in percentage
var moveMonths = ndx.dimension(function (d) {
return d.month;
});
TheBuffer = [new Date(t0[0],(t0[1]-1),t0[2]), new Date(t1[0],(t1[1]-1),t1[2])];
//moveChart:(3)
var monthlyMoveGroup = moveMonths.group().reduceSum(function (d) {
return d.volume;
//return Math.abs(+d.close - +d.open);
});
//volumeChart:(2)
var volumeByMonthGroup = moveMonths.group().reduceSum(function (d) {
return d.volume / 500000;
});
var arrayd3 = []
for(var e in data.records) {
var date = data.records[e]["date"];
if(justdates[date]!=false) {
var info = {}
info.date = date
info.dd = dateFormat.parse(date)
info.month = d3.time.month(info.dd)
info.volume = justdates[date]
arrayd3.push(info)
justdates[date] = false;
//moveChart:(2)
var indexAvgByMonthGroup = moveMonths.group().reduce(
function (p, v) {
++p.days;
p.total += (+v.open + +v.close) / 2;
p.avg = Math.round(p.total / p.days);
return p;
},
function (p, v) {
--p.days;
p.total -= (+v.open + +v.close) / 2;
p.avg = p.days == 0 ? 0 : Math.round(p.total / p.days);
return p;
},
function () {
return {days: 0, total: 0, avg: 0};
}
}
);
moveChart.width(800)
.height(150)
.transitionDuration(1000)
.margins({top: 10, right: 50, bottom: 25, left: 40})
.dimension(moveMonths)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.x(d3.time.scale().domain([new Date(t0[0],t0[1],t0[2]), new Date(t1[0],t1[1],t1[2])]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.elasticY(true)
.renderHorizontalGridLines(true)
.brushOn(false)
.compose([
dc.lineChart(moveChart)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.renderArea(true)
.stack(monthlyMoveGroup, function (d) {
return d.value;
})
.title(function (d) {
var value = d.value.avg ? d.value.avg : d.value;
if (isNaN(value)) value = 0;
return dateFormat(d.key) + "\n" + numberFormat(value);
})
])
.xAxis();
volumeChart.width(800)
.height(100)
.margins({top: 0, right: 50, bottom: 20, left: 40})
.dimension(moveMonths)
.group(volumeByMonthGroup)
.centerBar(true)
.gap(0)
.x(d3.time.scale().domain([new Date(t0[0],t0[1],t0[2]), new Date(t1[0],t1[1],t1[2])]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.renderlet(function (chart) {
chart.select("g.y").style("display", "none");
moveChart.filter(chart.filter());
})
.on("filtered", function (chart) {
dc.events.trigger(function () {
var chartfilt = chart.filter()
// tricky part: identifying when the moveChart changes.
if(chartfilt) {
Push2Buffer ( chart.filter() )
} else {
if(TheBuffer) {
Push2Buffer ( false )
}
}
moveChart.focus(chartfilt);
});
});
for(var i in justdates)
delete justdates[i];
delete justdates;
var ndx = crossfilter(arrayd3);
var all = ndx.groupAll();
//volumeChart:(1)
//moveChart:(1)
// monthly index avg fluctuation in percentage
var moveMonths = ndx.dimension(function (d) {
return d.month;
});
//moveChart:(3)
var monthlyMoveGroup = moveMonths.group().reduceSum(function (d) {
return d.volume;
//return Math.abs(+d.close - +d.open);
});
//volumeChart:(2)
var volumeByMonthGroup = moveMonths.group().reduceSum(function (d) {
return d.volume / 500000;
});
//moveChart:(2)
var indexAvgByMonthGroup = moveMonths.group().reduce(
function (p, v) {
++p.days;
p.total += (+v.open + +v.close) / 2;
p.avg = Math.round(p.total / p.days);
return p;
dc.renderAll();
MyTable = []
MyTable = $('#my-ajax-table').dynatable({
dataset: {
records: Data
},
function (p, v) {
--p.days;
p.total -= (+v.open + +v.close) / 2;
p.avg = p.days == 0 ? 0 : Math.round(p.total / p.days);
return p;
features: {
pushState: false,
// sort: false //i need to fix the sorting function... the current one just sucks
},
function () {
return {days: 0, total: 0, avg: 0};
writers: {
_rowWriter: ulWriter
// _cellWriter: customCellWriter
}
);
moveChart.width(800)
.height(150)
.transitionDuration(1000)
.margins({top: 10, right: 50, bottom: 25, left: 40})
.dimension(moveMonths)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.x(d3.time.scale().domain([new Date(t0[0],t0[1],t0[2]), new Date(t1[0],t1[1],t1[2])]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.elasticY(true)
.renderHorizontalGridLines(true)
.brushOn(false)
.compose([
dc.lineChart(moveChart)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.renderArea(true)
.stack(monthlyMoveGroup, function (d) {
return d.value;
})
.title(function (d) {
var value = d.value.avg ? d.value.avg : d.value;
if (isNaN(value)) value = 0;
return dateFormat(d.key) + "\n" + numberFormat(value);
})
])
.xAxis();
volumeChart.width(800)
.height(100)
.margins({top: 0, right: 50, bottom: 20, left: 40})
.dimension(moveMonths)
.group(volumeByMonthGroup)
.centerBar(true)
.gap(0)
.x(d3.time.scale().domain([new Date(t0[0],t0[1],t0[2]), new Date(t1[0],t1[1],t1[2])]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.renderlet(function (chart) {
chart.select("g.y").style("display", "none");
moveChart.filter(chart.filter());
})
.on("filtered", function (chart) {
dc.events.trigger(function () {
var chartfilt = chart.filter()
// tricky part: identifying when the moveChart changes.
if(chartfilt) {
Push2Buffer ( chart.filter() )
} else {
if(TheBuffer) {
Push2Buffer ( false )
}
}
moveChart.focus(chartfilt);
});
});
});
MyTable.data('dynatable').paginationPage.set(1);
MyTable.data('dynatable').process();
if ( $(".imadiv").length>0 ) return 1;
$('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
$(".dynatable-record-count").insertAfter(".imadiv")
$(".dynatable-pagination-links").insertAfter(".imadiv")
$("#filter_search").html( $("#filter_search").html().replace('selected="selected"') );
$("#"+SearchFilter).attr( "selected" , "selected" )
var the_content = $("#filter_search").html();
$(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
// .insertAfter("#dynatable-query-search-my-ajax-table")
return "OK"
}
dc.renderAll();
MyTable = $('#my-ajax-table').dynatable({
dataset: {
records: data.records
},
features: {
pushState: false,
sort: false //i need to fix the sorting function... the current one just sucks
},
writers: {
_rowWriter: ulWriter
// _cellWriter: customCellWriter
}
});
if ( $(".imadiv").length>0 ) return 1;
$('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
$(".dynatable-record-count").insertAfter(".imadiv")
$(".dynatable-pagination-links").insertAfter(".imadiv")
var the_content = $("#supmofos").html();
$(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
$("#supmofos").remove()
// .insertAfter("#dynatable-query-search-my-ajax-table")
function SearchFilters( elem ) {
var MODE = elem.value;
if( MODE == "filter_all") {
var result = Main_test(AjaxRecords , MODE)
console.log( result )
}
if( MODE == "filter_dupl-titles") {
var getDupl_API = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
$.ajax({
url: getDupl_API,
success: function(data) {
bisarray = data.data
for(var i in bisarray) {
titlebis = bisarray[i].values
BIS_dict[titlebis[0]] = true;
}
var Duplicates = []
for(var r in AjaxRecords) {
if ( BIS_dict[AjaxRecords[r].title] )
Duplicates.push( AjaxRecords[r] )
}
var result = Main_test(Duplicates , MODE)
console.log( result )
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('title', 1) // 1=ASCENDING,
MyTable.data('dynatable').process();
}
});
}
}
// FIRST portion of code to be EXECUTED:
// (3) Get records and hyperdata for paginator
$.ajax({
url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
success: function(data){
for(var i in data.records) {
var orig_id = parseInt(data.records[i].id)
var arr_id = parseInt(i)
RecDict[orig_id] = arr_id;
data.records[i]["title"] = data.records[i]["name"];
data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["del"] = false
}
});
AjaxRecords = data.records; // backup-ing in global variable!
var result = Main_test(data.records , "filter_all")
console.log( result )
},
});
static/js/Journals_dyna_chart_and_table.js
View file @ 25584756
......@@ -471,11 +471,17 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').process();
// // // $("#score_column_id").children()[0].text = FirstScore
// // // // MyTable.data('dynatable').process();
......
static/js/NGrams_dyna_chart_and_table.js
View file @ 25584756
......@@ -370,15 +370,18 @@ $("#Clean_All").click(function(){
});
$("#Save_All").click(function(){
console.log("click in save all 01")
var sum__selected_elems = 0;
var poubelle = []
for(var i in FlagsBuffer)
if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
for(var i in FlagsBuffer) {
if (Object.keys(FlagsBuffer[i]).length==0)
poubelle.push(i)
sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
}
console.log("click in save all 02")
for(var i in poubelle)
delete FlagsBuffer[poubelle[i]];
console.log("click in save all 03, sum:"+sum__selected_elems)
if ( sum__selected_elems>0 ) {
console.log("")
console.log("Do the ajax conexion with API and send this array to be processed:")
......@@ -498,11 +501,6 @@ function Main_test( data , initial) {
oldest = Number(min_occ);
latest = Number(max_occ);
var ndx = false;
ndx = crossfilter();
ndx.add(DistributionList);
......@@ -624,6 +622,9 @@ function Main_test( data , initial) {
// MyTable.data('dynatable').settings.dataset.originalRecords = []
// MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
MyTable.data('dynatable').sorts.clear();
MyTable.data('dynatable').sorts.add('score', 0) // 1=ASCENDING,
MyTable.data('dynatable').process();
MyTable.data('dynatable').paginationPage.set(1);
// MyTable.data('dynatable').process();
// MyTable.data('dynatable').sorts.clear();
......@@ -661,7 +662,7 @@ $.ajax({
// Building the Score-Selector
var FirstScore = data.scores.initial
var possible_scores = Object.keys( data.ngrams[0].scores );
var scores_div = '<select class="span1" id="scores_selector">'+"\n";
var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
for( var i in possible_scores ) {
if(possible_scores[i]!=FirstScore) {
......
templates/corpus/bannerBottome.html deleted 100644 → 0
View file @ d6cc0e32
<!--<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>--!>
templates/corpus/documents.html
View file @ 25584756
......@@ -4,7 +4,7 @@
{% load staticfiles %}
<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
<!-- here goes import stylesheet js/bootstrap/bootstrap-multiselect.css, mais ca marche pas-->
<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
......@@ -72,6 +72,7 @@ th a {
<input type="hidden" id="list_id" value="{{ list_id }}"></input>
<div class="container">
<div class="jumbotron">
......@@ -87,18 +88,7 @@ th a {
</div>
<div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
<div class="panel-body">
<p align="right">
<table id="my-ajax-table" class="table table-bordered">
<thead>
<!-- <th data-dynatable-column="id">ID</th> -->
<th width="100px;" data-dynatable-column="date">Date</th>
<th data-dynatable-column="name">Title</th>
<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>
</thead>
<tbody>
</tbody>
</table>
</p>
<div id="div-table"></div>
<p align="right">
<button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button>
</p>
......@@ -111,24 +101,26 @@ th a {
</div>
</div>
<div id="supmofos">
<select data-width="100px" dir="ltr" class="selectpicker">
<option selected>All</option>
<option>Title</option>
<option>Date</option>
<optgroup label="Duplicates">
<option>by DOI</option>
<option>by Title</option>
</optgroup>
</select>
<div id="filter_search" style="visibility:hidden">
<select id="example-single-optgroups" onchange="SearchFilters(this);">
<!-- <optgroup label=""> -->
<option id="filter_all" value="filter_all">All</option>
<!-- <option id="filter_title" value="filter_title">Title</option> -->
<!-- <option id="filter_date" value="filter_date">Date</option> -->
<!-- </optgroup> -->
<!-- <optgroup label="Duplicates"> -->
<!-- <option value="filter_doi">By DOI</option> -->
<option id="filter_dupl-titles" value="filter_dupl-titles">Duplicates by Title</option>
<!-- </optgroup> -->
</select>
</div>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
<!-- here goes import script js/bootstrap/bootstrap-multiselect.js, mais ca marche pas-->
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js -->
......
templates/corpus/menu.html
View file @ 25584756
{% extends "menu.html" %}
{% load staticfiles %}
{% block corpusBannerTop %}
......@@ -10,7 +11,7 @@
<div class="col-md-4">
{% if project %}
<h1>{{ project.name }}</h1>
<h1><a href="/project/{{project.id}}">{{ project.name }}</a></h1>
<h2>{{ corpus.name }}</h2>
{% endif %}
<!--<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
......@@ -32,9 +33,9 @@
<div class="col-md-4">
<br>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{corpus.id}}/{{view}}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Download</a>
<a type="button" class="btn btn-default btn-lg" data-container="body" data-toggle="popover" data-placement="bottom"
data-content='
<ul>
......@@ -62,9 +63,12 @@
<div class="btn-group btn-group-justified">
<center>
<a type="button" class="btn btn-default {% if view == "Documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/">{{number}} Documents</a>
<a type="button" class="btn btn-default {% if view == "Journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a>
<a type="button" class="btn btn-default {% if view == "Terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/ngrams">Terms</a>
<a type="button" class="btn btn-default {% if view == "documents" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/documents">{{number}} Documents</a>
<a type="button" class="btn btn-default {% if view == "journals" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/journals">Journals</a>
{% if processing == 0 %}
<a type="button" class="btn btn-default {% if view == "terms" %}active{%endif%}" href="/project/{{project.id}}/corpus/{{ corpus.id }}/terms">Terms</a>
{% endif %}
</center>
</div>
......@@ -76,7 +80,7 @@
<div class="container">
<div class="row">
<div class="col-md-4">
<div class="col-md-6">
<div class="jumbotron">
<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
<ol>
......@@ -88,23 +92,11 @@
</div>
</div>
<div class="col-md-4">
<div class="jumbotron">
<!-- <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3> -->
<h3>Matrix (soon)</h3>
<ol>
<li>Sort</li>
<li>Group</li>
<li>Cluster</li>
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
<div class="col-md-4">
<div class="col-md-6">
<div class="jumbotron">
{% if processing > 0 %}
<h3> Graph (later)</h3>
<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
{% else %}
<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
{% endif %}
......
templates/explorer.html
View file @ 25584756
......@@ -60,7 +60,7 @@
<li>
<a>
<div id="graphid" style="visibility: hidden;">/corpus/{{ corpus.id }}/node_link.json</div>
<div id="graphid" style="visibility: hidden;">{{graphfile}}</div>
</a>
</li>
......@@ -177,9 +177,7 @@
<div style="visibility: hidden;" id="sigma-othergraph"></div>
<div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
</div>
<div id="semLoader"></div>
<ul id="ctlzoom">
......
templates/project.html
View file @ 25584756
......@@ -87,7 +87,7 @@
{% ifequal corpus.processing 1 %}
{{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
{% else %}
<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} </a> , {{ corpus.count }} Documents
<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> {{corpus.name}} , {{ corpus.count }} Documents </a>
{% endifequal %}
<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
data-content='
......
test-list-management.py
View file @ 25584756
......@@ -6,8 +6,8 @@ from ngram.lists import *
#from cooccurrences import *
#from gargantext_web.views import empty_trash
#empty_trash()
from gargantext_web.views import empty_trash
empty_trash()
#user = session.query(User).all()[0]
......@@ -77,6 +77,7 @@ print('Stem Node.id is', stem_id)
# print(n, listNgramIds(list_id=n[0][0])[:3])
#
stop_list_id = listIds(user_id=user.id,
corpus_id=corpus.id,
typeList='StopList')[0][0]
......@@ -86,13 +87,13 @@ miam_list_id = listIds(user_id=user.id,
typeList='MiamList')[0][0]
print('Stop List', stop_list_id)
print('Miam List', miam_list_id)
print('StopList', stop_list_id)
print('MiamList', miam_list_id)
ngrams2miam(user_id=user.id, corpus_id=corpus.id)
print(session.query(Node.id).filter(Node.parent_id==corpus.id, Node.type_id==cache.NodeType['WhiteList'].id).first())
#ngrams2miam(user_id=user.id, corpus_id=corpus.id)
print(listNgramIds(list_id=miam_list_id, user_id=user.id, corpus_id=corpus.id))
#print(listNgramIds(list_id=stop_list_id, user_id=user.id, corpus_id=corpus.id))
......
tests/ngramstable/views.py
View file @ 25584756
......@@ -100,7 +100,7 @@ def get_ngrams(request , project_id , corpus_id ):
except ValueError:
raise Http404()
t = get_template('tests/ngrams.html')
t = get_template('corpus/terms.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
......@@ -173,7 +173,7 @@ def get_journals(request , project_id , corpus_id ):
except ValueError:
raise Http404()
t = get_template('tests/journals.html')
t = get_template('corpus/journals.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment