Skip to content

gargantext
gargantext
Commit 0b4a7eae authored by Administrator's avatar Administrator
Browse files
Options

Merge branch 'unstable-mat' into unstable

parents d8d27f26 b7ccc435
Show whitespace changes
Inline Side-by-side
Showing with 977 additions and 594 deletions
analysis/lists.py
View file @ 0b4a7eae
from collections import defaultdict from collections import defaultdict
from math import sqrt
from gargantext_web.db import session, NodeNgram, NodeNgramNgram, bulk_insert
class Translations:
class BaseClass:
def __add__(self, other):
if hasattr(self, '__radd__'):
return self.__radd__(other)
else:
return NotImplemented
def __sub__(self, other):
if hasattr(self, '__rsub__'):
return self.__rsub__(other)
else:
return NotImplemented
def __mul__(self, other):
if hasattr(self, '__rmul__'):
return self.__rmul__(other)
else:
return NotImplemented
def __div__(self, other):
if hasattr(self, '__rdiv__'):
return self.__rdiv__(other)
else:
return NotImplemented
def __and__(self, other):
if hasattr(self, '__rand__'):
return self.__rand__(other)
else:
return NotImplemented
def __or__(self, other):
if hasattr(self, '__ror__'):
return self.__ror__(other)
else:
return NotImplemented
def __repr__(self):
items = self.items
if isinstance(items, defaultdict):
if len(items) and isinstance(next(iter(items.values())), defaultdict):
items = {
key: dict(value)
for key, value in items.items()
}
else:
items = dict(items)
return '<%s %s>' % (
self.__class__.__name__,
repr(items),
)
__str__ = __repr__
class Translations(BaseClass):
def __init__(self, other=None): def __init__(self, other=None):
if other is None: if other is None:
self.items = defaultdict(int) self.items = defaultdict(int)
self.groups = defaultdict(set) self.groups = defaultdict(set)
elif isinstance(other, int):
query = (session
.query(NodeNgramNgram.ngramy_id, NodeNgramNgram.ngramx_id)
.filter(NodeNgramNgram.node_id == other)
)
self.items = defaultdict(int, query)
self.groups = defaultdict(set)
for key, value in self.items.items():
self.groups[value].add(key)
elif isinstance(other, Translations): elif isinstance(other, Translations):
self.items = other.items.copy() self.items = other.items.copy()
self.groups = other.groups.copy() self.groups = other.groups.copy()
...@@ -18,39 +86,69 @@ class Translations: ...@@ -18,39 +86,69 @@ class Translations:
else: else:
raise TypeError raise TypeError
def __add__(self, other): def __rmul__(self, other):
result = self.__class__(self) result = NotImplemented
result.items.update(other) if isinstance(other, UnweightedList):
for key, value in other.groups: result = UnweightedList()
result.groups[key] += value result.items = set(
return result self.items.get(key, key)
for key in other.items
def __sub__(self, other): )
result = self.__class__(self) elif isinstance(other, WeightedList):
if isinstance(other, Translations): result = WeightedList()
for key, value in other.items.items(): for key, value in other.items.items():
result.items.pop(key, None) result.items[
result.groups[value].remove(key) self.items.get(key, key)
if len(result.groups[value]) == 0: ] += value
result.groups.pop(value) elif isinstance(other, Translations):
result = Translations()
items = self.items
items.update(other.items)
for key, value in items.items():
if value in items:
value = items[value]
if key != value:
result.items[key] = value
result.groups[value].add(key)
return result return result
def __iter__(self): def __iter__(self):
for key, value in self.items.items(): for key, value in self.items.items():
yield key, value yield key, value
def save(self, node_id):
# delete previous data
session.query(NodeNgramNgram).filter(NodeNgramNgram.node_id == node_id).delete()
session.commit()
# insert new data
bulk_insert(
NodeNgramNgram,
('node_id', 'ngramy_id', 'ngramx_id', 'score'),
((node_id, key, value, 1.0) for key, value in self.items.items())
)
class WeightedMatrix: class WeightedMatrix(BaseClass):
def __init__(self, other=None): def __init__(self, other=None):
if other is None: if other is None:
self.items = defaultdict(lambda: defaultdict(float)) self.items = defaultdict(lambda: defaultdict(float))
elif isinstance(other, int):
query = (session
.query(NodeNgramNgram.ngramx_id, NodeNgramNgram.ngramy_id, NodeNgramNgram.score)
.filter(NodeNgramNgram.node_id == other)
)
self.items = defaultdict(lambda: defaultdict(float))
for key1, key2, value in self.items.items():
self.items[key1][key2] = value
elif isinstance(other, WeightedMatrix): elif isinstance(other, WeightedMatrix):
self.items = other.items.copy() self.items = defaultdict(lambda: defaultdict(float))
for key1, key2, value in other:
self.items[key1][key2] = value
elif hasattr(other, '__iter__'): elif hasattr(other, '__iter__'):
self.items = defaultdict(lambda: defaultdict(float)) self.items = defaultdict(lambda: defaultdict(float))
for row in other: for row in other:
self.items[other[0]][other[1]] = [other[2]] self.items[row[0]][row[1]] = row[2]
else: else:
raise TypeError raise TypeError
...@@ -59,36 +157,103 @@ class WeightedMatrix: ...@@ -59,36 +157,103 @@ class WeightedMatrix:
for key2, value in key2_value.items(): for key2, value in key2_value.items():
yield key1, key2, value yield key1, key2, value
def __sub__(self, other): def save(self, node_id):
"""Remove elements of the other list from the current one # delete previous data
Can only be substracted to another list of coocurrences. session.query(NodeNgramNgram).filter(NodeNgramNgram.node_id == node_id).delete()
""" session.commit()
pass # insert new data
bulk_insert(
NodeNgramNgram,
('node_id', 'ngramx_id', 'ngramy_id', 'score'),
((node_id, key1, key2, value) for key1, key2, value in self)
)
def __mul__(self, other): def __radd__(self, other):
result = NotImplemented
if isinstance(other, WeightedMatrix):
result = WeightedMatrix()
for key1, key2, value in self:
value = value + other.items[key1][key2]
if value != 0.0:
result.items[key1][key2] = value
return result
def __rsub__(self, other):
result = NotImplemented
if isinstance(other, (UnweightedList, WeightedList)):
result = WeightedMatrix()
for key1, key2, value in self:
if key1 in other.items or key2 in other.items:
continue
result.items[key1][key2] = value
elif isinstance(other, WeightedMatrix):
result = WeightedMatrix()
for key1, key2, value in self:
value = value - other.items[key1][key2]
if value != 0.0:
result.items[key1][key2] = value
return result
def __rand__(self, other):
result = NotImplemented
if isinstance(other, (UnweightedList, WeightedList)):
result = WeightedMatrix()
for key1, key2, value in self:
if key1 not in other.items or key2 not in other.items:
continue
result.items[key1][key2] = value
return result
def __rmul__(self, other):
result = NotImplemented
if isinstance(other, Translations): if isinstance(other, Translations):
result = WeightedMatrix() result = WeightedMatrix()
for key1, key2_value in self.items.items(): for key1, key2_value in self.items.items():
for key2, value in self.items: key1 = other.items.get(key1, key1)
result.items[ for key2, value in key2_value.items():
other.items.get(key, key) result.items[key1][
] = value other.items.get(key2, key2)
else: ] += value
raise TypeError elif isinstance(other, UnweightedList):
result = self.__rand__(other)
# elif isinstance(other, WeightedMatrix):
# result = WeightedMatrix()
elif isinstance(other, WeightedList):
result = WeightedMatrix()
for key1, key2, value in self:
if key1 not in other.items or key2 not in other.items:
continue
result.items[key1][key2] = value * sqrt(other.items[key1] * other.items[key2])
return result
def __rdiv__(self, other):
result = NotImplemented
if isinstance(other, WeightedList):
result = WeightedMatrix()
for key1, key2, value in self:
if key1 not in other.items or key2 not in other.items:
continue
result.items[key1][key2] = value / sqrt(other.items[key1] * other.items[key2])
return result return result
class UnweightedList: class UnweightedList(BaseClass):
def __init__(self, other=None): def __init__(self, other=None):
if other is None: if other is None:
self.items = set() self.items = set()
elif isinstance(other, int):
query = (session
.query(NodeNgram.ngram_id)
.filter(NodeNgram.node_id == other)
)
self.items = {row[0] for row in query}
elif isinstance(other, WeightedList): elif isinstance(other, WeightedList):
self.items = set(other.items.keys()) self.items = set(other.items.keys())
elif isinstance(other, UnweightedList): elif isinstance(other, UnweightedList):
self.items = other.items.copy() self.items = other.items.copy()
elif hasattr(other, '__iter__'): elif hasattr(other, '__iter__'):
items = (item for item in other) items = tuple(item for item in other)
if len(items) == 0: if len(items) == 0:
self.items = set() self.items = set()
else: else:
...@@ -99,44 +264,86 @@ class UnweightedList: ...@@ -99,44 +264,86 @@ class UnweightedList:
else: else:
raise TypeError raise TypeError
def __add__(self, other): def __radd__(self, other):
result = self.__class__(self) result = NotImplemented
if isinstance(other, UnweightedList): if isinstance(other, UnweightedList):
result.items |= other.items result = UnweightedList(other)
result.items |= self.items
elif isinstance(other, WeightedList): elif isinstance(other, WeightedList):
result.items |= set(other.items.keys()) result = WeightedList(other)
else: for key in self.items:
raise TypeError result.items[key] += 1.0
return result return result
__or__ = __add__ def __rsub__(self, other):
result = NotImplemented
def __sub__(self, other):
result = self.__class__(self)
if isinstance(other, UnweightedList): if isinstance(other, UnweightedList):
result = UnweightedList(self)
result.items -= other.items result.items -= other.items
elif isinstance(other, WeightedList): elif isinstance(other, WeightedList):
result = UnweightedList(self)
result.items -= set(other.items.keys()) result.items -= set(other.items.keys())
else:
raise TypeError
return result return result
def __and__(self, other): def __ror__(self, other):
result = self.__class__(self) result = NotImplemented
if isinstance(other, UnweightedList):
result = UnweightedList(self)
result.items |= other.items
elif isinstance(other, WeightedList):
result = UnweightedList(self)
result.items |= set(other.items.keys())
return result
def __rand__(self, other):
result = NotImplemented
if isinstance(other, UnweightedList): if isinstance(other, UnweightedList):
result = UnweightedList(self)
result.items &= other.items result.items &= other.items
elif isinstance(other, WeightedList): elif isinstance(other, WeightedList):
result.items &= set(other.items.keys()) result = UnweightedList(self)
else: result.items &= set(other.items)
raise TypeError
return result return result
def __rmul__(self, other):
result = NotImplemented
if isinstance(other, Translations):
result = UnweightedList()
result.items = set(
other.items.get(key, key)
for key in self.items
)
elif isinstance(other, UnweightedList):
result = WeightedList(self)
result.items = {key: 1.0 for key in self.items & other.items}
elif isinstance(other, WeightedList):
result = WeightedList()
result.items = {key: value for key, value in other.items.items() if key in self.items}
return result
class WeightedList: def save(self, node_id):
# delete previous data
session.query(NodeNgram).filter(NodeNgram.node_id == node_id).delete()
session.commit()
# insert new data
bulk_insert(
NodeNgram,
('node_id', 'ngram_id', 'weight'),
((node_id, key, 1.0) for key in self.items)
)
class WeightedList(BaseClass):
def __init__(self, other=None): def __init__(self, other=None):
if other is None: if other is None:
self.items = defaultdict(float) self.items = defaultdict(float)
elif isinstance(other, int):
query = (session
.query(NodeNgram.ngram_id, NodeNgram.weight)
.filter(NodeNgram.node_id == other)
)
self.items = defaultdict(float, query)
elif isinstance(other, WeightedList): elif isinstance(other, WeightedList):
self.items = other.items.copy() self.items = other.items.copy()
elif isinstance(other, UnweightedList): elif isinstance(other, UnweightedList):
...@@ -144,7 +351,7 @@ class WeightedList: ...@@ -144,7 +351,7 @@ class WeightedList:
for key in other.items: for key in other.items:
self.items[key] = 1.0 self.items[key] = 1.0
elif hasattr(other, '__iter__'): elif hasattr(other, '__iter__'):
self.items = defaultdict(float, items) self.items = defaultdict(float, other)
else: else:
raise TypeError raise TypeError
...@@ -152,60 +359,135 @@ class WeightedList: ...@@ -152,60 +359,135 @@ class WeightedList:
for key, value in self.items.items(): for key, value in self.items.items():
yield key, value yield key, value
def __add__(self, other): def __radd__(self, other):
"""Add elements from the other list to the current one result = NotImplemented
""" if isinstance(other, WeightedList):
result = self.__class__(self) result = WeightedList(self)
if isinstance(other, UnweightedList): for key, value in other.items.items():
for key, value in other.items:
result.items[key] += 1.0
elif isinstance(other, WeightedList):
for key, value in other.items:
result.items[key] += value result.items[key] += value
else: elif isinstance(other, UnweightedList):
raise TypeError result = WeightedList(self)
for key in other.items:
result.items[key] += 1.0
return result return result
def __sub__(self, other): def __rsub__(self, other):
"""Remove elements of the other list from the current one """Remove elements of the other list from the current one
""" """
result = self.__class__(self) result = NotImplemented
if isinstance(other, UnweightedList): if isinstance(other, UnweightedList):
for key in other.items: result = WeightedList()
result.items.pop(key, None) result.items = {key: value for key, value in self.items.items() if key not in other.items}
elif isinstance(other, WeightedList):
result = WeightedList(self)
for key, value in other.items.items():
if key in result.items and result.items[key] == value:
result.items.pop(key)
else: else:
raise TypeError result.items[key] -= value
return result return result
def __and__(self, other): def __ror__(self, other):
result = NotImplemented
if isinstance(other, UnweightedList): if isinstance(other, UnweightedList):
result = defaultdict(float) result = UnweightedList(self)
for key, value in self.items.items(): result.items |= other.items
if item in other.items: elif isinstance(other, WeightedList):
result[key] = value result = UnweightedList(self)
else: result.items |= set(other.items.keys())
raise TypeError
return result return result
def __mul__(self, other): def __rmul__(self, other):
if isinstance(other, Translations): result = NotImplemented
if isinstance(other, WeightedList):
result = WeightedList()
result.items = {
key: value * other.items[key]
for key, value
in self.items.items()
if key in other.items
}
if isinstance(other, UnweightedList):
result = WeightedList()
result.items = {
key: value
for key, value
in self.items.items()
if key in other.items
}
elif isinstance(other, Translations):
result = WeightedList() result = WeightedList()
for key, value in self.items: for key, value in self.items.items():
result.items[ result.items[
other.items.get(key, key) other.items.get(key, key)
] += value ] += value
else:
raise TypeError
return result return result
def __rand__(self, other):
result = NotImplemented
if isinstance(other, UnweightedList):
result = UnweightedList(self)
result.items &= other.items
elif isinstance(other, WeightedList):
result = UnweightedList(self)
result.items &= set(other.items.keys())
return result
def save(self, node_id):
# delete previous data
session.query(NodeNgram).filter(NodeNgram.node_id == node_id).delete()
session.commit()
# insert new data
bulk_insert(
NodeNgram,
('node_id', 'ngram_id', 'weight'),
((node_id, key, value) for key, value in self.items.items())
)
def test():
from collections import OrderedDict
# define operands
operands = OrderedDict()
operands['wm'] = WeightedMatrix(((1, 2, .5), (1, 3, .75), (2, 3, .6), (3, 3, 1), ))
operands['ul'] = UnweightedList((1, 2, 3, 4, 5))
# operands['ul2'] = UnweightedList((1, 2, 3, 6))
# operands['ul2'].save(5)
# operands['ul3'] = UnweightedList(5)
operands['wl'] = WeightedList({1:.7, 2:.8, 7: 1.1})
# operands['wl1'].save(5)
# operands['wl2'] = WeightedList(5)
# operands['t1'] = Translations({1:2, 4:5})
operands['t'] = Translations({3:2, 4:5})
# operands['t2'].save(5)
# operands['t3'] = Translations(5)
# define operators
operators = OrderedDict()
operators['+'] = '__add__'
operators['-'] = '__sub__'
operators['*'] = '__mul__'
operators['|'] = '__or__'
operators['&'] = '__and__'
# show operands
for operand_name, operand in operands.items():
print('%4s = %s' % (operand_name, operand))
# show operations results
for operator_name, operator in operators.items():
print()
for operand1_name, operand1 in operands.items():
for operand2_name, operand2 in operands.items():
if hasattr(operand1, operator):
result = getattr(operand1, operator)(operand2)
else:
result = '?'
print('%4s %s %-4s = %s' % (
operand1_name,
operator_name,
operand2_name,
'?' if result == NotImplemented else result,
))
if __name__ == '__main__':
test()
# if __name__ == '__main__':
# l = Coocurrences()
# l = List()
# for i in l:
# print(i)
# t1 = Translations()
# t2 = Translations()
# t2.items = {1: 2}
# for i in t1 + t2:
# print(i)
annotations/static/annotations/app.css
View file @ 0b4a7eae
/* app css stylesheet */ /* app css stylesheet */
/*
* Class names corresponding to server-side list names
* To display another list name, please add a new class under this
*/
.MiamList {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.StopList {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.delete-keyword, .occurrences { .delete-keyword, .occurrences {
vertical-align: super; vertical-align: super;
font-size: 70%; font-size: 70%;
...@@ -27,27 +43,9 @@ ...@@ -27,27 +43,9 @@
border-bottom: none; border-bottom: none;
} }
.miamword {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.stopword {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.global-stopword {
color: black;
background-color: rgba(169, 68, 66, 0.05);
cursor: pointer;
}
.main-panel, .text-panel, .words-panel { .main-panel, .text-panel, .words-panel {
height: 800px; height: 800px;
margin: 10px 0px; margin: 10px 0;
} }
.text-panel { .text-panel {
...@@ -59,11 +57,7 @@ ...@@ -59,11 +57,7 @@
height: 250px; height: 250px;
} }
.keyword-container { .keyword-text {
/*display: inline-block;*/
}
.keyword {
word-break: break-all; word-break: break-all;
} }
......
annotations/static/annotations/app.js
View file @ 0b4a7eae
(function () { (function () {
'use strict'; 'use strict';
/*
* Django STATIC_URL given to angular to load async resources
*/
var S = window.STATIC_URL; var S = window.STATIC_URL;
window.annotationsApp = angular.module('annotationsApp', ['annotationsAppHttp']); window.annotationsApp = angular.module('annotationsApp', ['annotationsAppHttp']);
/*
* Angular Templates must not conflict with Django's
*/
window.annotationsApp.config(function($interpolateProvider) { window.annotationsApp.config(function($interpolateProvider) {
$interpolateProvider.startSymbol('{[{'); $interpolateProvider.startSymbol('{[{');
$interpolateProvider.endSymbol('}]}'); $interpolateProvider.endSymbol('}]}');
}); });
/*
* Template of the ngram element displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.directive('keywordTemplate', function () { window.annotationsApp.directive('keywordTemplate', function () {
return { return {
templateUrl: function ($element, $attributes) { templateUrl: function ($element, $attributes) {
...@@ -18,25 +27,36 @@ ...@@ -18,25 +27,36 @@
}; };
}); });
/*
* For ngram elements displayed in the flat lists (called "extra-text")
*/
window.annotationsApp.controller('ExtraAnnotationController', window.annotationsApp.controller('ExtraAnnotationController',
['$scope', '$rootScope', '$element', 'NgramHttpService', ['$scope', '$rootScope', '$element', 'NgramHttpService',
function ($scope, $rootScope, $element, NgramHttpService) { function ($scope, $rootScope, $element, NgramHttpService) {
// TODO use the tooltip ? /*
* Click on the 'delete' cross button
*/
$scope.onDeleteClick = function () { $scope.onDeleteClick = function () {
NgramHttpService.delete({ NgramHttpService.delete({
'listId': $scope.keyword.list_id, 'listId': $scope.keyword.list_id,
'ngramId': $scope.keyword.uuid 'ngramId': $scope.keyword.uuid
}).$promise.then(function(data) { }, function(data) {
$.each($rootScope.annotations, function(index, element) { $.each($rootScope.annotations, function(index, element) {
if (element.list_id == $scope.keyword.list_id && element.uuid == $scope.keyword.uuid) { if (element.list_id == $scope.keyword.list_id && element.uuid == $scope.keyword.uuid) {
$rootScope.annotations.splice(index, 1); $rootScope.annotations.splice(index, 1);
return false; return false;
} }
}); });
}, function(data) {
console.log(data);
console.error("unable to delete the Ngram " + $scope.keyword.uuid);
}); });
} };
}]); }]);
/*
* For mouse selection on the text
*/
window.annotationsApp.controller('AnnotationController', window.annotationsApp.controller('AnnotationController',
['$scope', '$rootScope', '$element', ['$scope', '$rootScope', '$element',
function ($scope, $rootScope, $element) { function ($scope, $rootScope, $element) {
...@@ -57,14 +77,9 @@ ...@@ -57,14 +77,9 @@
}; };
}]); }]);
window.annotationsApp.directive('selectionTemplate', function () { /*
return { * Controller of the menu over the current mouse selection
templateUrl: function ($element, $attributes) { */
return S + 'annotations/selection_tpl.html';
}
};
});
window.annotationsApp.controller('AnnotationMenuController', window.annotationsApp.controller('AnnotationMenuController',
['$scope', '$rootScope', '$element', '$timeout', 'NgramHttpService', ['$scope', '$rootScope', '$element', '$timeout', 'NgramHttpService',
function ($scope, $rootScope, $element, $timeout, NgramHttpService) { function ($scope, $rootScope, $element, $timeout, NgramHttpService) {
...@@ -87,9 +102,12 @@ ...@@ -87,9 +102,12 @@
} }
return false; return false;
} }
// we only need one singleton at a time
var selection = getSelected(); var selection = getSelected();
/*
* When mouse selection is started, we highlight it
*/
function toggleSelectionHighlight(text) { function toggleSelectionHighlight(text) {
if (text.trim() !== "") { if (text.trim() !== "") {
$(".text-panel").addClass("selection"); $(".text-panel").addClass("selection");
...@@ -98,53 +116,68 @@ ...@@ -98,53 +116,68 @@
} }
} }
/*
* Dynamically construct the selection menu scope
*/
function toggleMenu(context, annotation) { function toggleMenu(context, annotation) {
$timeout(function() { $timeout(function() {
$scope.$apply(function() { $scope.$apply(function() {
var miamlist_id = _.invert($rootScope.activeLists).MiamList;
if (angular.isObject(annotation)) { var stoplist_id = _.invert($rootScope.activeLists).StopList;
$scope.level = angular.copy(annotation.level || 'global'); // variable used in onClick
$scope.category = $rootScope.lists[annotation.list_id].toLowerCase();
$scope.listId = angular.copy(annotation.list_id);
// used in onClick
$scope.selection_text = angular.copy(annotation); $scope.selection_text = angular.copy(annotation);
if ($scope.category == "miamlist") { if (angular.isObject(annotation)) {
$scope.local_miamlist = false; // existing ngram
$scope.global_stoplist = true; // Delete from the current list
$scope.local_stoplist = true; $scope.menuItems = [
} else if ($scope.category == "stoplist") { {
'action': 'delete',
if ($scope.level == "local") { 'listId': annotation.list_id,
$scope.local_stoplist = false; 'verb': 'Delete from',
$scope.global_stoplist = true; 'listName': $rootScope.lists[annotation.list_id]
} }
if ($scope.level == "global") { ];
$scope.global_stoplist = false; if ($rootScope.lists[annotation.list_id] == "MiamList") {
$scope.local_stoplist = true; // Add to the alternative list
} $scope.menuItems.push({
$scope.local_miamlist = true; 'action': 'post',
'listId': stoplist_id,
'verb': 'Add to',
'listName': $rootScope.lists[stoplist_id]
});
} else if ($rootScope.lists[annotation.list_id] == "StopList") {
// Add to the alternative list
$scope.menuItems.push({
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.lists[miamlist_id]
});
} }
// show menu // show the menu
$element.fadeIn(100); $element.fadeIn(100);
} else if (annotation.trim() !== "") {
// new ngram
$scope.menuItems = [
{
'action': 'post',
'listId': miamlist_id,
'verb': 'Add to',
'listName': $rootScope.activeLists[miamlist_id]
} }
else if (annotation.trim() !== "") { ];
$scope.selection_text = angular.copy(annotation); // show the menu
$scope.level = "New Ngram from selection";
$scope.category = null;
$scope.local_miamlist = true;
$scope.local_stoplist = true;
$scope.global_stoplist = true;
// show menu
$element.fadeIn(100); $element.fadeIn(100);
} else { } else {
// close menu $scope.menuItems = [];
// close the menu
$element.fadeOut(100); $element.fadeOut(100);
} }
}); });
}); });
} }
var elt = $(".text-panel")[0];
var pos = $(".text-panel").position(); var pos = $(".text-panel").position();
function positionElement(context, x, y) { function positionElement(context, x, y) {
...@@ -157,18 +190,26 @@ ...@@ -157,18 +190,26 @@
positionElement(null, e.pageX, e.pageY); positionElement(null, e.pageX, e.pageY);
} }
// TODO is mousedown necessary ? /*
$(".text-panel").mousedown(function(){ * Dynamically position the menu
$(".text-panel").mousemove(positionMenu); */
$(".text-container").mousedown(function(){
$(".text-container").mousemove(positionMenu);
}); });
$(".text-panel").mouseup(function(){ /*
$(".text-panel").unbind("mousemove", positionMenu); * Finish positioning the menu then display the menu
*/
$(".text-container").mouseup(function(){
$(".text-container").unbind("mousemove", positionMenu);
toggleSelectionHighlight(selection.toString().trim()); toggleSelectionHighlight(selection.toString().trim());
toggleMenu(null, selection.toString().trim()); toggleMenu(null, selection.toString().trim());
}); });
$(".text-panel").delegate(':not("#selection")', "click", function(e) { /*
* Toggle the menu when clicking on an existing ngram keyword
*/
$(".text-container").delegate(':not("#selection")', "click", function(e) {
if ($(e.target).hasClass("keyword-inline")) return; if ($(e.target).hasClass("keyword-inline")) return;
positionMenu(e); positionMenu(e);
toggleSelectionHighlight(selection.toString().trim()); toggleSelectionHighlight(selection.toString().trim());
...@@ -178,81 +219,80 @@ ...@@ -178,81 +219,80 @@
$rootScope.$on("positionAnnotationMenu", positionElement); $rootScope.$on("positionAnnotationMenu", positionElement);
$rootScope.$on("toggleAnnotationMenu", toggleMenu); $rootScope.$on("toggleAnnotationMenu", toggleMenu);
$scope.onClick = function($event, action, listId, level) { /*
* Menu click action
*/
$scope.onMenuClick = function($event, action, listId) {
if (angular.isObject($scope.selection_text)) { if (angular.isObject($scope.selection_text)) {
// delete from the current list // action on an existing Ngram
NgramHttpService[action]({ NgramHttpService[action]({
'listId': listId, 'listId': listId,
'ngramId': $scope.selection_text.uuid 'ngramId': $scope.selection_text.uuid
}).$promise.then(function(data) { }, function(data) {
$.each($rootScope.annotations, function(index, element) { $.each($rootScope.annotations, function(index, element) {
if (element.list_id == listId && element.uuid == $scope.selection_text.uuid) { if (element.list_id == listId && element.uuid == $scope.selection_text.uuid) {
$rootScope.annotations.splice(index, 1); $rootScope.annotations.splice(index, 1);
return false; return false;
} }
}); });
}); }, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
} else if ($scope.selection_text.trim() !== "") { } else if ($scope.selection_text.trim() !== "") {
// new annotation from selection // new annotation from selection
NgramHttpService.post( NgramHttpService.post(
{ {
'listId': listId 'listId': listId,
'ngramId': 'new'
}, },
{'annotation' : {'text': $scope.selection_text.trim()}} {
).$promise.then(function(data) { 'annotation' : {'text': $scope.selection_text.trim()}
}, function(data) {
$rootScope.annotations.push(data); $rootScope.annotations.push(data);
}); }, function(data) {
console.log(data);
console.error("unable to edit the Ngram " + $scope.selection_text);
}
);
} }
// hide selection highlighted text and the menu // hide the highlighted text the the menu
$(".text-panel").removeClass("selection"); $(".text-panel").removeClass("selection");
$element.fadeOut(100); $element.fadeOut(100);
}; };
} }
]); ]);
/*
* Text highlighting controller
*/
window.annotationsApp.controller('IntraTextController', window.annotationsApp.controller('IntraTextController',
['$scope', '$rootScope', '$compile', 'NgramHttpService', ['$scope', '$rootScope', '$compile', 'NgramHttpService',
function ($scope, $rootScope, $compile, NgramHttpService) { function ($scope, $rootScope, $compile, NgramHttpService) {
$scope.extra_stoplist = [];
$scope.extra_miamlist = [];
$scope.currentStopPage = 0;
$scope.currentMiamPage = 0;
$scope.pageSize = 15;
var counter = 0; var counter = 0;
/* /*
* Replace the text by and html template * Replace the text by an html template for ngram keywords
*/ */
function replaceTextByTemplate(text, annotation, template, pattern, lists) { function replaceTextByTemplate(text, ngram, template, pattern, lists) {
return text.replace(pattern, function(matched) { return text.replace(pattern, function(matched) {
var tpl = angular.element(template); var tpl = angular.element(template);
tpl.append(matched); tpl.append(matched);
tpl.attr('title', annotation.tooltip_content); tpl.attr('title', ngram.tooltip_content);
tpl.attr('uuid', annotation.uuid); tpl.attr('uuid', ngram.uuid);
/*
if ('MiamList' == lists[annotation.list_id]) tpl.addClass("miamword"); * Add CSS class depending on the list the ngram is into
if ('StopList' == lists[annotation.list_id]) tpl.addClass("stopword"); * FIXME Lists names and css classes are fixed, can do better
//if (annotation.category == 'stoplist' && annotation.level == 'global') tpl.addClass("global-stopword"); */
tpl.addClass(ngram.listName);
return tpl.get(0).outerHTML; return tpl.get(0).outerHTML;
}); });
} }
function compileText(annotations, fullText, abstractText, $rootScope) {
counter = 0;
var templateBegin = "<span ng-controller='AnnotationController' ng-click='onClick($event)' class='keyword-inline'>";
var templateBeginRegexp = "<span ng-controller='AnnotationController' ng-click='onClick\(\$event\)' class='keyword-inline'>";
var templateEnd = "</span>";
var template = templateBegin + templateEnd;
var startPattern = "\\b((?:"+templateBeginRegexp+")*";
var middlePattern = "(?:<\/span>)*\\s(?:"+templateBeginRegexp+")*";
var endPattern = "(?:<\/span>)*)\\b";
/* /*
* Sorts annotations on the number of words * Sorts annotations on the number of words
* Required for overlapping ngrams
*/ */
function lengthSort(listitems, valuekey) { function lengthSort(listitems, valuekey) {
listitems.sort(function(a, b) { listitems.sort(function(a, b) {
...@@ -262,86 +302,111 @@ ...@@ -262,86 +302,111 @@
}); });
return listitems; return listitems;
} }
/*
* Match and replace Ngram into the text
*/
function compileNgramsHtml(annotations, textMapping, $rootScope) {
// TODO remove this debug counter
counter = 0;
var templateBegin = "<span ng-controller='AnnotationController' ng-click='onClick($event)' class='keyword-inline'>";
var templateBeginRegexp = "<span ng-controller='AnnotationController' ng-click='onClick\(\$event\)' class='keyword-inline'>";
var sortedSizeAnnotations = lengthSort(annotations, "text"); var templateEnd = "</span>";
var extra_stoplist = [], var template = templateBegin + templateEnd;
extra_miamlist = [];
_.each(sortedSizeAnnotations, function (annotation) { var startPattern = "\\b((?:"+templateBeginRegexp+")*";
// TODO better split to manage two-words with minus sign var middlePattern = "(?:<\/span>)*\\s(?:"+templateBeginRegexp+")*";
annotation.category = $rootScope.lists[annotation.list_id].toLowerCase(); var endPattern = "(?:<\/span>)*)\\b";
var sortedSizeAnnotations = lengthSort(annotations, "text"),
extraNgramList = angular.copy($rootScope.extraNgramList);
// reinitialize an empty list
extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = [];
});
angular.forEach(sortedSizeAnnotations, function (annotation) {
// exclude ngrams that are into inactive lists
if ($rootScope.activeLists[annotation.list_id] === undefined) return;
// used to setup css class
annotation.listName = $rootScope.lists[annotation.list_id];
// regexps
var words = annotation.text.split(" "); var words = annotation.text.split(" ");
var pattern = new RegExp(startPattern + words.join(middlePattern) + endPattern, 'gmi'); var pattern = new RegExp(startPattern + words.join(middlePattern) + endPattern, 'gmi');
var textRegexp = new RegExp("\\b"+annotation.text+"\\b", 'igm'); var textRegexp = new RegExp("\\b"+annotation.text+"\\b", 'igm');
var isDisplayedIntraText = false;
if (pattern.test(fullText) === true) { // highlight text as html
fullText = replaceTextByTemplate(fullText, annotation, template, pattern, $rootScope.lists); angular.forEach(textMapping, function(text, eltId) {
if (pattern.test(text) === true) {
textMapping[eltId] = replaceTextByTemplate(text, annotation, template, pattern, $rootScope.lists);
// TODO remove debug // TODO remove debug
counter++; counter++;
} else if (pattern.test(abstractText) === true) { isDisplayedIntraText = true;
abstractText = replaceTextByTemplate(abstractText, annotation, template, pattern, $rootScope.lists); }
counter++; });
} else if (!textRegexp.test($rootScope.full_text) && !textRegexp.test($rootScope.abstract_text)) {
if (annotation.category == "stoplist") { if (!isDisplayedIntraText) {
// Deactivated stoplist for the moment // add extra-text ngrams that are not already displayed
// if ($.inArray(annotation.uuid, $scope.extra_stoplist.map(function (item) { if ($.inArray(annotation.uuid, extraNgramList[annotation.list_id].map(function (item) {
// return item.uuid;
// })) == -1) {
// extra_stoplist = lengthSort(extra_stoplist.concat(annotation), "text");
// }
} else if (annotation.category == "miamlist") {
if ($.inArray(annotation.uuid, $scope.extra_miamlist.map(function (item) {
return item.uuid; return item.uuid;
})) == -1) { })) == -1) {
extra_miamlist = lengthSort(extra_miamlist.concat(annotation), "text"); // push the ngram and sort
} extraNgramList[annotation.list_id] = extraNgramList[annotation.list_id].concat(annotation);
} }
} }
}); });
$scope.extra_stoplist = extra_stoplist; // update extraNgramList
$scope.extra_miamlist = extra_miamlist; $rootScope.extraNgramList = angular.forEach(extraNgramList, function(name, id) {
extraNgramList[id] = lengthSort(extraNgramList[id], 'text');
return { });
'fullTextHtml': fullText, // return the object of element ID with the corresponding HTML
'abstractTextHtml': abstractText return textMapping;
};
} }
/*
* Listen changes on the ngram data
*/
$rootScope.$watchCollection('annotations', function (newValue, oldValue) { $rootScope.$watchCollection('annotations', function (newValue, oldValue) {
if ($rootScope.annotations === undefined) return; if ($rootScope.annotations === undefined) return;
if (angular.equals(newValue, oldValue)) return; if (angular.equals(newValue, oldValue)) return;
$rootScope.miamListId = _.invert($rootScope.lists)['MiamList']; // initialize extraNgramList
$rootScope.stopListId = _.invert($rootScope.lists)['StopList']; var extraNgramList = {};
$rootScope.extraNgramList = angular.forEach($rootScope.activeLists, function(name, id) {
this[id] = [];
}, extraNgramList);
$rootScope.extraNgramList = extraNgramList;
$scope.extra_stoplist = []; /*
$scope.extra_miamlist = []; * Transform text into HTML with higlighted ngrams
*/
var result = compileText( var result = compileNgramsHtml(
$rootScope.annotations, $rootScope.annotations,
angular.copy($rootScope.full_text), {
angular.copy($rootScope.abstract_text), '#full-text': angular.copy($rootScope.full_text),
'#abstract-text': angular.copy($rootScope.abstract_text),
'#title': angular.copy($rootScope.title)
},
$rootScope $rootScope
); );
// inject highlighted HTML
$.each($rootScope.annotations, function(index, element) { angular.forEach(result, function(html, eltId) {
if (element.list_id == $rootScope.stopListId) { angular.element(eltId).html(html);
$scope.extra_stoplist.push(element);
} else if (element.list_id == $rootScope.miamListId) {
$scope.extra_miamlist.push(element);
}
}); });
// inject one Angular controller on every highlighted text element
angular.element('#full-text').html(result.fullTextHtml);
angular.element('#abstract-text').html(result.abstractTextHtml);
angular.element('.text-container').find('[ng-controller=AnnotationController]').each(function(idx, elt) { angular.element('.text-container').find('[ng-controller=AnnotationController]').each(function(idx, elt) {
angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true))); angular.element(elt).replaceWith($compile(elt)($rootScope.$new(true)));
}); });
}); });
function submitNewAnnotation($event, inputEltId, listId) { /*
* Add a new NGram from the user input in the extra-text list
*/
$scope.onListSubmit = function ($event, listId) {
var inputEltId = "#"+ listId +"-input";
if ($event.keyCode !== undefined && $event.keyCode != 13) return; if ($event.keyCode !== undefined && $event.keyCode != 13) return;
var value = $(inputEltId).val().trim(); var value = $(inputEltId).val().trim();
if (value === "") return; if (value === "") return;
...@@ -350,47 +415,54 @@ ...@@ -350,47 +415,54 @@
'listId': listId, 'listId': listId,
'ngramId': 'new' 'ngramId': 'new'
}, },
{'annotation' : {'text': value}}, {
'annotation' : {'text': value}
},
function(data) { function(data) {
// on success // on success
if (data) { if (data) {
$rootScope.annotations.push(data); $rootScope.annotations.push(data);
$(inputEltId).val("");
} }
}); }, function(data) {
// on error
$(inputEltId).parent().addClass("has-error");
console.error("error adding Ngram "+ value);
}
);
};
$(inputEltId).val("");
} }
]);
$scope.onMiamlistSubmit = function ($event) { /*
submitNewAnnotation($event, "#miamlist-input", _.invert($rootScope.lists)['MiamList']); * Controller for one List Tab displaying extra-text ngram
}; */
// TODO refactor window.annotationsApp.controller('ExtraTextPaginationController',
$scope.onStoplistSubmit = function ($event) { ['$scope', '$rootScope', function ($scope, $rootScope) {
submitNewAnnotation($event, "#stoplist-input", _.invert($rootScope.lists)['MiamList']);
}; $rootScope.$watchCollection('extraNgramList', function (newValue, oldValue) {
$scope.numStopPages = function () { $scope.currentListPage = 0;
if ($scope.extra_stoplist === undefined) return 0; $scope.pageSize = 15;
return Math.ceil($scope.extra_stoplist.length / $scope.pageSize);
}; $scope.nextListPage = function() {
$scope.numMiamPages = function () { $scope.currentListPage = $scope.currentListPage + 1;
if ($scope.extra_miamlist === undefined) return 0;
return Math.ceil($scope.extra_miamlist.length / $scope.pageSize);
};
$scope.nextMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage + 1;
};
$scope.previousMiamPage = function() {
$scope.currentMiamPage = $scope.currentMiamPage - 1;
}; };
$scope.nextStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage + 1; $scope.previousListPage = function() {
$scope.currentListPage = $scope.currentListPage - 1;
}; };
$scope.previousStopPage = function() {
$scope.currentStopPage = $scope.currentStopPage - 1; $scope.totalListPages = function (listId) {
if ($rootScope.extraNgramList[listId] === undefined) return 0;
return Math.ceil($rootScope.extraNgramList[listId].length / $scope.pageSize);
}; };
} });
]); }]);
/*
* Filter used in Ngram flat lists pagination (extra-text panel)
*/
window.annotationsApp.filter('startFrom', function () { window.annotationsApp.filter('startFrom', function () {
return function (input, start) { return function (input, start) {
if (input === undefined) return; if (input === undefined) return;
...@@ -405,28 +477,32 @@ ...@@ -405,28 +477,32 @@
$rootScope.documentResource = DocumentHttpService.get( $rootScope.documentResource = DocumentHttpService.get(
{'docId': $rootScope.docId}, {'docId': $rootScope.docId},
function(data, responseHeaders) { function(data, responseHeaders) {
$scope.title = data.title;
$scope.authors = data.authors; $scope.authors = data.authors;
$scope.journal = data.journal; $scope.journal = data.journal;
$scope.publication_date = data.publication_date; $scope.publication_date = data.publication_date;
// TODO this data have to be deleted
//$scope.current_page_number = data.current_page_number; //$scope.current_page_number = data.current_page_number;
//$scope.last_page_number = data.last_page_number; //$scope.last_page_number = data.last_page_number;
// put in rootScope because used by many components $rootScope.title = data.title;
$rootScope.docId = data.id; $rootScope.docId = data.id;
$rootScope.full_text = data.full_text; $rootScope.full_text = data.full_text;
$rootScope.abstract_text = data.abstract_text; $rootScope.abstract_text = data.abstract_text;
// GET the annotations // GET the annotationss
// TODO
$rootScope.annotationsResource = NgramListHttpService.get( $rootScope.annotationsResource = NgramListHttpService.get(
{'corpusId': $rootScope.corpusId, 'docId': $rootScope.docId} {
).$promise.then(function(data) { 'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()]; $rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.lists = data[$rootScope.corpusId.toString()]['lists']; $rootScope.lists = data[$rootScope.corpusId.toString()].lists;
}); // TODO active list selection controller
$rootScope.activeLists = angular.copy($rootScope.lists);
$rootScope.mainListId = _.invert($rootScope.activeLists).MiamList;
}
);
}); });
// TODO setup pagination client-side // TODO setup article pagination
$scope.onPreviousClick = function () { $scope.onPreviousClick = function () {
DocumentHttpService.get($scope.docId - 1); DocumentHttpService.get($scope.docId - 1);
}; };
...@@ -435,8 +511,11 @@ ...@@ -435,8 +511,11 @@
}; };
}]); }]);
/*
* Main function
* GET the document node and all its ngrams
*/
window.annotationsApp.run(function ($rootScope) { window.annotationsApp.run(function ($rootScope) {
/* GET the document node and all the annotations in the list associated */
var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//); var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//);
$rootScope.projectId = path[1]; $rootScope.projectId = path[1];
$rootScope.corpusId = path[2]; $rootScope.corpusId = path[2];
......
annotations/static/annotations/http.js
View file @ 0b4a7eae
...@@ -57,11 +57,11 @@ ...@@ -57,11 +57,11 @@
{ {
post: { post: {
method: 'POST', method: 'POST',
params: {'listId': '@listId', 'ngramId': ''} params: {'listId': '@listId', 'ngramId': '@ngramId'}
}, },
delete: { delete: {
method: 'DELETE', method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@id'} params: {'listId': '@listId', 'ngramId': '@ngramId'}
} }
} }
); );
......
annotations/static/annotations/keyword_tpl.html
View file @ 0b4a7eae
<span ng-if="keyword.category == 'miamlist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="miamlist">×</span> <span ng-click='onDeleteClick()' class="delete-keyword">×</span>
<span ng-if="keyword.category == 'miamlist'" data-toggle="tooltip" class="keyword miamword">{[{keyword.text}]}</span> <span data-toggle="tooltip" class="keyword-text {[{keyword.listName}]}">{[{keyword.text}]}</span>
<span ng-if="keyword.category == 'stoplist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="stoplist">×</span>
<span ng-if="keyword.category == 'stoplist'" data-toggle="tooltip" class="keyword stopword">{[{keyword.text}]}</span>
<span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span> <span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span>
annotations/static/annotations/selection_tpl.html deleted 100644 → 0
View file @ d8d27f26
<ul class="noselection">
<li class="miamword" ng-if="local_miamlist === true" ng-click="onClick($event, 'post', miamListId, 'local')">add to miam-list</li>
<li class="miamword" ng-if="local_miamlist === false" ng-click="onClick($event, 'delete', miamListId, 'local')">remove from miam-list</li>
<li class="stopword" ng-if="local_stoplist === true" ng-click="onClick($event, 'post', stopListId, 'local')">add to stop-list</li>
<li class="stopword" ng-if="local_stoplist === false" ng-click="onClick($event, 'delete', stopListId, 'local')">remove from stop-list</li>
<!--<li class="stopword" ng-if="global_stoplist === true" ng-click="onClick($event, 'post', 'stoplist', 'global')">add to global stop-list</li>
<li class="stopword" ng-if="global_stoplist === false" ng-click="onClick($event, 'delete', 'stoplist', 'global')">remove from global stop-list</li>-->
</ul>
annotations/templates/annotations/main.html
View file @ 0b4a7eae
...@@ -9,8 +9,8 @@ ...@@ -9,8 +9,8 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>Gargantext corpus annotations editor</title> <title>Gargantext article editor</title>
<meta name="description" content=""> <meta name="description" content="Gargantext">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}"> <link rel="stylesheet" href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}"> <link rel="stylesheet" href="{% static 'bower_components/angular/angular-csp.css' %}">
...@@ -18,51 +18,38 @@ ...@@ -18,51 +18,38 @@
<script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script> <script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script>
</head> </head>
<body> <body>
<!-- TODO integrate this later into the corpus.html django template --> <!-- TODO integrate this later into the any other django template -->
<div id="annotationsApp"> <div id="annotationsApp">
<div class="container-fluid"> <div class="container-fluid">
<div class="row-fluid main-panel" ng-controller="IntraTextController"> <div class="row-fluid main-panel" ng-controller="IntraTextController">
<div class="col-md-4 col-xs-4 tabbable words-panel"> <div class="col-md-4 col-xs-4 tabbable words-panel">
<ul class="nav nav-pills nav-justified"> <ul class="nav nav-pills nav-justified">
<li class="active"><a href="#tab1" data-toggle="tab">Miamwords</a></li> <li ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}">
<li><a href="#tab2" data-toggle="tab">Stopwords</a></li> <a href="#tab-{[{listId}]}" data-toggle="tab">{[{listName}]}</a>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="tab1">
<div ng-if="extra_miamlist.length == 0" class="alert alert-info" role="alert">No extra text miam-word yet</div>
<ul class="list-group words-list">
<li ng-repeat="keyword in extra_miamlist | startFrom:currentMiamPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li> </li>
</ul> </ul>
<nav ng-class="{invisible: numMiamPages() - 1 == 0}" class="clearfix"> <div class="tab-content">
<ul class="pagination pagination-s pull-right words-pagination"> <div ng-controller="ExtraTextPaginationController" ng-repeat="(listId, listName) in activeLists" ng-class="{active: $first == true}" class="tab-pane" id="tab-{[{listId}]}">
<li ng-class="{disabled: currentMiamPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li> <div ng-if="extraNgramList[listId].length == 0" class="alert alert-info" role="alert">
<li ng-class="{disabled: currentMiamPage >= numMiamPages()-1}"><a ng-click="nextMiamPage()" class="glyphicon glyphicon-forward"></a></li> Input any keyword you want to link to this article and the list named '{[{listName}]}'
</ul>
</nav>
<div class="form-group">
<input type="text" class="form-control" id="miamlist-input" ng-keypress="onMiamlistSubmit($event)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onMiamlistSubmit($event)">Add</button>
</div>
</div> </div>
<div class="tab-pane" id="tab2">
<div ng-if="extra_stoplist.length == 0" class="alert alert-info" role="alert">No extra text stop-word yet</div> <ul class="list-group words-list clearfix">
<ul class="list-group words-list"> <li ng-repeat="keyword in extraNgramList[listId] | startFrom:currentListPage * pageSize | limitTo:pageSize" class="list-group-item">
<li ng-repeat="keyword in extra_stoplist | startFrom:currentStopPage*pageSize | limitTo:pageSize" class="list-group-item">
<div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div> <div ng-controller="ExtraAnnotationController" keyword-template class="keyword-container"></div>
</li> </li>
</ul> </ul>
<nav ng-class="{invisible: numStopPages() - 1 == 0}" class="clearfix">
<nav ng-class="{invisible: totalListPages - 1 == 0}" class="clearfix">
<ul class="pagination pagination-s pull-right words-pagination"> <ul class="pagination pagination-s pull-right words-pagination">
<li ng-class="{disabled: currentStopPage == 0}"><a ng-click="previousMiamPage()" class="glyphicon glyphicon-backward"></a></li> <li ng-class="{disabled: currentListPage == 0}"><a ng-click="previousListPage()" class="glyphicon glyphicon-backward"></a></li>
<li ng-class="{disabled: currentStopPage >= numStopPages()-1}"><a ng-click="nextStopPage()" class="glyphicon glyphicon-forward"></a></li> <li ng-class="{disabled: currentListPage >= totalListPages - 1}"><a ng-click="nextListPage()" class="glyphicon glyphicon-forward"></a></li>
</ul> </ul>
</nav> </nav>
<div class="form-group"> <div class="form-group">
<input type="text" class="form-control" id="stoplist-input" ng-keypress="onStoplistSubmit($event)"> <input type="text" class="form-control" id="{[{ listId }]}-input" ng-keypress="onListSubmit($event, listId)">
<button type="submit" class="btn btn-default btn-primary" ng-click="onStoplistSubmit($event)">Add</button> <button type="submit" class="form-control btn btn-default btn-primary" ng-click="onListSubmit($event, listId)">Add to {[{listName}]}</button>
</div> </div>
</div> </div>
</div> </div>
...@@ -70,7 +57,7 @@ ...@@ -70,7 +57,7 @@
<div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document"> <div class="col-md-8 col-xs-8 text-panel" ng-controller="DocController" id="document">
<div class="row-fluid clearfix"> <div class="row-fluid clearfix">
<div class="col-md-7 col-xs-7"> <div class="col-md-7 col-xs-7">
<h3>{[{title}]}</h3> <h3 class="text-container" id="title">{[{title}]}</h3>
</div> </div>
<div class="col-md-5 col-xs-5"> <div class="col-md-5 col-xs-5">
<nav> <nav>
...@@ -90,17 +77,21 @@ ...@@ -90,17 +77,21 @@
</div> </div>
<h4 ng-if="abstract_text != null">Abstract</h4> <h4 ng-if="abstract_text != null">Abstract</h4>
<p id="abstract-text" class="text-container"> <p id="abstract-text" class="text-container">
<div ng-if="abstract_text == null" class="alert alert-info" role="alert">No abstract text</div> <div ng-if="abstract_text == null" class="alert alert-info" role="alert">Empty abstract text</div>
</p> </p>
<h4 ng-if="full_text != null">Full Article</h4> <h4 ng-if="full_text != null">Full Article</h4>
<p id="full-text" class="text-container"> <p id="full-text" class="text-container">
<div ng-if="full_text == null" class="alert alert-info" role="alert">No full text</div> <div ng-if="full_text == null" class="alert alert-info" role="alert">Empty full text</div>
</p> </p>
</div> </div>
</div> <!-- end of the main row --> </div> <!-- end of the main row -->
</div> </div>
<!-- this menu is over the text --> <!-- this menu is over the text on mouse selection -->
<div ng-controller="AnnotationMenuController" id="selection" class="selection-menu" selection-template></div> <div ng-controller="AnnotationMenuController" id="selection" class="selection-menu">
<ul class="noselection">
<li ng-repeat="item in menuItems" class="{[{item.listName}]}" ng-click="onMenuClick($event, item.action, item.listId)">{[{item.verb}]} {[{item.listName}]}</li>
</ul>
</div>
</div> </div>
<!--[if lt IE 7]> <!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p> <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
......
annotations/views.py
View file @ 0b4a7eae
...@@ -40,15 +40,16 @@ class NgramList(APIView): ...@@ -40,15 +40,16 @@ class NgramList(APIView):
"""Get All for a doc id""" """Get All for a doc id"""
corpus_id = int(corpus_id) corpus_id = int(corpus_id)
doc_id = int(doc_id) doc_id = int(doc_id)
lists = dict() lists = {}
for list_type in ['MiamList', 'StopList']: for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type) list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type lists["%s" % list_id[0][0]] = list_type
# ngrams of list_id of corpus_id: # ngrams of list_id of corpus_id:
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id) doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id)
#doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)] doc_ngram_list = [(i, 'miam', i, 1931) for i in range(500)]
doc_ngram_list += [(i, 'stop', i, 1932) for i in range(501, 600)]
# doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
data = { '%s' % corpus_id : { data = { '%s' % corpus_id : {
'%s' % doc_id : [ '%s' % doc_id : [
......
gargantext_web/api.py
View file @ 0b4a7eae
...@@ -7,17 +7,29 @@ from sqlalchemy import text, distinct, or_ ...@@ -7,17 +7,29 @@ from sqlalchemy import text, distinct, or_
from sqlalchemy.sql import func from sqlalchemy.sql import func
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
import datetime
import copy
from gargantext_web.views import move_to_trash from gargantext_web.views import move_to_trash
from gargantext_web.db import * from gargantext_web.db import *
from gargantext_web.validation import validate, ValidationException
from node import models from node import models
def DebugHttpResponse(data): def DebugHttpResponse(data):
return HttpResponse('<html><body style="background:#000;color:#FFF"><pre>%s</pre></body></html>' % (str(data), )) return HttpResponse('<html><body style="background:#000;color:#FFF"><pre>%s</pre></body></html>' % (str(data), ))
import json import json
class JSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return obj.isoformat()[:19] + 'Z'
else:
return super(self.__class__, self).default(obj)
json_encoder = JSONEncoder(indent=4)
def JsonHttpResponse(data, status=200): def JsonHttpResponse(data, status=200):
return HttpResponse( return HttpResponse(
content = json.dumps(data, indent=4), content = json_encoder.encode(data),
content_type = 'application/json; charset=utf-8', content_type = 'application/json; charset=utf-8',
status = status status = status
) )
...@@ -54,7 +66,7 @@ class APIException(_APIException): ...@@ -54,7 +66,7 @@ class APIException(_APIException):
self.detail = message self.detail = message
_operators = { _operators_dict = {
"=": lambda field, value: (field == value), "=": lambda field, value: (field == value),
"!=": lambda field, value: (field != value), "!=": lambda field, value: (field != value),
"<": lambda field, value: (field < value), "<": lambda field, value: (field < value),
...@@ -65,6 +77,14 @@ _operators = { ...@@ -65,6 +77,14 @@ _operators = {
"contains": lambda field, value: (field.contains(value)), "contains": lambda field, value: (field.contains(value)),
"startswith": lambda field, value: (field.startswith(value)), "startswith": lambda field, value: (field.startswith(value)),
} }
_hyperdata_list = [
hyperdata
for hyperdata in session.query(Hyperdata).order_by(Hyperdata.name)
]
_hyperdata_dict = {
hyperdata.name: hyperdata
for hyperdata in _hyperdata_list
}
from rest_framework.decorators import api_view from rest_framework.decorators import api_view
...@@ -75,6 +95,7 @@ def Root(request, format=None): ...@@ -75,6 +95,7 @@ def Root(request, format=None):
'snippets': reverse('snippet-list', request=request, format=format) 'snippets': reverse('snippet-list', request=request, format=format)
}) })
class NodesChildrenNgrams(APIView): class NodesChildrenNgrams(APIView):
def get(self, request, node_id): def get(self, request, node_id):
...@@ -117,6 +138,7 @@ class NodesChildrenNgrams(APIView): ...@@ -117,6 +138,7 @@ class NodesChildrenNgrams(APIView):
], ],
}) })
class NodesChildrenDuplicates(APIView): class NodesChildrenDuplicates(APIView):
def _fetch_duplicates(self, request, node_id, extra_columns=None, min_count=1): def _fetch_duplicates(self, request, node_id, extra_columns=None, min_count=1):
...@@ -205,6 +227,7 @@ class NodesChildrenDuplicates(APIView): ...@@ -205,6 +227,7 @@ class NodesChildrenDuplicates(APIView):
'deleted': count 'deleted': count
}) })
class NodesChildrenMetatadata(APIView): class NodesChildrenMetatadata(APIView):
def get(self, request, node_id): def get(self, request, node_id):
...@@ -264,49 +287,98 @@ class NodesChildrenMetatadata(APIView): ...@@ -264,49 +287,98 @@ class NodesChildrenMetatadata(APIView):
'data': collection, 'data': collection,
}) })
class NodesChildrenQueries(APIView): class NodesChildrenQueries(APIView):
def _parse_filter(self, filter): def _sql(self, input, node_id):
fields = dict()
# validate filter keys tables = set('nodes')
filter_keys = {'field', 'operator', 'value'} hyperdata_aliases = dict()
if set(filter) != filter_keys: # retrieve all unique fields names
raise APIException('Every filter should have exactly %d keys: "%s"'% (len(filter_keys), '", "'.join(filter_keys)), 400) fields_names = input['retrieve']['fields'].copy()
field, operator, value = filter['field'], filter['operator'], filter['value'] fields_names += [filter['field'] for filter in input['filters']]
fields_names += input['sort']
# validate operator fields_names = set(fields_names)
if operator not in _operators: # relate fields to their respective ORM counterparts
raise APIException('Invalid operator: "%s"'% (operator, ), 400) for field_name in fields_names:
field_name_parts = field_name.split('.')
# validate value, depending on the operator field = None
if operator == 'in': if len(field_name_parts) == 1:
if not isinstance(value, list): field = getattr(Node, field_name)
raise APIException('Parameter "value" should be an array when using operator "%s"'% (operator, ), 400) elif field_name_parts[0] == 'ngrams':
for v in value: field = getattr(Ngram, field_name)
if not isinstance(v, (int, float, str)): tables.add('ngrams')
raise APIException('Parameter "value" should be an array of numbers or strings when using operator "%s"'% (operator, ), 400) elif field_name_parts[1] == 'count':
if field_name_parts[0] == 'nodes':
field = func.count(Node.id)
elif field_name_parts[0] == 'ngrams':
field = func.count(Ngram.id)
tables.add('ngrams')
elif field_name_parts[0] == 'hyperdata':
hyperdata = _hyperdata_dict[field_name_parts[1]]
if hyperdata not in hyperdata_aliases:
hyperdata_aliases[hyperdata] = aliased(Node_Hyperdata)
hyperdata_alias = hyperdata_aliases[hyperdata]
field = getattr(hyperdata_alias, 'value_%s' % hyperdata.type)
if len(field_name_parts) == 3:
field = func.date_trunc(field_name_parts[2], field)
fields[field_name] = field
# build query: selected fields
query = (session
.query(*(fields[field_name] for field_name in input['retrieve']['fields']))
)
# build query: selected tables
query = query.select_from(Node)
if 'ngrams' in tables:
query = (query
.join(Node_Ngram, Node_Ngram.node_id == Node.id)
.join(Ngram, Ngram.id == Node_Ngram.ngram_id)
)
for hyperdata, hyperdata_alias in hyperdata_aliases.items():
query = (query
.join(hyperdata_alias, hyperdata_alias.node_id == Node.id)
.filter(hyperdata_alias.hyperdata_id == hyperdata.id)
)
# build query: filtering
query = (query
.filter(Node.parent_id == node_id)
)
for filter in input['filters']:
query = (query
.filter(_operators_dict[filter['operator']](
fields[filter['field']],
filter['value']
))
)
# build query: aggregations
if input['retrieve']['aggregate']:
for field_name in input['retrieve']['fields']:
if not field_name.endswith('.count'):
query = query.group_by(fields[field_name])
# build query: sorting
for field_name in input['sort']:
last = field_name[-1:]
if last in ('+', '-'):
field_name = field_name[:-1]
if last == '-':
query = query.order_by(fields[field_name].desc())
else: else:
if not isinstance(value, (int, float, str)): query = query.order_by(fields[field_name])
raise APIException('Parameter "value" should be a number or string when using operator "%s"'% (operator, ), 400) # build and return result
output = copy.deepcopy(input)
output['pagination']['total'] = query.count()
output['results'] = list(
query[input['pagination']['offset']:input['pagination']['offset']+input['pagination']['limit']]
if input['pagination']['limit']
else query[input['pagination']['offset']:]
)
return output
# parse field def _haskell(self, input, node_id):
field_objects = { output = copy.deepcopy(input)
'hyperdata': None, output['pagination']['total'] = 0
'ngrams': ['terms', 'n'], output['results'] = list()
} return output
field = field.split('.')
if len(field) < 2 or field[0] not in field_objects:
raise APIException('Parameter "field" should be a in the form "object.key", where "object" takes one of the following values: "%s". "%s" was found instead' % ('", "'.join(field_objects), '.'.join(field)), 400)
if field_objects[field[0]] is not None and field[1] not in field_objects[field[0]]:
raise APIException('Invalid key for "%s" in parameter "field", should be one of the following values: "%s". "%s" was found instead' % (field[0], '", "'.join(field_objects[field[0]]), field[1]), 400)
# return value
return field, _operators[operator], value
def _count_documents(self, query):
return {
'fields': []
}
def post(self, request, node_id): def post(self, request, node_id):
""" Query the children of the given node. """ Query the children of the given node.
...@@ -348,199 +420,53 @@ class NodesChildrenQueries(APIView): ...@@ -348,199 +420,53 @@ class NodesChildrenQueries(APIView):
} }
""" """
hyperdata_aliases = {} # authorized field names
sql_fields = set({
# validate query 'id', 'name',
query_fields = {'pagination', 'retrieve', 'sort', 'filters'} 'nodes.count', 'ngrams.count',
for key in request.DATA: 'ngrams.terms', 'ngrams.n',
if key not in query_fields: })
raise APIException('Unrecognized field "%s" in query object. Accepted fields are: "%s"' % (key, '", "'.join(query_fields)), 400) for hyperdata in _hyperdata_list:
sql_fields.add('hyperdata.' + hyperdata.name)
# selecting info
if 'retrieve' not in request.DATA:
raise APIException('The query should have a "retrieve" parameter.', 400)
retrieve = request.DATA['retrieve']
retrieve_types = {'fields', 'aggregates'}
if 'type' not in retrieve:
raise APIException('In the query\'s "retrieve" parameter, a "type" should be specified. Possible values are: "%s".' % ('", "'.join(retrieve_types), ), 400)
if 'list' not in retrieve or not isinstance(retrieve['list'], list):
raise APIException('In the query\'s "retrieve" parameter, a "list" should be provided as an array', 400)
if retrieve['type'] not in retrieve_types:
raise APIException('Unrecognized "type": "%s" in the query\'s "retrieve" parameter. Possible values are: "%s".' % (retrieve['type'], '", "'.join(retrieve_types), ), 400)
if retrieve['type'] == 'fields':
fields_names = ['id'] + retrieve['list'] if 'id' not in retrieve['list'] else retrieve['list']
elif retrieve['type'] == 'aggregates':
fields_names = list(retrieve['list'])
fields_list = []
for field_name in fields_names:
split_field_name = field_name.split('.')
if split_field_name[0] == 'hyperdata':
hyperdata = session.query(Hyperdata).filter(Hyperdata.name == split_field_name[1]).first()
if hyperdata is None:
hyperdata_query = session.query(Hyperdata.name).order_by(Hyperdata.name)
hyperdata_names = [hyperdata.name for hyperdata in hyperdata_query.all()]
raise APIException('Invalid key for "%s" in parameter "field", should be one of the following values: "%s". "%s" was found instead' % (field[0], '", "'.join(hyperdata_names), field[1]), 400)
# check or create Node_Hyperdata alias; join if necessary
if hyperdata.id in hyperdata_aliases:
hyperdata_alias = hyperdata_aliases[hyperdata.id]
else:
hyperdata_alias = hyperdata_aliases[hyperdata.id] = aliased(Node_Hyperdata)
field = getattr(hyperdata_alias, 'value_' + hyperdata.type)
# operation on field
if len(split_field_name) > 2:
# datetime truncation
if hyperdata.type == 'datetime': if hyperdata.type == 'datetime':
datepart = split_field_name[2] for part in ['year', 'month', 'day', 'hour', 'minute']:
accepted_dateparts = ['year', 'month', 'day', 'hour', 'minute'] sql_fields.add('hyperdata.' + hyperdata.name + '.' + part)
if datepart not in accepted_dateparts:
raise APIException('Invalid date truncation for "%s": "%s". Accepted values are: "%s".' % (split_field_name[1], split_field_name[2], '", "'.join(accepted_dateparts), ), 400)
# field = extract(datepart, field)
field = func.date_trunc(datepart, field)
# field = func.date_trunc(text('"%s"'% (datepart,)), field)
else:
authorized_field_names = {'id', 'name', }
authorized_aggregates = {
'nodes.count': func.count(Node.id),
'ngrams.count': func.count(Ngram.id),
}
if retrieve['type'] == 'aggregates' and field_name in authorized_aggregates:
field = authorized_aggregates[field_name]
elif field_name in authorized_field_names:
field = getattr(Node, field_name)
else:
raise APIException('Unrecognized "field": "%s" in the query\'s "retrieve" parameter. Possible values are: "%s".' % (field_name, '", "'.join(authorized_field_names), ))
fields_list.append(
field.label(
field_name if '.' in field_name else 'node.' + field_name
)
)
# starting the query! # authorized field names: Haskell
document_type_id = cache.NodeType['Document'].id ##session.query(NodeType.id).filter(NodeType.name == 'Document').scalar() haskell_fields = set({
query = (session 'haskell.test',
.query(*fields_list) })
.select_from(Node)
.filter(Node.type_id == document_type_id)
.filter(Node.parent_id == node_id)
)
# join ngrams if necessary
if 'ngrams.count' in fields_names:
query = (query
.join(Node_Ngram, Node_Ngram.node_id == Node.id)
.join(Ngram, Ngram.id == Node_Ngram.ngram_id)
)
# join hyperdata aliases # authorized field names: all of them
for hyperdata_id, hyperdata_alias in hyperdata_aliases.items(): authorized_fields = sql_fields | haskell_fields
query = (query
.join(hyperdata_alias, hyperdata_alias.node_id == Node.id)
.filter(hyperdata_alias.hyperdata_id == hyperdata_id)
)
# filtering # input validation
for filter in request.DATA.get('filters', []): input = validate(request.DATA, {'type': dict, 'items': {
# parameters extraction & validation 'pagination': {'type': dict, 'items': {
field, operator, value = self._parse_filter(filter) 'limit': {'type': int, 'default': 0},
# 'offset': {'type': int, 'default': 0},
if field[0] == 'hyperdata': }, 'default': {'limit': 0, 'offset': 0}},
# which hyperdata? 'filters': {'type': list, 'items': {'type': dict, 'items': {
hyperdata = session.query(Hyperdata).filter(Hyperdata.name == field[1]).first() 'field': {'type': str, 'required': True, 'range': authorized_fields},
if hyperdata is None: 'operator': {'type': str, 'required': True, 'range': list(_operators_dict.keys())},
hyperdata_query = session.query(Hyperdata.name).order_by(Hyperdata.name) 'value': {'required': True},
hyperdata_names = [hyperdata.name for hyperdata in hyperdata_query.all()] }}, 'default': list()},
raise APIException('Invalid key for "%s" in parameter "field", should be one of the following values: "%s". "%s" was found instead' % (field[0], '", "'.join(hyperdata_names), field[1]), 400) 'retrieve': {'type': dict, 'required': True, 'items': {
# check or create Node_Hyperdata alias; join if necessary 'aggregate': {'type': bool, 'default': False},
if hyperdata.id in hyperdata_aliases: 'fields': {'type': list, 'items': {'type': str, 'range': authorized_fields}, 'range': (1, )},
hyperdata_alias = hyperdata_aliases[hyperdata.id] }},
'sort': {'type': list, 'items': {'type': str}, 'default': list()},
}})
# return result, depending on the queried fields
if set(input['retrieve']['fields']) <= sql_fields:
method = self._sql
elif set(input['retrieve']['fields']) <= haskell_fields:
method = self._haskell
else: else:
hyperdata_alias = hyperdata_aliases[hyperdata.id] = aliased(Node_Hyperdata) raise ValidationException('queried fields are mixing incompatible types of fields')
query = (query return JsonHttpResponse(method(input, node_id), 201)
.join(hyperdata_alias, hyperdata_alias.node_id == Node.id)
.filter(hyperdata_alias.hyperdata_id == hyperdata.id)
)
# adjust date
if hyperdata.type == 'datetime':
value = value + '2000-01-01T00:00:00Z'[len(value):]
# filter query
query = query.filter(operator(
getattr(hyperdata_alias, 'value_' + hyperdata.type),
value
))
elif field[0] == 'ngrams':
query = query.filter(
Node.id.in_(session
.query(Node_Ngram.node_id)
.join(Ngram, Ngram.id == Node_Ngram.ngram_id)
.filter(operator(
getattr(Ngram, field[1]),
map(lambda x: x.replace('-', ' '), value)
))
)
)
# TODO: date_trunc (psql) -> index also
# groupping
for field_name in fields_names:
if field_name not in authorized_aggregates:
# query = query.group_by(text(field_name))
query = query.group_by('"%s"' % (
field_name if '.' in field_name else 'node.' + field_name
, ))
# sorting
sort_fields_names = request.DATA.get('sort', ['id'])
if not isinstance(sort_fields_names, list):
raise APIException('The query\'s "sort" parameter should be an array', 400)
sort_fields_list = []
for sort_field_name in sort_fields_names:
try:
desc = sort_field_name[0] == '-'
if sort_field_name[0] in {'-', '+'}:
sort_field_name = sort_field_name[1:]
field = fields_list[fields_names.index(sort_field_name)]
if desc:
field = field.desc()
sort_fields_list.append(field)
except:
raise APIException('Unrecognized field "%s" in the query\'s "sort" parameter. Accepted values are: "%s"' % (sort_field_name, '", "'.join(fields_names)), 400)
query = query.order_by(*sort_fields_list)
# pagination
pagination = request.DATA.get('pagination', {})
for key, value in pagination.items():
if key not in {'limit', 'offset'}:
raise APIException('Unrecognized parameter in "pagination": "%s"' % (key, ), 400)
if not isinstance(value, int):
raise APIException('In "pagination", "%s" should be an integer.' % (key, ), 400)
if 'offset' not in pagination:
pagination['offset'] = 0
if 'limit' not in pagination:
pagination['limit'] = 0
# respond to client!
# return DebugHttpResponse(str(query))
# return DebugHttpResponse(literalquery(query))
results = [
list(row)
# dict(zip(fields_names, row))
for row in (
query[pagination["offset"]:pagination["offset"]+pagination["limit"]]
if pagination['limit']
else query[pagination["offset"]:]
)
]
pagination["total"] = query.count()
return Response({
"pagination": pagination,
"retrieve": fields_names,
"sorted": sort_fields_names,
"results": results,
}, 201)
class NodesList(APIView): class NodesList(APIView):
authentication_classes = (SessionAuthentication, BasicAuthentication) authentication_classes = (SessionAuthentication, BasicAuthentication)
...@@ -598,6 +524,7 @@ class Nodes(APIView): ...@@ -598,6 +524,7 @@ class Nodes(APIView):
except Exception as error: except Exception as error:
msgres ="error deleting : " + node_id + str(error) msgres ="error deleting : " + node_id + str(error)
class CorpusController: class CorpusController:
@classmethod @classmethod
......
gargantext_web/validation.py 0 → 100644
View file @ 0b4a7eae
from rest_framework.exceptions import APIException
from datetime import datetime
__all__ = ['validate']
_types_names = {
bool: 'boolean',
int: 'integer',
float: 'float',
str: 'string',
dict: 'object',
list: 'array',
datetime: 'datetime',
}
class ValidationException(APIException):
status_code = 400
default_detail = 'Bad request!'
def validate(value, expected, path='input'):
# Is the expected type respected?
if 'type' in expected:
expected_type = expected['type']
if not isinstance(value, expected_type):
if expected_type in (bool, int, float, str, datetime, ):
try:
if expected_type == bool:
value = value not in {0, 0.0, '', '0', 'false'}
elif expected_type == datetime:
value = value + '2000-01-01T00:00:00Z'[len(value):]
value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
else:
value = expected_type(value)
except ValueError:
raise ValidationException('%s should be a JSON %s, but could not be parsed as such' % (path, _types_names[expected_type], ))
else:
raise ValidationException('%s should be a JSON %s' % (path, _types_names[expected_type], ))
else:
expected_type = type(value)
# Is the value in the expected range?
if 'range' in expected:
expected_range = expected['range']
if isinstance(expected_range, tuple):
if expected_type in (int, float):
tested_value = value
tested_name = 'value'
elif expected_type in (str, list):
tested_value = len(value)
tested_name = 'length'
if tested_value < expected_range[0]:
raise ValidationException('%s should have a minimum %s of %d' % (path, tested_name, expected_range[0], ))
if len(expected_range) > 1 and tested_value > expected_range[1]:
raise ValidationException('%s should have a maximum %s of %d' % (path, tested_name, expected_range[1], ))
elif isinstance(expected_range, (list, set, dict, )) and value not in expected_range:
expected_values = expected_range if isinstance(expected_range, list) else expected_range.keys()
expected_values = [str(value) for value in expected_values if isinstance(value, expected_type)]
if len(expected_values) < 16:
expected_values_str = '", "'.join(expected_values)
expected_values_str = '"' + expected_values_str + '"'
else:
expected_values_str = '", "'.join(expected_values[:16])
expected_values_str = '"' + expected_values_str + '"...'
raise ValidationException('%s should take one of the following values: %s' % (path, expected_values_str, ))
# Do we have to translate through a dictionary?
if 'translate' in expected:
translate = expected['translate']
if callable(translate):
value = translate(value)
if value is None and expected.get('required', False):
raise ValidationException('%s has been given an invalid value' % (path, ))
return value
try:
value = expected['translate'][value]
except KeyError:
if expected.get('translate_fallback_keep', False):
return value
if expected.get('required', False):
raise ValidationException('%s has been given an invalid value' % (path, ))
else:
return expected.get('default', value)
# Are we handling an iterable?
if expected_type in (list, dict):
if 'items' in expected:
expected_items = expected['items']
if expected_type == list:
for i, element in enumerate(value):
value[i] = validate(element, expected_items, '%s[%d]' % (path, i, ))
elif expected_type == dict:
if expected_items:
for key in value:
if key not in expected_items:
raise ValidationException('%s should not have a "%s" key.' % (path, key, ))
for expected_key, expected_value in expected_items.items():
if expected_key in value:
value[expected_key] = validate(value[expected_key], expected_value, '%s["%s"]' % (path, expected_key, ))
elif 'required' in expected_value and expected_value['required']:
raise ValidationException('%s should have a "%s" key.' % (path, expected_key, ))
elif 'default' in expected_value:
value[expected_key] = expected_value['default']
# Let's return the proper value!
return value
static/js/gargantext.angular.js
View file @ 0b4a7eae
...@@ -259,11 +259,21 @@ gargantext.controller("DatasetController", function($scope, $http) { ...@@ -259,11 +259,21 @@ gargantext.controller("DatasetController", function($scope, $http) {
$scope.corpora = []; $scope.corpora = [];
$http.get('/api/nodes?type=Project', {cache: true}).success(function(response){ $http.get('/api/nodes?type=Project', {cache: true}).success(function(response){
$scope.projects = response.data; $scope.projects = response.data;
// Initially set to what is indicated in the URL
if (/^\/project\/\d+\/corpus\/\d+/.test(location.pathname)) {
$scope.projectId = parseInt(location.pathname.split('/')[2]);
$scope.updateCorpora();
}
}); });
// update corpora according to the select parent project // update corpora according to the select parent project
$scope.updateCorpora = function() { $scope.updateCorpora = function() {
$http.get('/api/nodes?type=Corpus&parent=' + $scope.projectId, {cache: true}).success(function(response){ $http.get('/api/nodes?type=Corpus&parent=' + $scope.projectId, {cache: true}).success(function(response){
$scope.corpora = response.data; $scope.corpora = response.data;
// Initially set to what is indicated in the URL
if (/^\/project\/\d+\/corpus\/\d+/.test(location.pathname)) {
$scope.corpusId = parseInt(location.pathname.split('/')[4]);
$scope.updateEntities();
}
}); });
}; };
// update entities depending on the selected corpus // update entities depending on the selected corpus
...@@ -522,8 +532,8 @@ gargantext.controller("GraphController", function($scope, $http, $element) { ...@@ -522,8 +532,8 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
filters: query.filters, filters: query.filters,
sort: ['hyperdata.publication_date.day'], sort: ['hyperdata.publication_date.day'],
retrieve: { retrieve: {
type: 'aggregates', aggregate: true,
list: ['hyperdata.publication_date.day', query.mesured] fields: ['hyperdata.publication_date.day', query.mesured]
} }
}; };
// request to the server // request to the server
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment