Skip to content

gargantext
gargantext
Commit 6bb38b1c authored by Administrator's avatar Administrator
Browse files
Options

Merge branch 'mat-settings' into unstable

parents 21fec896 144ed412
Hide whitespace changes
Inline Side-by-side
Showing with 156 additions and 43 deletions
gargantext_web/api.py
View file @ 6bb38b1c
......@@ -136,9 +136,8 @@ _ngrams_order_columns = {
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.exceptions import APIException
from rest_framework.exceptions import APIException as _APIException
_APIException = APIException
class APIException(_APIException):
def __init__(self, message, code=500):
self.status_code = code
......@@ -167,6 +166,91 @@ def Root(request, format=None):
})
class NodesChildrenDuplicates(APIView):
def _fetch_duplicates(self, request, node_id, extra_columns=[], min_count=1):
# input validation
if 'keys' not in request.GET:
raise APIException('Missing GET parameter: "keys"', 400)
keys = request.GET['keys'].split(',')
# metadata retrieval
metadata_query = (Metadata
.query(Metadata)
.filter(Metadata.name.in_(keys))
)
# build query elements
columns = []
aliases = []
for metadata in metadata_query:
# aliases
_Metadata = aliased(Metadata)
_Node_Metadata = aliased(Node_Metadata)
aliases.append(_Node_Metadata)
# what shall we retrieve?
columns.append(
getattr(_Node_Metadata, 'value_' + metadata.type)
)
# build the query
groups = list(columns)
duplicates_query = (get_session()
.query(*(extra_columns + [func.count()] + columns))
.select_from(Node)
)
for _Node_Metadata, metadata in zip(aliases, metadata_query):
duplicates_query = duplicates_query.outerjoin(_Node_Metadata, _Node_Metadata.node_id == Node.id)
duplicates_query = duplicates_query.filter(_Node_Metadata.metadata_id == metadata.id)
duplicates_query = duplicates_query.filter(Node.parent_id == node_id)
duplicates_query = duplicates_query.group_by(*columns)
duplicates_query = duplicates_query.order_by(func.count().desc())
duplicates_query = duplicates_query.having(func.count() > min_count)
# and now, return it
return duplicates_query
def get(self, request, node_id):
# data to be returned
duplicates = self._fetch_duplicates(request, node_id)
# pagination
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 10))
total = duplicates.count()
# response building
return JsonHttpResponse({
'pagination': {
'offset': offset,
'limit': limit,
'total': total,
},
'data': [
{
'count': duplicate[0],
'values': duplicate[1:],
}
for duplicate in duplicates[offset : offset+limit]
]
})
def delete(self, request, node_id):
session = get_session()
# get the minimum ID for each of the nodes sharing the same metadata
kept_node_ids_query = self._fetch_duplicates(request, node_id, [func.min(Node.id).label('id')], 0)
kept_node_ids = [kept_node.id for kept_node in kept_node_ids_query]
# delete the stuff
delete_query = (session
.query(Node)
.filter(Node.parent_id == node_id)
.filter(~Node.id.in_(kept_node_ids))
)
count = delete_query.count()
delete_query.delete(synchronize_session=False)
session.flush()
# return the result
return JsonHttpResponse({
'deleted': count,
})
# return duplicates_query
class NodesChildrenMetatadata(APIView):
def get(self, request, node_id):
......
gargantext_web/settings.py
View file @ 6bb38b1c
......@@ -62,7 +62,7 @@ INSTALLED_APPS = (
'django.contrib.messages',
'django.contrib.staticfiles',
'django_extensions',
#'south',
'south',
'cte_tree',
'node',
'ngram',
......
gargantext_web/urls.py
View file @ 6bb38b1c
......@@ -51,6 +51,7 @@ urlpatterns = patterns('',
url(r'^api$', gargantext_web.api.Root),
url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
......
node/models.py
View file @ 6bb38b1c
......@@ -245,7 +245,7 @@ class Node(CTENode):
print("In workflow() END")
class Node_Metadata(models.Model):
node = models.ForeignKey(Node)
node = models.ForeignKey(Node, on_delete=models.CASCADE)
metadata = models.ForeignKey(Metadata)
value_int = models.IntegerField(null=True, db_index=True)
value_float = models.FloatField(null=True, db_index=True)
......@@ -254,12 +254,12 @@ class Node_Metadata(models.Model):
value_text = models.TextField(null=True)
class Node_Resource(models.Model):
node = models.ForeignKey(Node, related_name='node_resource')
node = models.ForeignKey(Node, related_name='node_resource', on_delete=models.CASCADE)
resource = models.ForeignKey(Resource)
parsed = models.BooleanField(default=False)
class Node_Ngram(models.Model):
node = models.ForeignKey(Node)
node = models.ForeignKey(Node, on_delete=models.CASCADE)
ngram = models.ForeignKey(Ngram)
weight = models.FloatField()
def __str__(self):
......@@ -291,7 +291,7 @@ class Document(Node):
proxy=True
class NodeNgramNgram(models.Model):
node = models.ForeignKey(Node)
node = models.ForeignKey(Node, on_delete=models.CASCADE)
ngramx = models.ForeignKey(Ngram, related_name="nodengramngramx", on_delete=models.CASCADE)
ngramy = models.ForeignKey(Ngram, related_name="nodengramngramy", on_delete=models.CASCADE)
......@@ -303,8 +303,8 @@ class NodeNgramNgram(models.Model):
class NodeNodeNgram(models.Model):
nodex = models.ForeignKey(Node, related_name="nodex")
nodey = models.ForeignKey(Node, related_name="nodey")
nodex = models.ForeignKey(Node, related_name="nodex", on_delete=models.CASCADE)
nodey = models.ForeignKey(Node, related_name="nodey", on_delete=models.CASCADE)
ngram = models.ForeignKey(Ngram, on_delete=models.CASCADE)
......@@ -314,8 +314,8 @@ class NodeNodeNgram(models.Model):
return "%s: %s / %s = %s" % (self.nodex.name, self.nodey.name, self.ngram.terms, self.score)
class NodeNodeNgram(models.Model):
nodex = models.ForeignKey(Node, related_name="nodex")
nodey = models.ForeignKey(Node, related_name="nodey")
nodex = models.ForeignKey(Node, related_name="nodex", on_delete=models.CASCADE)
nodey = models.ForeignKey(Node, related_name="nodey", on_delete=models.CASCADE)
ngram = models.ForeignKey(Ngram, on_delete=models.CASCADE)
......
static/js/gargantext.angular.js
View file @ 6bb38b1c
......@@ -29,12 +29,12 @@ var operators = {
};
var strDate = function(date) {
return date.getUTCFullYear() + '-' +
('00' + (date.getUTCMonth() + 1)).slice(-2) + '-' +
('00' + date.getUTCDate()).slice(-2) + 'T' +
('00' + date.getUTCHours()).slice(-2) + ':' +
('00' + date.getUTCMinutes()).slice(-2) + ':' +
('00' + date.getUTCSeconds()).slice(-2) + 'Z';
return date.getFullYear() + '-' +
('00' + (date.getMonth() + 1)).slice(-2) + '-' +
('00' + date.getDate()).slice(-2) + 'T' +
('00' + date.getHours()).slice(-2) + ':' +
('00' + date.getMinutes()).slice(-2) + ':' +
('00' + date.getSeconds()).slice(-2) + 'Z';
}
var addZero = function(x) {
return (x<10) ? ('0'+x) : x;
......@@ -46,14 +46,27 @@ var addZeros = function(x, n) {
var groupings = {
datetime: {
century: {
representation: function(x) {return x.toISOString().substr(0, 2) + 'th century'},
truncate: function(x) {return x.substr(0, 2) + '00-01-01T00:00:00Z';},
next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+100); return strDate(x);},
next: function(x) {
x = new Date(x);
x.setFullYear(x.getFullYear()+100);
x.setHours(0);
return strDate(x);
},
},
decade: {
representation: function(x) {return x.toISOString().substr(0, 3)} + '0s',
truncate: function(x) {return x.substr(0, 3) + '0-01-01T00:00:00Z';},
next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+10); return strDate(x);},
next: function(x) {
x = new Date(x);
x.setFullYear(x.getFullYear() + 10);
x.setHours(0);
return strDate(x);
},
},
year: {
representation: function(x) {return x.toISOString().substr(0, 4)},
truncate: function(x) {return x.substr(0, 4) + '-01-01T00:00:00Z';},
next: function(x) {
var y = parseInt(x.substr(0, 4));
......@@ -61,6 +74,7 @@ var groupings = {
},
},
month: {
representation: function(x) {return x.toISOString().substr(0, 7)},
truncate: function(x) {return x.substr(0, 7) + '-01T00:00:00Z';},
next: function(x) {
var m = parseInt(x.substr(5, 2));
......@@ -73,12 +87,19 @@ var groupings = {
},
},
day: {
representation: function(x) {return x.toISOString().substr(0, 10)},
truncate: function(x) {return x.substr(0, 10) + 'T00:00:00Z';},
next: function(x) {x = new Date(x); x.setDate(x.getDate()+1); return strDate(x);},
next: function(x) {
x = new Date(x);
x.setDate(x.getDate() + 1);
x.setHours(0);
return strDate(x);
},
},
},
numeric: {
unit: {
representation: function(x) {return x.toString()},
truncate: function(x) {return Math.round(x)},
next: function(x) {return x+1;},
},
......@@ -323,11 +344,14 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
options: {
axes: {
x: {key: 'x', type: 'date'},
y: {type: 'linear', type: 'numeric'},
y: {key: 'y', type: 'linear', type: 'numeric'},
},
tension: 1.0,
lineMode: 'bundle',
tooltip: {mode: 'scrubber', formatter: function(x, y, series) {return x + ' → ' + y;}},
tooltip: {mode: 'scrubber', formatter: function(x, y, series) {
var grouping = groupings.datetime[$scope.groupingKey];
return grouping.representation(x) + ' → ' + y;
}},
drawLegend: false,
drawDots: true,
columnsHGap: 5
......@@ -354,33 +378,37 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
return false;
}
var results = dataset.results;
var xMinTmp = results[0][0];
var xMaxTmp = results[results.length - 1][0];
if (xMin === undefined || xMinTmp < xMin) {
xMin = xMinTmp;
}
if (xMax === undefined || xMaxTmp < xMax) {
xMax = xMaxTmp;
if (results.length) {
var xMinTmp = results[0][0];
var xMaxTmp = results[results.length - 1][0];
if (xMin === undefined || xMinTmp < xMin) {
xMin = xMinTmp;
}
if (xMax === undefined || xMaxTmp < xMax) {
xMax = xMaxTmp;
}
}
});
// Create the dataObject for interpolation
var dataObject = {};
xMin = grouping.truncate(xMin);
xMax = grouping.truncate(xMax);
for (var x=xMin; x<=xMax; x=grouping.next(x)) {
var row = [];
angular.forEach($scope.datasets, function(){
row.push(0);
});
dataObject[x] = row;
if (xMin != undefined && xMax != undefined) {
xMin = grouping.truncate(xMin);
xMax = grouping.truncate(xMax);
for (var x=xMin; x<=xMax; x=grouping.next(x)) {
var row = [];
angular.forEach($scope.datasets, function(){
row.push(0);
});
dataObject[x] = row;
}
}
// Fill the dataObject with results
angular.forEach($scope.datasets, function(dataset, datasetIndex){
var results = dataset.results;
angular.forEach(results, function(result, r){
var x = grouping.truncate(result[0]);
var y = result[1];
dataObject[x][datasetIndex] += parseFloat(y);
var y = parseFloat(result[1]);
dataObject[x][datasetIndex] += y;
});
});
// Convert this object back to a sorted array
......@@ -401,10 +429,10 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
}
linearData.push(row);
}
// Update the axis
$scope.graph.options.axes.y.min = yMin;
$scope.graph.options.axes.y.max = yMax;
$scope.graph.options.axes.y.ticks = 100;
// // Update the axis
// $scope.graph.options.axes.y.min = yMin;
// $scope.graph.options.axes.y.max = yMax;
// $scope.graph.options.axes.y.ticks = Math.pow(10, Math.floor(Math.abs(Math.log10(yMax - yMin))));
// Finally, update the graph
var series = [];
for (var i=0, n=$scope.datasets.length; i<n; i++) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment