Merge branch 'mat-settings' into unstable

gargantext_web/api.py

@@ -136,9 +136,8 @@ _ngrams_order_columns = {
 
 from rest_framework.views import APIView
 from rest_framework.response import Response
-from rest_framework.exceptions import APIException
+from rest_framework.exceptions import APIException as _APIException
 
-_APIException = APIException
 class APIException(_APIException):
     def __init__(self, message, code=500):
         self.status_code = code
@@ -167,6 +166,91 @@ def Root(request, format=None):
     })
 
 
+class NodesChildrenDuplicates(APIView):
+
+    def _fetch_duplicates(self, request, node_id, extra_columns=[], min_count=1):
+        # input validation
+        if 'keys' not in request.GET:
+            raise APIException('Missing GET parameter: "keys"', 400)
+        keys = request.GET['keys'].split(',')
+        # metadata retrieval
+        metadata_query = (Metadata
+            .query(Metadata)
+            .filter(Metadata.name.in_(keys))
+        )
+        # build query elements
+        columns = []
+        aliases = []
+        for metadata in metadata_query:
+            # aliases
+            _Metadata = aliased(Metadata)
+            _Node_Metadata = aliased(Node_Metadata)
+            aliases.append(_Node_Metadata)
+            # what shall we retrieve?
+            columns.append(
+                getattr(_Node_Metadata, 'value_' + metadata.type)
+            )
+        # build the query
+        groups = list(columns)
+        duplicates_query = (get_session()
+            .query(*(extra_columns + [func.count()] + columns))
+            .select_from(Node)
+        )
+        for _Node_Metadata, metadata in zip(aliases, metadata_query):
+            duplicates_query = duplicates_query.outerjoin(_Node_Metadata, _Node_Metadata.node_id == Node.id)
+            duplicates_query = duplicates_query.filter(_Node_Metadata.metadata_id == metadata.id)
+        duplicates_query = duplicates_query.filter(Node.parent_id == node_id)
+        duplicates_query = duplicates_query.group_by(*columns)
+        duplicates_query = duplicates_query.order_by(func.count().desc())
+        duplicates_query = duplicates_query.having(func.count() > min_count)
+        # and now, return it
+        return duplicates_query
+
+    def get(self, request, node_id):
+        # data to be returned
+        duplicates = self._fetch_duplicates(request, node_id)
+        # pagination
+        offset = int(request.GET.get('offset', 0))
+        limit = int(request.GET.get('limit', 10))
+        total = duplicates.count()
+        # response building
+        return JsonHttpResponse({
+            'pagination': {
+                'offset': offset,
+                'limit': limit,
+                'total': total,
+            },
+            'data': [
+                {
+                    'count': duplicate[0],
+                    'values': duplicate[1:],
+                }
+                for duplicate in duplicates[offset : offset+limit]
+            ]
+        })
+
+    def delete(self, request, node_id):
+        session = get_session()
+        # get the minimum ID for each of the nodes sharing the same metadata
+        kept_node_ids_query = self._fetch_duplicates(request, node_id, [func.min(Node.id).label('id')], 0)
+        kept_node_ids = [kept_node.id for kept_node in kept_node_ids_query]
+        # delete the stuff
+        delete_query = (session
+            .query(Node)
+            .filter(Node.parent_id == node_id)
+            .filter(~Node.id.in_(kept_node_ids))
+        )
+        count = delete_query.count()
+        delete_query.delete(synchronize_session=False)
+        session.flush()
+        # return the result
+        return JsonHttpResponse({
+            'deleted': count,
+        })
+        # return duplicates_query
+
+
+
 class NodesChildrenMetatadata(APIView):
 
     def get(self, request, node_id):

gargantext_web/settings.py

@@ -62,7 +62,7 @@ INSTALLED_APPS = (
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_extensions',
-    #'south',
+    'south',
     'cte_tree',
     'node',
     'ngram',

gargantext_web/urls.py

@@ -51,6 +51,7 @@ urlpatterns = patterns('',
     url(r'^api$', gargantext_web.api.Root),
     url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
     url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
+    url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
     url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
     url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
     

node/models.py

@@ -245,7 +245,7 @@ class Node(CTENode):
         print("In workflow() END")
 
 class Node_Metadata(models.Model):
-    node        = models.ForeignKey(Node)
+    node        = models.ForeignKey(Node, on_delete=models.CASCADE)
     metadata    = models.ForeignKey(Metadata)
     value_int   = models.IntegerField(null=True, db_index=True)
     value_float = models.FloatField(null=True, db_index=True)
@@ -254,12 +254,12 @@ class Node_Metadata(models.Model):
     value_text  = models.TextField(null=True)
 
 class Node_Resource(models.Model):
-    node     = models.ForeignKey(Node, related_name='node_resource')
+    node     = models.ForeignKey(Node, related_name='node_resource', on_delete=models.CASCADE)
     resource = models.ForeignKey(Resource)
     parsed   = models.BooleanField(default=False)
             
 class Node_Ngram(models.Model):
-    node   = models.ForeignKey(Node)
+    node   = models.ForeignKey(Node, on_delete=models.CASCADE)
     ngram  = models.ForeignKey(Ngram)
     weight = models.FloatField()
     def __str__(self):
@@ -291,7 +291,7 @@ class Document(Node):
         proxy=True
 
 class NodeNgramNgram(models.Model):
-    node        = models.ForeignKey(Node)
+    node        = models.ForeignKey(Node, on_delete=models.CASCADE)
     
     ngramx      = models.ForeignKey(Ngram, related_name="nodengramngramx", on_delete=models.CASCADE)
     ngramy      = models.ForeignKey(Ngram, related_name="nodengramngramy", on_delete=models.CASCADE)
@@ -303,8 +303,8 @@ class NodeNgramNgram(models.Model):
 
 
 class NodeNodeNgram(models.Model):
-    nodex        = models.ForeignKey(Node, related_name="nodex")
-    nodey        = models.ForeignKey(Node, related_name="nodey")
+    nodex        = models.ForeignKey(Node, related_name="nodex", on_delete=models.CASCADE)
+    nodey        = models.ForeignKey(Node, related_name="nodey", on_delete=models.CASCADE)
     
     ngram      = models.ForeignKey(Ngram, on_delete=models.CASCADE)
 
@@ -314,8 +314,8 @@ class NodeNodeNgram(models.Model):
         return "%s: %s / %s = %s" % (self.nodex.name, self.nodey.name, self.ngram.terms, self.score)
 
 class NodeNodeNgram(models.Model):
-    nodex        = models.ForeignKey(Node, related_name="nodex")
-    nodey        = models.ForeignKey(Node, related_name="nodey")
+    nodex        = models.ForeignKey(Node, related_name="nodex", on_delete=models.CASCADE)
+    nodey        = models.ForeignKey(Node, related_name="nodey", on_delete=models.CASCADE)
     
     ngram      = models.ForeignKey(Ngram, on_delete=models.CASCADE)
 

static/js/gargantext.angular.js

@@ -29,12 +29,12 @@ var operators = {
 };
 
 var strDate = function(date) {
-    return date.getUTCFullYear() + '-' +
-            ('00' + (date.getUTCMonth() + 1)).slice(-2) + '-' +
-            ('00' + date.getUTCDate()).slice(-2) + 'T' +
-            ('00' + date.getUTCHours()).slice(-2) + ':' +
-            ('00' + date.getUTCMinutes()).slice(-2) + ':' +
-            ('00' + date.getUTCSeconds()).slice(-2) + 'Z';
+    return date.getFullYear() + '-' +
+            ('00' + (date.getMonth() + 1)).slice(-2) + '-' +
+            ('00' + date.getDate()).slice(-2) + 'T' +
+            ('00' + date.getHours()).slice(-2) + ':' +
+            ('00' + date.getMinutes()).slice(-2) + ':' +
+            ('00' + date.getSeconds()).slice(-2) + 'Z';
 }
 var addZero = function(x) {
     return (x<10) ? ('0'+x) : x;
@@ -46,14 +46,27 @@ var addZeros = function(x, n) {
 var groupings = {
     datetime: {
         century: {
+            representation: function(x) {return x.toISOString().substr(0, 2) + 'th century'},
             truncate: function(x) {return x.substr(0, 2) + '00-01-01T00:00:00Z';},
-            next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+100); return strDate(x);},
+            next: function(x) {
+                x = new Date(x);
+                x.setFullYear(x.getFullYear()+100);
+                x.setHours(0);
+                return strDate(x);
+            },
         },
         decade: {
+            representation: function(x) {return x.toISOString().substr(0, 3)} + '0s',
             truncate: function(x) {return x.substr(0, 3) + '0-01-01T00:00:00Z';},
-            next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+10); return strDate(x);},
+            next: function(x) {
+                x = new Date(x);
+                x.setFullYear(x.getFullYear() + 10);
+                x.setHours(0);
+                return strDate(x);
+            },
         },
         year: {
+            representation: function(x) {return x.toISOString().substr(0, 4)},
             truncate: function(x) {return x.substr(0, 4) + '-01-01T00:00:00Z';},
             next: function(x) {
                 var y = parseInt(x.substr(0, 4));
@@ -61,6 +74,7 @@ var groupings = {
             },
         },
         month: {
+            representation: function(x) {return x.toISOString().substr(0, 7)},
             truncate: function(x) {return x.substr(0, 7) + '-01T00:00:00Z';},
             next: function(x) {
                 var m = parseInt(x.substr(5, 2));
@@ -73,12 +87,19 @@ var groupings = {
             },
         },
         day: {
+            representation: function(x) {return x.toISOString().substr(0, 10)},
             truncate: function(x) {return x.substr(0, 10) + 'T00:00:00Z';},
-            next: function(x) {x = new Date(x); x.setDate(x.getDate()+1); return strDate(x);},
+            next: function(x) {
+                x = new Date(x);
+                x.setDate(x.getDate() + 1);
+                x.setHours(0);
+                return strDate(x);
+            },
         },
     },
     numeric: {
         unit: {
+            representation: function(x) {return x.toString()},
             truncate: function(x) {return Math.round(x)},
             next: function(x) {return x+1;},
         },
@@ -323,11 +344,14 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
         options: {
             axes: {
                 x: {key: 'x', type: 'date'},
-                y: {type: 'linear', type: 'numeric'},
+                y: {key: 'y', type: 'linear', type: 'numeric'},
             },
             tension: 1.0,
             lineMode: 'bundle',
-            tooltip: {mode: 'scrubber', formatter: function(x, y, series) {return x + ' → ' + y;}},
+            tooltip: {mode: 'scrubber', formatter: function(x, y, series) {
+                var grouping = groupings.datetime[$scope.groupingKey];
+                return grouping.representation(x) + ' → ' + y;
+            }},
             drawLegend: false,
             drawDots: true,
             columnsHGap: 5
@@ -354,33 +378,37 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
                 return false;
             }
             var results = dataset.results;
-            var xMinTmp = results[0][0];
-            var xMaxTmp = results[results.length - 1][0];
-            if (xMin === undefined || xMinTmp < xMin) {
-                xMin = xMinTmp;
-            }
-            if (xMax === undefined || xMaxTmp < xMax) {
-                xMax = xMaxTmp;
+            if (results.length) {
+                var xMinTmp = results[0][0];
+                var xMaxTmp = results[results.length - 1][0];
+                if (xMin === undefined || xMinTmp < xMin) {
+                    xMin = xMinTmp;
+                }
+                if (xMax === undefined || xMaxTmp < xMax) {
+                    xMax = xMaxTmp;
+                }
             }
         });
         // Create the dataObject for interpolation
         var dataObject = {};
-        xMin = grouping.truncate(xMin);
-        xMax = grouping.truncate(xMax);
-        for (var x=xMin; x<=xMax; x=grouping.next(x)) {
-            var row = [];
-            angular.forEach($scope.datasets, function(){
-                row.push(0);
-            });
-            dataObject[x] = row;
+        if (xMin != undefined && xMax != undefined) {
+            xMin = grouping.truncate(xMin);
+            xMax = grouping.truncate(xMax);
+            for (var x=xMin; x<=xMax; x=grouping.next(x)) {
+                var row = [];
+                angular.forEach($scope.datasets, function(){
+                    row.push(0);
+                });
+                dataObject[x] = row;
+            }
         }
         // Fill the dataObject with results
         angular.forEach($scope.datasets, function(dataset, datasetIndex){
             var results = dataset.results;
             angular.forEach(results, function(result, r){
                 var x = grouping.truncate(result[0]);
-                var y = result[1];
-                dataObject[x][datasetIndex] += parseFloat(y);
+                var y = parseFloat(result[1]);
+                dataObject[x][datasetIndex] += y;
             });
         });
         // Convert this object back to a sorted array
@@ -401,10 +429,10 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
             }
             linearData.push(row);
         }
-        // Update the axis
-        $scope.graph.options.axes.y.min = yMin;
-        $scope.graph.options.axes.y.max = yMax;
-        $scope.graph.options.axes.y.ticks = 100;
+        // // Update the axis
+        // $scope.graph.options.axes.y.min = yMin;
+        // $scope.graph.options.axes.y.max = yMax;
+        // $scope.graph.options.axes.y.ticks = Math.pow(10, Math.floor(Math.abs(Math.log10(yMax - yMin))));
         // Finally, update the graph
         var series = [];
         for (var i=0, n=$scope.datasets.length; i<n; i++) {