[FEATURE] super-custom charts are now working on /graph-it

gargantext_web/api.py

@@ -189,7 +189,7 @@ class CorpusController:
                 WHERE parent_id = \'%d\'
             ) AS keys
             GROUP BY key
-            ORDER BY count DESC
+            ORDER BY count
         ''' % (Node._meta.db_table, Node._meta.db_table, corpus.id, ))
         # response building
         collection = []
@@ -267,13 +267,41 @@ class CorpusController:
             raise ValidationError('The "mesured" parameter should take one of the following values: "documents.count", "ngrams.count"')
         # query building: filters
         for filter in request.GET.getlist('filters[]', ''):
-            if '|' in filter:
-                filter_array = filter.split("|")
-                key = filter_array[0]
-                values = filter_array[1].replace("'", "\\'").split(",")
-                if key == 'ngram.terms':
-                    conditions.append("%s.terms IN ('%s')" % (Ngram._meta.db_table, "', '".join(values), ))
+            splitFilter = filter.split('.')
+            origin = splitFilter[0]
+            # 127.0.0.1:8000/api/corpus/13410/data
+            #     ?mesured=ngrams.count
+            #     &parameters[]=metadata.publication_date
+            #     &format=json
+            #     &filters[]=ngrams.in.bee,bees
+            #     &filters[]=metadata.language_fullname.eq.English
+            #     &filters[]=metadata.publication_date.gt.1950-01-01
+            #     &filters[]=metadata.publication_date.lt.2000-01-01
+            #     &filters[]=metadata.title.contains.e
+            if origin == 'ngrams':
+                if splitFilter[1] == 'in':
+                    ngrams = '.'.join(splitFilter[2:]).split(',')
+                    map(str.strip, ngrams)
+                    map(lambda ngram: ngram.replace("'", "''"), ngrams)
+                    conditions.append(
+                        "%s.terms IN ('%s')" % (Ngram._meta.db_table, "', '".join(ngrams), )
+                    )
                     join_ngrams = True
+            elif origin == 'metadata':
+                key = splitFilter[1].replace("'", "''")
+                operator = splitFilter[2]
+                value = '.'.join(splitFilter[3:]).replace("'", "''")
+                condition = "%s.metadata->'%s' " % (Node._meta.db_table, key, )
+                if operator == 'contains':
+                    condition += "LIKE '%%%s%%'" % (value, )
+                else:
+                    condition += {
+                        'eq': '=',
+                        'lt': '<=',
+                        'gt': '>=',
+                    }[operator]
+                    condition += " '%s'" % (value, )
+                conditions.append(condition)
             else:
                 raise ValidationError('Unrecognized "filter[]=%s"' % (filter, ))
         # query building: initializing SQL

static/js/graph-it.js

-(function($) {
-
-
-    $.fn.graphIt = function(_width, _height) {
-
-        // Settings: ways to group data
-        var groupings = this.groupings = {
-            datetime: {
-                century: {
-                    truncate: function(x) {return x.substr(0, 2)},
-                    next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+100); return x;},                
-                },
-                decade: {
-                    truncate: function(x) {return x.substr(0, 3)},
-                    next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+10); return x;},                
-                },
-                year: {
-                    truncate: function(x) {return x.substr(0, 4)},
-                    next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+1); return x;},                
-                },
-                month: {
-                    truncate: function(x) {return x.substr(0, 7)},
-                    next: function(x) {x = new Date(x); x.setMonth(x.getMonth()+1); return x;},                
-                },
-                day: {
-                    truncate: function(x) {return x.substr(0, 10)},
-                    next: function(x) {x = new Date(x); x.setDate(x.getDate()+1); return x;},                
-                },
-            },
-            numeric: {
-                unit: {
-                    truncate: function(x) {return Math.round(x)},
-                    next: function(x) {return x+1;},
-                },
-            },
-        };
-        
-        // Main container
-        var container = $('<div>').addClass('graphit-container').appendTo( this.first() );
-
-
-        var data;
-
-        var method;
-        var chartObject;
-
-        var width, height;
-
-        container.feed = function(_data) {
-            data = _data;
-            var dimensions = data[0].length;
-        };
-        container.resize = function(_width, _height) {
-            width = _width;
-            height = _height;
-            dcChart.width(width).height(height);
-            container.width(width).height(height);
-        // .dimension(fluctuation)
-        // .group(fluctuationGroup)
-        // .elasticY(true)
-            return container;
-        };
-        container.plot = function(_method) {
-            method = _method;
-            container.empty();
-        };
+// Über graph container
+var divChart = $('<div>').appendTo('.visualization');
+
+// Settings: ways to group data
+var groupings = this.groupings = {
+    datetime: {
+        century: {
+            truncate: function(x) {return x.substr(0, 2)},
+            next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+100); return x;},                
+        },
+        decade: {
+            truncate: function(x) {return x.substr(0, 3)},
+            next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+10); return x;},                
+        },
+        year: {
+            truncate: function(x) {return x.substr(0, 4)},
+            next: function(x) {x = new Date(x); x.setFullYear(x.getFullYear()+1); return x;},                
+        },
+        month: {
+            truncate: function(x) {return x.substr(0, 7)},
+            next: function(x) {x = new Date(x); x.setMonth(x.getMonth()+1); return x;},                
+        },
+        day: {
+            truncate: function(x) {return x.substr(0, 10)},
+            next: function(x) {x = new Date(x); x.setDate(x.getDate()+1); return x;},                
+        },
+    },
+    numeric: {
+        unit: {
+            truncate: function(x) {return Math.round(x)},
+            next: function(x) {return x+1;},
+        },
+    },
+};
+
+
+
+var graphIt = function(corpusId, getDataCollection) {
+
+    divChart.empty();
+
+    // Get data from server
+    var dimensions;
+    var series = [];
+    $.each(getDataCollection, function(i, getData) {
+        var responseData;
+        $.ajax('/api/corpus/' + corpusId + '/data', {
+            async: false,
+            success: function(response) {responseData = response;},
+            data: getData,
+        });
+        dimensions = responseData.dimensions;
+        // add to the series
+        series.push({
+            name: '#' + i,
+            data: responseData.list,
+        });
+    });
 
-        container.css('background', '#FFF');
-
-        
-        var corpusId = 13410;
-        var series = [];
-        var keywordsList = ['bee,bees', 'brain,virus'];
-        var dimensions;
-
-        // generate data
-        for (var i=0; i<keywordsList.length; i++) {
-            keywords = keywordsList[i];
-            // get data from server
-            var data;
-            $.ajax('/api/corpus/' + corpusId + '/data', {
-                async: false,
-                success: function(response) {data = response;},
-                data: {
-                    mesured:    'documents.count',
-                    parameters: ['metadata.publication_date'],
-                    filters:    ['ngram.terms|' + keywords],
-                    format:     'json',
-                },
-            });
-            dimensions = data.dimensions;
-            // add to the series
-            series.push({
-                name: keywords,
-                data: data.list,
-            });
-        }
-        
-        
-
-        var grouping = groupings.datetime.year;
-
-        // generate associative data
-        var associativeData = {};
-        for (var s=0; s<series.length; s++) {
-            var data = series[s].data;
-            for (var d=0; d<data.length; d++) {
-                var row = data[d];
-                var x = grouping.truncate(row[0]);
-                var y = row[1];
-                if (!associativeData[x]) {
-                    associativeData[x] = new Array(series.length);
-                    for (var i=0; i<series.length; i++) {
-                        associativeData[x][i] = 0;
-                    }
+    var grouping = groupings.datetime.year;
+
+    // generate associative data
+    var associativeData = {};
+    for (var s=0; s<series.length; s++) {
+        var data = series[s].data;
+        for (var d=0; d<data.length; d++) {
+            var row = data[d];
+            var x = grouping.truncate(row[0]);
+            var y = row[1];
+            if (!associativeData[x]) {
+                associativeData[x] = new Array(series.length);
+                for (var i=0; i<series.length; i++) {
+                    associativeData[x][i] = 0;
                 }
-                associativeData[x][s] += y;
             }
-        };
-
-        // now, flatten this
-        var linearData = [];
-        for (var x in associativeData) {
-            var row = associativeData[x];
-            row.unshift(x);
-            linearData.push(row);
+            associativeData[x][s] += y;
         }
+    };
+
+    // now, flatten this
+    var linearData = [];
+    for (var x in associativeData) {
+        var row = associativeData[x];
+        row.unshift(x);
+        linearData.push(row);
+    }
 
-        // extrema retrieval & scalar data formatting
+    // extrema retrieval & scalar data formatting
+    for (var d=0; d<dimensions.length; d++) {
+        dimensions[d].extrema = {};
+    }
+    var keys = {};
+    for (var r=0; r<linearData.length; r++) {
+        var row = linearData[r];
         for (var d=0; d<dimensions.length; d++) {
-            dimensions[d].extrema = {};
-        }
-        var keys = {};
-        for (var r=0; r<linearData.length; r++) {
-            var row = linearData[r];
-            for (var d=0; d<dimensions.length; d++) {
-                var value = row[d];
-                var dimension = dimensions[d];
-                switch (dimension.type) {
-                    case 'datetime':
-                        value += '2000-01-01 00:00:00'.substr(value.length);
-                        value = new Date(value.replace(' ', 'T') + '.000Z');
-                        break;
-                    case 'numeric':
-                        value = +value;
-                        break;
-                }
-                if (dimension.extrema.min == undefined || value < dimension.extrema.min) {
-                    dimension.extrema.min = value;
-                }
-                if (dimension.extrema.max == undefined || value > dimension.extrema.max) {
-                    dimension.extrema.max = value;
-                }
-                row[d] = value;
+            var value = row[d];
+            var dimension = dimensions[d];
+            switch (dimension.type) {
+                case 'datetime':
+                    value += '2000-01-01 00:00:00'.substr(value.length);
+                    value = new Date(value.replace(' ', 'T') + '.000Z');
+                    break;
+                case 'numeric':
+                    value = +value;
+                    break;
+            }
+            if (dimension.extrema.min == undefined || value < dimension.extrema.min) {
+                dimension.extrema.min = value;
+            }
+            if (dimension.extrema.max == undefined || value > dimension.extrema.max) {
+                dimension.extrema.max = value;
             }
-            keys[row[0]] = true;
+            row[d] = value;
         }
+        keys[row[0]] = true;
+    }
 
-        // interpolation
-        var xMin = dimensions[0].extrema.min;
-        var xMax = dimensions[0].extrema.max;
-        for (var x=xMin; x<xMax; x=grouping.next(x)) {
-            if (!keys[x]) {
-                // TODO: this below is just WRONG
-                var row = [x, 0, 0];
-                linearData.push(row);
-            }
+    // interpolation
+    var xMin = dimensions[0].extrema.min;
+    var xMax = dimensions[0].extrema.max;
+    for (var x=xMin; x<xMax; x=grouping.next(x)) {
+        if (!keys[x]) {
+            // TODO: this below is just WRONG
+            var row = [x, 0, 0];
+            linearData.push(row);
         }
-        linearData.sort(function(row1, row2) {
-            return row1[0] > row2[0];
-        });
+    }
+    linearData.sort(function(row1, row2) {
+        return row1[0] > row2[0];
+    });
 
 
-        // do the graph!
-        var labels = [dimensions[0].key];
-        for (var k=0; k<keywordsList.length; k++) {
-            labels.push(keywordsList[k]);
-        }
-        // var _chartObject = new Dygraph(container[0], linearData);
-        chartObject = new Dygraph(container[0], linearData, {
-            // legends
-            legend: 'always',
-            xlabel: dimensions[0].key,
-            ylabel: dimensions[1].key,
-            labels: labels,
-            axisLabelColor: 'black',
-
-            // appearance
-            fillGraph: true,
-
-            // smoothing
-            showRoller: false,
-            rollPeriod: 5,
-        });
+    // do the graph!
+    // var labels = [dimensions[0].key];
+    // for (var k=0; k<keywordsList.length; k++) {
+    //     labels.push(keywordsList[k]);
+    // }
+    // var _chartObject = new Dygraph(container[0], linearData);
+    chartObject = new Dygraph(divChart[0], linearData, {
+        // legends
+        legend: 'always',
+        xlabel: dimensions[0].key,
+        ylabel: dimensions[1].key,
+        // labels: labels,
+        axisLabelColor: 'black',
+
+        // appearance
+        fillGraph: true,
+
+        // smoothing
+        showRoller: false,
+        rollPeriod: 1,
+    });
 
-        // console.log(associativeData);
-        console.log(linearData);
-        console.log(dimensions);
+    // console.log(associativeData);
+    // console.log(linearData);
+    // console.log(dimensions);
 
+};
 
-        return container;
-    };
 
-})(jQuery);
 
 
+var ulDatasets = $('<ul>').prependTo('.visualization');
+var selectProject = $('<select>').prependTo('.visualization');
+var selectCorpus = $('<select>').prependTo('.visualization');
 
+var metadataCollection;
+var corpusId;
 
-var selectProject = $('<select>').appendTo('.visualization');
-var selectCorpus = $('<select>').appendTo('.visualization');
-var divFilter = $('<div>').appendTo('.visualization');
+var buttonAddDataset = $('<button>').text('Add a dataset').insertAfter(ulDatasets);
+var buttonView = $('<button>').text('Graph it!').click(function(e) {
+    var getDataCollection = [];
+    ulDatasets.children().filter('li').each(function(i, liDataset) {
+        liDataset = $(liDataset);
+        var getData = {
+            mesured:      liDataset.find('*[name]').first().val(),
+            parameters:   ['metadata.publication_date'],
+            filters:      [],
+            format:       'json',
+        };
+        liDataset.find('li *[name]:visible').each(function(i, field){
+            field = $(field);
+            var filter = field.attr('name') + '.' + field.val();
+            getData.filters.push(filter);
+        });
+        getDataCollection.push(getData);
+    });
+    graphIt(selectCorpus.val(), getDataCollection);
+    console.log(getDataCollection);
+}).insertAfter(ulDatasets);
 
-// Load projects
-$.get('/api/nodes', {type:'Project'}, function(collection) {
-    selectProject.empty();
-    for (var i=0; i<collection.length; i++) {
-        var node = collection[i];
-        $('<option>').val(node.id).text(node.text).appendTo(selectProject);
-    }
-    selectProject.change();
+
+// Load metadata
+selectCorpus.change(function() {
+    corpusId = selectCorpus.val();
+    ulDatasets.empty();
+    $.get('/api/corpus/' + corpusId + '/metadata', function(collection) {
+        // Unleash the power of the filter!
+        metadataCollection = collection;
+        buttonAddDataset.click();
+    });
 });
 
 // Load corpora
@@ -232,74 +214,126 @@ selectProject.change(function() {
     });
 });
 
-// Load metadata
-selectCorpus.change(function() {
-    var corpusId = selectCorpus.val();
-    // alert(corpusId);
-    divFilter.empty();
-    //
-    $.get('/api/corpus/' + corpusId + '/metadata', function(collection) {
-        var selectType = $('<select>').appendTo(divFilter);
-        //
+// Load projects
+$.get('/api/nodes', {type:'Project'}, function(collection) {
+    selectProject.empty();
+    for (var i=0; i<collection.length; i++) {
+        var node = collection[i];
+        $('<option>').val(node.id).text(node.text).appendTo(selectProject);
+    }
+    selectProject.change();
+});
+
+// Add a dataset
+buttonAddDataset.click(function() {
+
+    var liDataset = $('<li>').appendTo(ulDatasets);
+    // Can we remove this please?
+    $('<button>').appendTo(liDataset).text('x').click(function() {
+        liDataset.remove();
+    });
+    // What do we count?
+    liDataset.append(' Count ');
+    var selectCounted = $('<select>')
+        .attr('name', 'mesured')
+        .appendTo(liDataset);
+    $('<option>')
+        .text('documents')
+        .val('documents.count')
+        .appendTo(selectCounted);
+    $('<option>')
+        .text('ngrams')
+        .val('ngrams.count')
+        .appendTo(selectCounted);
+    liDataset.append(' ');
+    var buttonFilter = $('<button>')
+        .text('(add a filter)')
+        .appendTo(liDataset);
+    // Add a filter when asked
+    var ulFilters = $('<ul>').appendTo(liDataset);
+    var addFilter = function(metadataCollection) {
+        var liFilter = $('<li>').appendTo(ulFilters);
+        liFilter.append('...where the ');
+        // Type of filter: ngrams
+        var selectType = $('<select>').appendTo(liFilter);
         $('<option>').text('ngrams').appendTo(selectType);
-        var spanNgrams = $('<span>').appendTo(divFilter).hide();
-        var inputNgrams = $('<input>').appendTo(spanNgrams);
-        //
+        var spanNgrams = $('<span>').appendTo(liFilter).hide();
+        spanNgrams.append(' contain ');
+        var inputNgrams = $('<input>')
+            .attr('name', 'ngrams.in')
+            .appendTo(spanNgrams);
+        spanNgrams.append(' (comma-separated ngrams)')
+        // Type of filter: metadata
         $('<option>').text('metadata').appendTo(selectType);
-        var spanMetadata = $('<span>').appendTo(divFilter).hide();
+        var spanMetadata = $('<span>').appendTo(liFilter).hide();
         var selectMetadata = $('<select>').appendTo(spanMetadata);
         var spanMetadataValue = $('<span>').appendTo(spanMetadata);
-        $.each(collection, function(i, metadata) {
+        $.each(metadataCollection, function(i, metadata) {
             $('<option>')
-                .data(metadata)
+                .appendTo(selectMetadata)
                 .text(metadata.text)
-                .appendTo(selectMetadata);
+                .data(metadata);
         });
-        //
+        // How do we present the metadata?
         selectMetadata.change(function() {
             var metadata = selectMetadata.find(':selected').data();
             spanMetadataValue.empty();
             if (metadata.type == 'datetime') {
-                $('<span>').text(' between: ').appendTo(spanMetadataValue);
+                spanMetadataValue.append(' is between ');
                 $('<input>').appendTo(spanMetadataValue)
+                    .attr('name', 'metadata.' + metadata.key + '.gt')
+                    .datepicker({dateFormat: 'yy-mm-dd'})
                     .blur(function() {
                         var input = $(this);
                         var date = input.val();
                         date += '2000-01-01'.substr(date.length);
                         input.val(date);
-                    }).datepicker({dateFormat: 'yy-mm-dd'});
-                $('<span>').text(' and: ').appendTo(spanMetadataValue);
+                    });
+                spanMetadataValue.append(' and ');
                 $('<input>').appendTo(spanMetadataValue)
+                    .attr('name', 'metadata.' + metadata.key + '.lt')
+                    .datepicker({dateFormat: 'yy-mm-dd'})
                     .blur(function() {
                         var input = $(this);
                         var date = input.val();
                         date += '2000-01-01'.substr(date.length);
                         input.val(date);
-                    }).datepicker({dateFormat: 'yy-mm-dd'});
+                    });
             } else if (metadata.values) {
-                $('<span>').text(' is: ').appendTo(spanMetadataValue);
-                var selectMetadataValue = $('<select>').appendTo(spanMetadataValue);
+                $('<span>').text(' is ').appendTo(spanMetadataValue);
+                var selectMetadataValue = $('<select>')
+                    .attr('name', 'metadata.' + metadata.key + '.eq')
+                    .appendTo(spanMetadataValue);
                 $.each(metadata.values, function(i, value) {
-                    $('<option>').text(value).appendTo(selectMetadataValue);
+                    $('<option>')
+                        .text(value)
+                        .appendTo(selectMetadataValue);
                 });
                 selectMetadataValue.change().focus();
             } else {
-                $('<span>').text(' contains: ').appendTo(spanMetadataValue);
-                $('<input>').appendTo(spanMetadataValue).focus();
+                spanMetadataValue.append(' contains ');
+                $('<input>')
+                    .attr('name', 'metadata.' + metadata.key + '.contains')
+                    .appendTo(spanMetadataValue)
+                    .focus();
             }
-        });
-        //
+        }).change();
+        // Ngram or metadata?
         selectType.change(function() {
-            divFilter.children().filter('span').hide();
+            var spans = liFilter.children().filter('span').hide();
             switch (selectType.val()) {
                 case 'ngrams':
-                    spanNgrams.show();
+                    spanNgrams.show().find('input').focus();
                     break;
                 case 'metadata':
                     spanMetadata.show();
                     break;
             }
         }).change();
+    };
+
+    buttonFilter.click(function(e) {
+        addFilter(metadataCollection);
     });
 });
 
@@ -307,7 +341,6 @@ selectCorpus.change(function() {
 
 
 
-
 // $('.tree').jstree({
 //     'core' : {
 //         'data' : {