Skip to content

gargantext
gargantext
Commit 147d96f6 authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

[FEATURE] improved back-end API 

[TEST] tried to do something useful with "dc.js"
parent bf0d4e88
Hide whitespace changes
Inline Side-by-side
Showing with 259 additions and 29 deletions
gargantext_web/api.py
View file @ 147d96f6
...@@ -50,7 +50,7 @@ def CsvHttpResponse(data, headers=None, status=200): ...@@ -50,7 +50,7 @@ def CsvHttpResponse(data, headers=None, status=200):
content_type = "text/csv", content_type = "text/csv",
status = status status = status
) )
writer = csv.writer(response) writer = csv.writer(response, delimiter=',')
if headers: if headers:
writer.writerow(headers) writer.writerow(headers)
for row in data: for row in data:
...@@ -98,7 +98,7 @@ class CorpusController: ...@@ -98,7 +98,7 @@ class CorpusController:
# query building # query building
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute(_sql_cte + ''' cursor.execute(_sql_cte + '''
SELECT ngram.terms SELECT ngram.terms, COUNT(*) AS occurrences
FROM cte FROM cte
INNER JOIN %s AS node ON node.id = cte.id INNER JOIN %s AS node ON node.id = cte.id
INNER JOIN %s AS nodetype ON nodetype.id = node.type_id INNER JOIN %s AS nodetype ON nodetype.id = node.type_id
...@@ -108,7 +108,7 @@ class CorpusController: ...@@ -108,7 +108,7 @@ class CorpusController:
AND nodetype.name = 'Document' AND nodetype.name = 'Document'
AND ngram.terms LIKE '%s%%' AND ngram.terms LIKE '%s%%'
GROUP BY ngram.terms GROUP BY ngram.terms
ORDER BY SUM(node_ngram.weight) DESC ORDER BY occurrences DESC
''' % ( ''' % (
Node._meta.db_table, Node._meta.db_table,
NodeType._meta.db_table, NodeType._meta.db_table,
...@@ -118,10 +118,26 @@ class CorpusController: ...@@ -118,10 +118,26 @@ class CorpusController:
corpus.id, corpus.id,
request.GET.get('startwith', '').replace("'", "\\'"), request.GET.get('startwith', '').replace("'", "\\'"),
)) ))
# # response building
# return JsonHttpResponse({
# "list" : [row[0] for row in cursor.fetchall()],
# })
# response building # response building
return JsonHttpResponse({ format = request.GET.get('format', 'json')
"list" : [row[0] for row in cursor.fetchall()], if format == 'json':
}) return JsonHttpResponse({
"list": [{
'terms': row[0],
'occurrences': row[1]
} for row in cursor.fetchall()],
})
elif format == 'csv':
return CsvHttpResponse(
[['terms', 'occurences']] + [row for row in cursor.fetchall()]
)
else:
raise ValidationError('Unrecognized "format=%s", should be "csv" or "json"' % (format, ))
@classmethod @classmethod
def metadata(cls, request, corpus_id): def metadata(cls, request, corpus_id):
...@@ -154,9 +170,11 @@ class CorpusController: ...@@ -154,9 +170,11 @@ class CorpusController:
conditions = [] conditions = []
group = [] group = []
order = [] order = []
having = []
join_ngrams = False join_ngrams = False
# query building: parameters # query building: parameters
for parameter in request.GET.getlist('parameters[]'): parameters = request.GET.getlist('parameters[]')
for parameter in parameters:
c = len(columns) c = len(columns)
parameter_array = parameter.split('.') parameter_array = parameter.split('.')
if len(parameter_array) != 2: if len(parameter_array) != 2:
...@@ -166,6 +184,7 @@ class CorpusController: ...@@ -166,6 +184,7 @@ class CorpusController:
if origin == "metadata": if origin == "metadata":
columns.append("%s.metadata->'%s' AS c%d" % (Node._meta.db_table, key, c, )) columns.append("%s.metadata->'%s' AS c%d" % (Node._meta.db_table, key, c, ))
conditions.append("%s.metadata ? '%s'" % (Node._meta.db_table, key, )) conditions.append("%s.metadata ? '%s'" % (Node._meta.db_table, key, ))
# conditions.append("c%d IS NOT NULL" % (c, ))
group.append("c%d" % (c, )) group.append("c%d" % (c, ))
order.append("c%d" % (c, )) order.append("c%d" % (c, ))
else: else:
...@@ -220,12 +239,26 @@ class CorpusController: ...@@ -220,12 +239,26 @@ class CorpusController:
cursor.execute(sql) cursor.execute(sql)
# response building # response building
format = request.GET.get('format', 'json') format = request.GET.get('format', 'json')
keys = parameters + [mesured]
rows = cursor.fetchall()
if format == 'json': if format == 'json':
dimensions = []
for key in keys:
suffix = key.split('_')[-1]
dimensions.append({
'key': key,
'type': 'date' if suffix == 'date' else 'numeric'
})
return JsonHttpResponse({ return JsonHttpResponse({
"list": [row for row in cursor.fetchall()], "collection": [
{key: value for key, value in zip(keys, row)}
for row in rows
],
"list": [row for row in rows],
"dimensions" : dimensions
}) })
elif format == 'csv': elif format == 'csv':
return CsvHttpResponse(row for row in cursor.fetchall()) return CsvHttpResponse([keys] + [row for row in rows])
else: else:
raise ValidationError('Unrecognized "format=%s", should be "csv" or "json"' % (format, )) raise ValidationError('Unrecognized "format=%s", should be "csv" or "json"' % (format, ))
static/js/graph-it.js
View file @ 147d96f6
...@@ -3,11 +3,13 @@ ...@@ -3,11 +3,13 @@
$.fn.graphIt = function(_width, _height) { $.fn.graphIt = function(_width, _height) {
var container = this.first(); var container = $('<div>').addClass('graphit-container').appendTo( this.first() );
var container2 = $('<div>').addClass('graphit-container-2').appendTo( this.first() );
var data; var data;
var method; var method;
var dcChart; var dcChart, dcVolumeChart;
var width, height;
container.feed = function(_data) { container.feed = function(_data) {
data = _data; data = _data;
...@@ -29,20 +31,213 @@ ...@@ -29,20 +31,213 @@
}; };
container.css('background', '#FFF'); container.css('background', '#FFF');
dcChart = dc.barChart(container.get(0));
.centerBar(true) $.get('/api/corpus/13410/data?mesured=ngrams.count&parameters[]=metadata.publication_date&filters[]=ngram.terms|bee,bees&format=json', function(data) {
.dimension(function(d){return d[0]})
// .gap(1) var dateFormat = d3.time.format('%Y-%m-%d %H:%M:%S');
// .round(dc.round.floor) var numberFormat = d3.format('.2f');
// .alwaysUseRounding(true)
// .x(d3.scale.linear().domain([-25, 25])) var unit = 'month';
// .renderHorizontalGridLines(true)
// .filterPrinter(function (filters) { // format all data
// var filter = filters[0], s = ""; for (var i=0; i<data.dimensions.length; i++) {
// s += numberFormat(filter[0]) + "% -> " + numberFormat(filter[1]) + "%"; var dimension = data.dimensions[i];
// return s; var key = dimension.key;
// }); var otherKey;
container.resize(_width, _height); if (dimension.type == 'date') {
otherKey = key.replace(/_date$/, '_' + unit);
}
data.collection.forEach(function(element) {
switch (dimension.type) {
case 'date':
element[key] = dateFormat.parse(element[key]);
break;
case 'numeric':
element[key] = +element[key];
break;
}
});
}
// organize data
var ndx = crossfilter(data.collection);
var all = ndx.groupAll();
// define accessors for every dimension
for (var i=0; i<data.dimensions.length; i++) {
var dimension = data.dimensions[i];
dimension.accessor = ndx.dimension(function(element) {
return element[otherKey];
});
}
var monthlyMoveGroup = data.dimensions[0].accessor.group().reduceSum(function (d) {
return d.volume;
//return Math.abs(+d.close - +d.open);
});
var volumeByMonthGroup = data.dimensions[0].accessor.group().reduceSum(function (d) {
return d.volume / 500000;
});
var indexAvgByMonthGroup = data.dimensions[0].accessor.group().reduce(
function(p, v) {
++p.days;
p.total += (+v.open + +v.close) / 2;
p.avg = Math.round(p.total / p.days);
return p;
},
function(p, v) {
--p.days;
p.total -= (+v.open + +v.close) / 2;
p.avg = p.days == 0 ? 0 : Math.round(p.total / p.days);
return p;
},
function() {
return {days: 0, total: 0, avg: 0};
}
);
var moveChart = dc.compositeChart(".graphit-container");
moveChart.width(800)
.height(150)
.transitionDuration(1000)
.margins({top: 10, right: 50, bottom: 25, left: 40})
.dimension(data.dimensions[0].accessor)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.x(d3.time.scale().domain([new Date(1950,01,01), new Date(2014,12,31)]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.elasticY(true)
.renderHorizontalGridLines(true)
.renderVerticalGridLines(true)
.brushOn(false)
.compose([
dc.lineChart(moveChart)
.group(indexAvgByMonthGroup)
.valueAccessor(function (d) {
return d.value.avg;
})
.renderArea(true)
.stack(monthlyMoveGroup, function (d) {
return d.value;
})
.title(function (d) {
var value = d.value.avg ? d.value.avg : d.value;
if (isNaN(value)) value = 0;
return dateFormat(d.key) + "\n" + numberFormat(value);
})
])
.xAxis();
var volumeChart = dc.barChart(".graphit-container-2");
volumeChart
.width(800)
.height(100)
.margins({top: 0, right: 50, bottom: 20, left: 40})
.dimension(data.dimensions[0].accessor)
.group(volumeByMonthGroup)
.centerBar(true)
.gap(0)
.x(d3.time.scale().domain([new Date(1950,01,01), new Date(2014,12,31)]))
.round(d3.time.month.round)
.xUnits(d3.time.months)
.renderlet(function (chart) {
chart.select("g.y").style("display", "none");
moveChart.filter(chart.filter());
})
.on("filtered", function (chart) {
dc.events.trigger(function () {
moveChart.focus(chart.filter());
});
});
dc.renderAll();
});
return;
// d3.csv('/static/tests/morley.csv', function(error, experiments) {
d3.csv('/api/corpus/13410/data?mesured=ngrams.count&parameters[]=metadata.publication_year&filters[]=ngram.terms|bee,bees&format=csv', function(error, data) {
// DATA PARSING
data.forEach(function(x) {
x.publication_year = +x.publication_year;
x.count = +x.count;
});
var ndx = crossfilter(data),
x = ndx.dimension(function(d) {return +d['metadata.publication_year'];}),
y = [
x.group().reduceSum(function(d) {return +d['ngrams.count'];}),
x.group().reduceSum(function(d) {return +d['ngrams.count'] * 2;})
]
// WHAT'S UNDER THE GRAPH?
// dcVolumeChart = dc.barChart('.graphit-container-2');
// THAT'S THE GRAPH!
dcChart = dc.lineChart('.graphit-container');
// dcChart = dc.barChart('.graphit-container');
dcChart
.width(_width)
.height(_height)
.x(d3.scale.linear())
.elasticX(true)
.elasticY(true)
.renderArea(true)
// .brushOn(false)
// .rangeChart(dcVolumeChart)
.legend(dc.legend().x(.75 * _width).y(.125 * _height).itemHeight(13).gap(5))
// .title(function (d) {
// var value = d.value.avg ? d.value.avg : d.value;
// if (isNaN(value)) value = 0;
// return d.key + "\n" + value;
// })
.renderVerticalGridLines(true)
.renderHorizontalGridLines(true)
// .mouseZoomable(true)
.dimension(x)
.group(y[0], 'Test 1')
.stack(y[1], 'Test 2')
// .renderlet(function(chart) {
// dcChart.selectAll('rect').on("click", function(d) {
// alert('Boo!');
// });
// });
// SERIOUSLY, WHAT'S UNDER THE GRAPH?
// dcVolumeChart
// .width(width)
// .height(.25 * height)
// .dimension(x)
// .group(y[0])
// .stack(y[1])
// .centerBar(true)
// // .gap(1)
// .x(d3.scale.linear())
// // .x(d3.time.scale());
dcChart.render();
});
return container; return container;
}; };
...@@ -50,4 +245,4 @@ ...@@ -50,4 +245,4 @@
var graph = $('.graph-it').graphIt(); var graph = $('.graph-it').graphIt(640, 480);
\ No newline at end of file \ No newline at end of file
templates/graph-it.html
View file @ 147d96f6
...@@ -18,10 +18,12 @@ ...@@ -18,10 +18,12 @@
</div> </div>
</div> </div>
<div class="container graph-it"></div> <div class="container graph-it"></div>
<script type="text/javascript" src="http://jun9.github.io/dc.js/js/d3.js"></script>
<script type="text/javascript" src="http://jun9.github.io/dc.js/js/crossfilter.js"></script>
<script type="text/javascript" src="http://jun9.github.io/dc.js/js/dc.js"></script>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script> <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script type="text/javascript" src="http://dc-js.github.io/dc.js/js/d3.js"></script>
<script type="text/javascript" src="http://dc-js.github.io/dc.js/js/crossfilter.js"></script>
<script type="text/javascript" src="http://dc-js.github.io/dc.js/js/dc.js"></script>
<script type="text/javascript" src="http://dc-js.github.io/dc.js/js/colorbrewer.js"></script>
<script type="text/javascript" src="{% static "js/graph-it.js" %}"></script> <script type="text/javascript" src="{% static "js/graph-it.js" %}"></script>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment