[FEAT] started working on corpus view

[FEAT] started implementing REST API
(corpus view is still quite buggy, but without errors)

gargantext/urls.py

@@ -10,7 +10,7 @@ from django.conf.urls import include, url
 
 from django.contrib import admin
 
-# import gargantext.views.api
+import gargantext.views.api.urls
 import gargantext.views.generated.urls
 import gargantext.views.pages.urls
 
@@ -18,5 +18,6 @@ import gargantext.views.pages.urls
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^generated/', include(gargantext.views.generated.urls)),
+    url(r'^api/', include(gargantext.views.api.urls)),
     url(r'^', include(gargantext.views.pages.urls)),
 ]

gargantext/util/http.py

@@ -9,6 +9,8 @@ from urllib.parse import quote_plus as urlencode
 from gargantext import settings
 
 
+# authentication
+
 def requires_auth(func):
     """Provides a decorator to force authentication on a given view.
     Also passes the URL to redirect towards as a GET parameter.
@@ -21,8 +23,57 @@ def requires_auth(func):
     return _requires_auth
 
 
+# download from a given URL
+
 import urllib.request
 
 def get(url):
     response = urllib.request.urlopen(url)
     html = response.read()
+
+
+# retrieve GET parameters from a request
+
+def get_parameters(request):
+    parameters = {}
+    print(request.GET)
+    print(request.GET._iterlists())
+    for key, value in request.GET._iterlists():
+        if key.endswith('[]'):
+            parameters[key[:-2]] = value
+        else:
+            parameters[key] = value[0]
+    return parameters
+
+
+# REST
+
+from rest_framework.views import APIView
+
+
+# provide a JSON response
+
+import json
+import datetime
+
+class JSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime.datetime):
+            return obj.isoformat()[:19] + 'Z'
+        elif isinstance(obj, (set, tuple)):
+            return list(obj)
+        else:
+            return super(self.__class__, self).default(obj)
+json_encoder = JSONEncoder(indent=4)
+def JsonHttpResponse(data, status=200):
+    return HttpResponse(
+        content      = json_encoder.encode(data),
+        content_type = 'application/json; charset=utf-8',
+        status       = status
+    )
+
+
+# provide exceptions for JSON APIs
+
+from rest_framework.exceptions import APIException
+from rest_framework.exceptions import ValidationError as ValidationException

gargantext/util/validation.py

+from gargantext.util.http import ValidationException
+
+from datetime import datetime
+
+
+__all__ = ['validate']
+
+
+_types_names = {
+    bool:       'boolean',
+    int:        'integer',
+    float:      'float',
+    str:        'string',
+    dict:       'object',
+    list:       'array',
+    datetime:   'datetime',
+}
+
+
+def validate(value, expected, path='input'):
+    # Is the expected type respected?
+    if 'type' in expected:
+        expected_type = expected['type']
+        if not isinstance(value, expected_type):
+            if expected_type in (bool, int, float, str, datetime, ):
+                try:
+                    if expected_type == bool:
+                        value = value not in {0, 0.0, '', '0', 'false'}
+                    elif expected_type == datetime:
+                        value = value + '2000-01-01T00:00:00Z'[len(value):]
+                        value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
+                    else:
+                        value = expected_type(value)
+                except ValueError:
+                    raise ValidationException('%s should be a JSON %s, but could not be parsed as such' % (path, _types_names[expected_type], ))
+            else:
+                raise ValidationException('%s should be a JSON %s' % (path, _types_names[expected_type], ))
+    else:
+        expected_type = type(value)
+
+    # Is the value in the expected range?
+    if 'range' in expected:
+        expected_range = expected['range']
+        if isinstance(expected_range, tuple):
+            if expected_type in (int, float):
+                tested_value = value
+                tested_name = 'value'
+            elif expected_type in (str, list):
+                tested_value = len(value)
+                tested_name = 'length'
+            if tested_value < expected_range[0]:
+                raise ValidationException('%s should have a minimum %s of %d' % (path, tested_name, expected_range[0], ))
+            if len(expected_range) > 1 and tested_value > expected_range[1]:
+                raise ValidationException('%s should have a maximum %s of %d' % (path, tested_name, expected_range[1], ))
+        elif isinstance(expected_range, (list, set, )) and value not in expected_range:
+            expected_values = expected_range if isinstance(expected_range, list) else expected_range
+            expected_values = [str(value) for value in expected_values if isinstance(value, expected_type)]
+            if len(expected_values) < 16:
+                expected_values_str = '", "'.join(expected_values)
+                expected_values_str = '"' + expected_values_str + '"'
+            else:
+                expected_values_str = '", "'.join(expected_values[:16])
+                expected_values_str = '"' + expected_values_str + '"...'
+
+            raise ValidationException('%s should take one of the following values: %s' % (path, expected_values_str, ))
+
+    # Do we have to translate through a dictionary?
+    if 'translate' in expected:
+        translate = expected['translate']
+        if callable(translate):
+            value = translate(value)
+            if value is None and expected.get('required', False):
+                raise ValidationException('%s has been given an invalid value' % (path, ))
+            return value
+        try:
+            value = expected['translate'][value]
+        except KeyError:
+            if expected.get('translate_fallback_keep', False):
+                return value
+            if expected.get('required', False):
+                raise ValidationException('%s has been given an invalid value' % (path, ))
+            else:
+                return expected.get('default', value)
+
+    # Are we handling an iterable?
+    if expected_type in (list, dict):
+        if 'items' in expected:
+            expected_items = expected['items']
+            if expected_type == list:
+                for i, element in enumerate(value):
+                    value[i] = validate(element, expected_items, '%s[%d]' % (path, i, ))
+            elif expected_type == dict:
+                if expected_items:
+                    for key in value:
+                        if key not in expected_items:
+                            raise ValidationException('%s should not have a "%s" key.' % (path, key, ))
+                for expected_key, expected_value in expected_items.items():
+                    if expected_key in value:
+                        value[expected_key] = validate(value[expected_key], expected_value, '%s["%s"]' % (path, expected_key, ))
+                    elif 'required' in expected_value and expected_value['required']:
+                        raise ValidationException('%s should have a "%s" key.' % (path, expected_key, ))
+                    elif 'default' in expected_value:
+                        value[expected_key] = expected_value['default']
+
+    # Let's return the proper value!
+    return value

gargantext/views/api/nodes.py

+from gargantext.util.http import *
+from gargantext.util.db import *
+from gargantext.models import *
+from gargantext.constants import *
+
+from gargantext.util.validation import validate
+
+
+class NodesList(APIView):
+
+    _fields = ['id', 'parent_id', 'name', 'typename', 'hyperdata']
+    _types = NODETYPES
+
+    def _query(self, request):
+        # parameters validation
+        parameters = get_parameters(request)
+        parameters = validate(parameters, {'type': dict, 'items': {
+            'pagination_limit': {'type': int, 'default': 10},
+            'pagination_offset': {'type': int, 'default': 0},
+            'pagination_offset': {'type': int, 'default': 0},
+            'fields': {'type': list, 'default': self._fields, 'items': {
+                'type': str, 'range': self._fields,
+            }},
+            # optional filtering parameters
+            'type': {'type': list, 'default': self._types, 'required': False, 'items': {
+                'type': str, 'range': self._types,
+            }},
+            'parent_id': {'type': int, 'required': False},
+        }})
+        # start the query
+        query = session.query(*tuple(
+            getattr(Node, field) for field in parameters['fields']
+        ))
+        # filter by type
+        if 'type' in parameters:
+            query = query.filter(Node.typename.in_(parameters['type']))
+        # filter by parent
+        if 'parent_id' in parameters:
+            query = query.filter(Node.parent_id == parameters['parent_id'])
+        # paginate the query
+        count = query.count()
+        if parameters['pagination_limit'] == -1:
+            query = query[parameters['pagination_offset']:]
+        else:
+            query = query[
+                parameters['pagination_offset'] :
+                parameters['pagination_limit']
+            ]
+        # return the result!
+        return parameters, query, count
+
+
+    def get(self, request):
+        """Displays the list of nodes corresponding to the query.
+        """
+        parameters, query, count = self._query(request)
+        return JsonHttpResponse({
+            'parameters': parameters,
+            'count': count,
+            'records': [dict(zip(parameters['fields'], node)) for node in query]
+        })
+
+    def delete(self, request):
+        """Removes the list of nodes corresponding to the query.
+        WARNING! THIS IS TOTALLY UNTESTED!!!!!
+        """
+        parameters, query, count = self._query(request)
+        for node in query:
+            node.delete()
+        session.commit()
+        return JsonHttpResponse({
+            'parameters': parameters,
+            'count': count,
+        }, 200)

gargantext/views/api/urls.py

 from django.conf.urls import url
 
-# from . import main
+from . import nodes
 
 
 urlpatterns = [
-    # url(r'^$', main.home),
+    url(r'^nodes$', nodes.NodesList.as_view()),
 ]

gargantext/views/pages/corpora.py

+from gargantext.util.http import *
+from gargantext.util.db import *
+from gargantext.util.db_cache import cache
+from gargantext.models import *
+from gargantext.constants import *
+from gargantext.settings import *
+
+from datetime import datetime
+
+
+def _get_user_project_corpus(request, project_id, corpus_id):
+    """Helper method to get a corpus, knowing the project's and corpus' ID.
+    Raises HTTP errors when parameters (user, IDs...) are invalid.
+    """
+    user = cache.User[request.user.username]
+    project = session.query(Node).filter(Node.id == project_id).first()
+    corpus = session.query(Node).filter(Node.id == corpus_id).filter(Node.parent_id == project_id).first()
+    if corpus is None:
+        raise Http404()
+    if not user.owns(corpus):
+        raise HttpResponseForbidden()
+    return user, project, corpus
+
+
+@requires_auth
+def corpus(request, project_id, corpus_id):
+    user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)
+    # response!
+    return render(
+        template_name = 'pages/corpora/corpus.html',
+        request = request,
+        context = {
+            'debug': DEBUG,
+            'user': user,
+            'date': datetime.now(),
+            'project': project,
+            'corpus': corpus,
+            # 'processing': processing,
+            # 'number': number,
+            'view': 'documents'
+        },
+    )
+
+@requires_auth
+def chart(request, project_id, corpus_id):
+    user, project, corpus = _get_user_project_corpus(request, project_id, corpus_id)

gargantext/views/pages/urls.py

 from django.conf.urls import url
 
-from . import main, auth, projects
+from . import main, auth
+from . import projects, corpora
 
 
 urlpatterns = [
@@ -14,8 +15,12 @@ urlpatterns = [
     url(r'^auth/login/?$', auth.login),
     url(r'^auth/logout/?$', auth.logout),
 
-    # overview on projects
+    # projects
     url(r'^projects/?$', projects.overview),
     url(r'^projects/(\d+)/?$', projects.project),
 
+    # corpora
+    url(r'^projects/(\d+)/corpora/(\d+)?$', corpora.corpus),
+    url(r'^projects/(\d+)/corpora/(\d+)/chart?$', corpora.chart),
+
 ]

requirements.txt

@@ -10,6 +10,7 @@ dateparser==0.3.2
 django-celery==3.1.17
 django-pgfields==1.4.4
 django-pgjsonb==0.0.16
+djangorestframework==3.3.2
 html5lib==0.9999999
 jdatetime==1.7.2
 kombu==3.0.33

static/js/gargantext/Docs_dyna_chart_and_table.js

@@ -128,15 +128,14 @@ function Final_UpdateTable( action ) {
 var current_docs = {}
 var BIS_dict = {}
 
-var url_elems = window.location.href.split("/")
-var url_mainIDs = {}
-for(var i=0; i<url_elems.length; i++) {
-  // if the this element is a number:
-  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
-    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
-  }
-}
 
+var id_from_url = function(name) {
+    var regex = new RegExp(name + '/(\\d+)');
+    var result = regex.exec(location.href);
+    return result ? result[1] : null;
+};
+var project_id = id_from_url('projects');
+var corpus_id = id_from_url('corpora');
 
 
 
@@ -293,28 +292,35 @@ function Main_test( Data , SearchFilter ) {
     // console.log(Data[i]["date"]+"  :  originalRecords["+arr_id+"] <- "+orig_id+" | "+Data[i]["name"])
   }
 
-  var t0 = AjaxRecords[0].date.split("-").map(Number)
-  var t1 = AjaxRecords.slice(-1)[0].date.split("-").map(Number)
-  oldest = t0;
-  latest = t1;
-
-  TheBuffer = [new Date(t0[0],t0[1]-1,t0[2]), new Date(t1[0],t1[1]-1,t1[2] ) ];
+  var get_date = function(node) {
+      var hyperdata = node.hyperdata;
+      return new Date(
+          Number(hyperdata.publication_year),
+          Number(hyperdata.publication_month) - 1,
+          Number(hyperdata.publication_day)
+      );
+  };
+  var t0 = get_date(AjaxRecords[0]);
+  var t1 = get_date(AjaxRecords.slice(-1)[0]);
+  console.log(t0, t1)
+
+  TheBuffer = [t0, t1];
   TheBuffer[0] = new Date(TheBuffer[0].setDate(TheBuffer[0].getDate()-1) );
   TheBuffer[1] = new Date(TheBuffer[1].setDate(TheBuffer[1].getDate()+1) );
 
   var arrayd3 = []
-  for(var e in Data) {
-      var date = Data[e]["date"];
-      if(justdates[date]!=false) {
-          var info = {}
-          info.date = date
-          info.dd = dateFormat.parse(date)
-          info.month = d3.time.month(info.dd)
-          info.volume = justdates[date]
-          arrayd3.push(info)
+  $.each(Data, function(i, node) {
+      var date = node.hyperdata.publication_date;
+      if (justdates[date] != false) {
+          var info = {};
+          info.date = date;
+          info.dd = get_date(node);
+          info.month = d3.time.month(info.dd);
+          info.volume = justdates[date];
+          arrayd3.push(info);
           justdates[date] = false;
       }
-  }
+  });
 
   for(var i in justdates)
       delete justdates[i];
@@ -370,7 +376,7 @@ function Main_test( Data , SearchFilter ) {
           .valueAccessor(function (d) {
               return d.value.avg;
           })
-          .x(d3.time.scale().domain([new Date(t0[0],t0[1],t0[2]), new Date(t1[0],t1[1],t1[2])]))
+          .x(d3.time.scale().domain([t0, t1]))
           .round(d3.time.month.round)
           .xUnits(d3.time.months)
           .elasticY(true)
@@ -488,7 +494,7 @@ function SearchFilters( elem ) {
 
   if( MODE == "filter_dupl-titles") {
 
-    var getDupl_API = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
+    var getDupl_API = "/api/nodes/" + corpus_id + "/children/duplicates?keys=title&limit=9999"
     $.ajax({
       url: getDupl_API,
       success: function(data) {
@@ -523,16 +529,18 @@ $("#corpusdisplayer").hide()
 // FIRST portion of code to be EXECUTED:
 // (3) Get records and hyperdata for paginator
 $.ajax({
-  url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
+  url: '/api/nodes?type[]=DOCUMENT&parent_id=' + corpus_id,
   success: function(data){
-    $("#content_loader").remove()
-    for(var i in data.records) {
-      var orig_id = parseInt(data.records[i].id)
+    $("#content_loader").remove();
+    $.each(data.records, function(i, record){
+      var orig_id = parseInt(record.id);
       var arr_id = parseInt(i)
       RecDict[orig_id] = arr_id;
-      data.records[i]["title"] = data.records[i]["name"];
-      data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
-      data.records[i]["del"] = false
+      record.title = record.name;
+      record.name = '<a target="_blank" href="/projects/' + project_id + '/corpora/'+ corpus_id + '/documents/' + record.id + '">' + record.name + '</a>';
+      record.del = false;
+    });
+    for (var i in data.records) {
     }
     AjaxRecords = data.records; // backup-ing in global variable!
 

templates/pages/corpora/corpus.html

+{% extends "pages/menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+
+<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<!-- here goes import stylesheet js/bootstrap/bootstrap-multiselect.css, mais ca marche pas-->
+
+<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
+
+<link rel="stylesheet" type="text/css" href="{% static "css/d3/dc.css"%}"/>
+
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.easy-pie-chart.css"%}">
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery/jquery.dynatable.css"%}"/>
+
+<script type="text/javascript" src="{% static "js/d3/d3.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/dc.js"%}"></script>
+<script type="text/javascript" src="{% static "js/d3/crossfilter.js"%}"></script>
+
+<style>
+.no-transition {
+  -webkit-transition: height 0.1s;
+  -moz-transition: height 0.1s;
+  -ms-transition: height 0.1s;
+  -o-transition: height 0.1s;
+  transition: height 0.1s;
+}
+th { color: #fff; }
+
+th a {
+  color: #fff;
+  font-weight: normal;
+  font-style: italic;
+  font-size: 0.9em;
+}
+
+.dynatable-record-count {
+  font-size: 0.7em;
+}
+.dynatable-pagination-links {
+  font-size: 0.7em;
+}
+
+</style>
+
+{% endblock %}
+
+
+
+{% block content %}
+
+<div class="container">
+    <div class="container">
+        <div class="jumbotron">
+            <div class="row">
+                <div id="monthly-move-chart">
+                    <center>
+                        Select a time range in the chart with blue bars to zoom in
+                    <p align="center">
+                    <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a>
+                    <a class="btn btn-xs btn-default" href="javascript:volumeChart.filterAll();dc.redrawAll();">Reset</a></p>
+
+                    <div class="clearfix"></div>
+                    </center>
+                </div>
+            </div>
+        <div class="row">
+            <div id="monthly-volume-chart"></div>
+        </div>
+
+        <div id="content_loader">
+          <br>
+          <center>
+            <img width="10%" src="{% static "img/ajax-loader.gif"%}"></img>
+          </center>
+          <br>
+        </div>
+
+    </div>
+</div>
+
+
+<input type="hidden" id="list_id" value="{{ list_id }}"></input>
+
+<div class="container">
+
+
+    <div class="jumbotron">
+        <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+            <div class="panel panel-body">
+                <div class="panel-heading">
+                    <h4 class="panel-title">
+                        <a data-toggle="collapse" data-parent="#accordion" href="#collapseOne">
+                        <p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg" style="width:200px; margin:0 auto; display:block;">Open Folder</h2></p>
+                        </a>
+                    </h4>
+                </div>
+                <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
+                  <div class="panel-body">
+                    <div id="div-table"></div>
+                            <p align="right">
+                                <button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button>
+                            </p>
+
+                  </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+<div id="filter_search" style="visibility:hidden">
+    <span style="font-size:70%;">
+      <input title="Search in Titles" id="searchTI" name="searchTI" type="checkbox" checked onclick="return false">TI&nbsp;
+      <input title="Search in Abstracts" id="searchAB" name="searchAB" type="checkbox">AB
+    </span>&nbsp;&nbsp;
+
+    <select id="example-single-optgroups" onchange="SearchFilters(this);">
+        <!-- <optgroup label=""> -->
+            <option id="filter_all" value="filter_all">All</option>
+            <!-- <option id="filter_title" value="filter_title">Title</option> -->
+            <!-- <option id="filter_date" value="filter_date">Date</option> -->
+        <!-- </optgroup> -->
+        <!-- <optgroup label="Duplicates"> -->
+            <!-- <option value="filter_doi">By DOI</option> -->
+            <option id="filter_dupl-titles" value="filter_dupl-titles">Duplicates by Title</option>
+        <!-- </optgroup> -->
+    </select>
+
+</div>
+
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script src="{% static "js/bootstrap/bootstrap.min.js" %}"></script>
+<!-- here goes import script js/bootstrap/bootstrap-multiselect.js, mais ca marche pas-->
+<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
+
+<!-- custom-lib for dynatable.js and dc.js -->
+<script type="text/javascript" src="{% static "js/gargantext/Docs_dyna_chart_and_table.js" %}"></script>
+
+
+
+{% endblock %}