[FEAT] implemented lists

[FEAT] linked nodes with links (cf. `gargantext.constants` and `gargantext.models.Node.as_list()`
[FEAT] implemented an `ngrams` pseudo-attribute on `gargantext.models.Node` (available from REST api)
[FEAT] on REST API, renamed GET parameter `type[]` to `types[]`
[FEAT] `json_dumps()` can even serialize SQLAlchemy models and models now

gargantext/constants.py

 # WARNING: to ensure consistency and retrocompatibility, lists should keep the
 #   initial order (ie., new elements should be appended at the end of the lists)
 
+from gargantext.util.lists import *
+
+LISTTYPES = {
+    'DOCUMENT': WeightedList,
+    'SYNONYMS': Translations,
+    'MIAMLIST': UnweightedList,
+    'STOPLIST': UnweightedList,
+    'COOCCURRENCES': WeightedMatrix,
+}
+
 NODETYPES = [
     None,
+    # documents hierachy
     'USER',
     'PROJECT',
     'CORPUS',
     'DOCUMENT',
+    # lists
+    'SYNONYMS',
+    'MIAMLIST',
+    'STOPLIST',
+    'COOCCURRENCES',
 ]
 
 

gargantext/models/ngrams.py

 from gargantext.util.db import *
-from gargantext.util.files import upload
-from gargantext.constants import *
 
 from .nodes import Node
 
-__all__ = ['Ngram', 'NodeNgram']
+__all__ = ['Ngram', 'NodeNgram', 'NodeNgramNgram']
 
 
 class Ngram(Base):
@@ -19,3 +17,11 @@ class NodeNgram(Base):
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float)
+
+
+class NodeNgramNgram(Base):
+    __tablename__ = 'nodes_ngrams_ngrams'
+    node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
+    ngram1_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
+    ngram2_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
+    weight = Column(Float)

gargantext/models/nodes.py

@@ -45,6 +45,26 @@ class Node(Base):
     def __setitem__(self, key, value):
         self.hyperdata[key] = value
 
+    @property
+    def ngrams(self):
+        from . import NodeNgram, Ngram
+        query = (session
+            .query(NodeNgram.weight, Ngram)
+            .select_from(NodeNgram)
+            .join(Ngram)
+            .filter(NodeNgram.node_id == self.id)
+        )
+        return query
+
+    def as_list(self):
+        try:
+            return LISTTYPES[self.typename](self.id)
+        except KeyError:
+            raise ValueError('This node\'s typename is not convertible to a list: %s (accepted values: %s)' % (
+                self.typename,
+                ', '.join(LISTTYPES.keys())
+            ))
+
     def save_hyperdata(self):
         """This is a necessary, yet ugly trick.
         Indeed, PostgreSQL does not yet manage incremental updates (see

gargantext/models/users.py

@@ -9,7 +9,7 @@ class User(Base):
 
     # Do not change!
     # The properties below are a reflection of Django's auth module's models.
-    __tablename__ = 'auth_user'
+    __tablename__ = models.User._meta.db_table
     id = Column(Integer, primary_key=True)
     password = Column(String(128))
     last_login = DateTime(timezone=False)
@@ -34,16 +34,14 @@ class User(Base):
 
     def nodes(self, typename=None):
         """get all nodes belonging to the user"""
-        # ↓ this below is a workaround because of Python's lame import system
         from .nodes import Node
         query = (session
             .query(Node)
             .filter(Node.user_id == self.id)
-            .order_by(Node.date)
         )
         if typename is not None:
             query = query.filter(Node.typename == typename)
-        return query.all()
+        return query
 
     def contacts_nodes(self, typename=None):
         for contact in self.contacts():

gargantext/util/json.py

 import json
+import types
 import datetime
 import traceback
+import inspect
+
+
+__all__ = ['json_encoder', 'json_dumps']
 
 
 class JSONEncoder(json.JSONEncoder):
     def default(self, obj):
-        if isinstance(obj, datetime.datetime):
+        from gargantext.util.db import Base
+        if isinstance(obj, Base):
+            return {
+                key: value
+                for key, value in obj.__dict__.items()
+                if not key.startswith('_')
+            }
+        elif isinstance(obj, datetime.datetime):
             return obj.isoformat()[:19] + 'Z'
-        elif isinstance(obj, (set, tuple)):
-            return list(obj)
         elif isinstance(obj, Exception):
             tbe = traceback.TracebackException.from_exception(obj)
             return list(line.strip() for line in tbe.format())
+        elif hasattr(obj, '__iter__') and not isinstance(obj, dict):
+            return list(obj)
         else:
             return super(self.__class__, self).default(obj)
 

gargantext/views/api/nodes.py

@@ -7,71 +7,80 @@ from gargantext.constants import *
 from gargantext.util.validation import validate
 
 
-class NodeListResource(APIView):
+_node_available_fields = ['id', 'parent_id', 'name', 'typename', 'hyperdata', 'ngrams']
+_node_default_fields = ['id', 'parent_id', 'name', 'typename']
+_node_available_types = NODETYPES
 
-    _fields = ['id', 'parent_id', 'name', 'typename', 'hyperdata']
-    _types = NODETYPES
 
-    def _query(self, request):
-        # parameters validation
-        parameters = get_parameters(request)
-        parameters = validate(parameters, {'type': dict, 'items': {
-            'pagination_limit': {'type': int, 'default': 10},
-            'pagination_offset': {'type': int, 'default': 0},
-            'fields': {'type': list, 'default': self._fields, 'items': {
-                'type': str, 'range': self._fields,
-            }},
-            # optional filtering parameters
-            'type': {'type': list, 'default': self._types, 'required': False, 'items': {
-                'type': str, 'range': self._types,
-            }},
-            'parent_id': {'type': int, 'required': False},
-        }})
-        # start the query
-        query = session.query(*tuple(
-            getattr(Node, field) for field in parameters['fields']
-        ))
-        # filter by type
-        if 'type' in parameters:
-            query = query.filter(Node.typename.in_(parameters['type']))
-        # filter by parent
-        if 'parent_id' in parameters:
-            query = query.filter(Node.parent_id == parameters['parent_id'])
-        # count
-        count = query.count()
-        # order
-        query = query.order_by(Node.hyperdata['publication_date'], Node.id)
-        # paginate the query
-        if parameters['pagination_limit'] == -1:
-            query = query[parameters['pagination_offset']:]
-        else:
-            query = query[
-                parameters['pagination_offset'] :
-                parameters['pagination_limit']
-            ]
-        # return the result!
-        return parameters, query, count
+def _query_nodes(request, node_id=None):
+    user = cache.User[request.user.username]
+    # parameters validation
+    parameters = get_parameters(request)
+    parameters = validate(parameters, {'type': dict, 'items': {
+        'pagination_limit': {'type': int, 'default': 10},
+        'pagination_offset': {'type': int, 'default': 0},
+        'fields': {'type': list, 'default': _node_default_fields, 'items': {
+            'type': str, 'range': _node_available_fields,
+        }},
+        # optional filtering parameters
+        'types': {'type': list, 'required': False, 'items': {
+            'type': str, 'range': _node_available_types,
+        }},
+        'parent_id': {'type': int, 'required': False},
+    }})
+    # start the query
+    query = user.nodes()
+    # filter by id
+    if node_id is not None:
+        query = query.filter(Node.id == node_id)
+    # filter by type
+    if 'types' in parameters:
+        query = query.filter(Node.typename.in_(parameters['types']))
+    # filter by parent
+    if 'parent_id' in parameters:
+        query = query.filter(Node.parent_id == parameters['parent_id'])
+    # count
+    count = query.count()
+    # order
+    query = query.order_by(Node.hyperdata['publication_date'], Node.id)
+    # paginate the query
+    if parameters['pagination_limit'] == -1:
+        query = query[parameters['pagination_offset']:]
+    else:
+        query = query[
+            parameters['pagination_offset'] :
+            parameters['pagination_limit']
+        ]
+    # return the result!
+    return parameters, query, count
+
+
+class NodeListResource(APIView):
 
     def get(self, request):
         """Displays the list of nodes corresponding to the query.
         """
-        parameters, query, count = self._query(request)
+        parameters, query, count = _query_nodes(request)
         return JsonHttpResponse({
             'parameters': parameters,
             'count': count,
-            'records': [dict(zip(parameters['fields'], node)) for node in query]
+            'records': [
+                {field: getattr(node, field) for field in parameters['fields']}
+                for node in query
+            ]
         })
 
     def post(self, request):
-        pass
+        """Create a new node.
+        NOT IMPLEMENTED
+        """
 
     def delete(self, request):
         """Removes the list of nodes corresponding to the query.
         WARNING! THIS IS TOTALLY UNTESTED!!!!!
         """
-        parameters, query, count = self._query(request)
-        for node in query:
-            node.delete()
+        parameters, query, count = _query_nodes(request)
+        query.delete()
         session.commit()
         return JsonHttpResponse({
             'parameters': parameters,
@@ -81,27 +90,20 @@ class NodeListResource(APIView):
 
 class NodeResource(APIView):
 
-    def _query(self, request, node_id):
-        user = cache.User[request.user.username]
-        node = session.query(Node).filter(Node.id == node_id).first()
-        if node is None:
-            raise Http404()
-        if not user.owns(node):
-            raise HttpResponseForbidden()
-        return user, node
-
     def get(self, request, node_id):
-        user, node = self._query(request, node_id)
+        parameters, query, count = _query_nodes(request, node_id)
+        if not len(query):
+            raise Http404()
+        node = query[0]
         return JsonHttpResponse({
-            'id': node.id,
-            'parent_id': node.parent_id,
-            'name': node.name,
-            'hyperdata': node.hyperdata,
+            field: getattr(node, field) for field in parameters['fields']
         })
 
     def delete(self, request, node_id):
+        parameters, query, count = _query_nodes(request, node_id)
+        if not len(query):
+            raise Http404()
         from sqlalchemy import delete
-        user, node = self._query(request, node_id)
         result = session.execute(
             delete(Node).where(Node.id == node_id)
         )

static/js/gargantext/Docs_dyna_chart_and_table.js

@@ -532,7 +532,7 @@ $("#corpusdisplayer").hide()
 // FIRST portion of code to be EXECUTED:
 // (3) Get records and hyperdata for paginator
 $.ajax({
-  url: '/api/nodes?type[]=DOCUMENT&pagination_limit=-1&parent_id=' + corpus_id,
+  url: '/api/nodes?types[]=DOCUMENT&pagination_limit=-1&parent_id=' + corpus_id,
   success: function(data){
     $("#content_loader").remove();
     $.each(data.records, function(i, record){