[WIP] task & auth

gargantext/backend/authentication.py

+import logging
+from django.contrib.auth.signals import user_logged_in
+from django.dispatch import receiver
+from rest_framework_jwt.authentication import JSONWebTokenAuthentication
+
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(user_logged_in)
+def login_handler(sender, user, request, **kwargs):
+    logger.debug("%r logged in." % user)
+    request.db.login(user)
+
+
+class JWTAuthentication(JSONWebTokenAuthentication):
+    def authenticate(self, request):
+        result = super().authenticate(request)
+
+        if result is None:
+            return
+
+        user, token = result
+        user_logged_in.send(sender=user.__class__, request=request, user=user)
+
+        return (user, token)

gargantext/backend/jwt.py

+from calendar import timegm
 from django.conf import settings
 from rest_framework_jwt.settings import api_settings
-from calendar import timegm
+from gargantext.utils.dates import datetime
 
-from gargantext.util.dates import datetime
 
 def jwt_payload_handler(user):
     username = user.username
@@ -31,3 +31,10 @@ def jwt_payload_handler(user):
 
     return payload
 
+
+def jwt_get_user_id_from_payload_handler(payload):
+    return payload.get('user_id')
+
+
+def jwt_get_username_from_payload_handler(payload):
+    return payload.get('sub')

gargantext/backend/tasks.py

+import time
+from django.conf import settings
+from rest_framework.decorators import api_view, renderer_classes
+from rest_framework.response import Response
+from rest_framework.renderers import JSONRenderer
+from gargantext.core.task import shared_task, get_task_logger, schedule
+from gargantext.models import UserNode
+
+
+logger = get_task_logger(__name__)
+
+
+@shared_task(bind=True)
+def test(self, duration=30):
+    logger.info('Start %r task (DEBUG=%r): wait %s seconds...' % (
+        self.name, settings.DEBUG, duration))
+    time.sleep(duration)
+    logger.info('End task %r.' % self.name)
+
+
+@api_view(['GET'])
+@renderer_classes((JSONRenderer,))
+def dummy_task(request):
+    schedule(test, args=[15])
+    return Response({
+        'me': request.db.query(UserNode).filter_by(user_id=request.user.id).one().name
+    })

gargantext/backend/urls.py

@@ -18,10 +18,14 @@ from django.contrib import admin
 from django.views.generic.base          import RedirectView        as Redirect
 from django.contrib.staticfiles.storage import staticfiles_storage as static
 from rest_framework_jwt.views import obtain_jwt_token
+from .tasks import dummy_task
+from .views import projects_view
 
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^favicon.ico$', Redirect.as_view(url=static.url('favicon.ico'),
                                            permanent=False), name="favicon"),
     url(r'^api/auth/token$', obtain_jwt_token),
+    url(r'^projects$', projects_view),
+    url(r'^dummy$', dummy_task),
 ]

gargantext/backend/views.py

+import logging
+from sqlalchemy.exc import DBAPIError as DatabaseError
+from sqlalchemy.orm.attributes import ScalarObjectAttributeImpl
+from rest_framework.decorators import api_view, renderer_classes
+from rest_framework.response import Response
+from rest_framework.renderers import JSONRenderer
+from gargantext.models import ProjectNode
+
+
+logger = logging.getLogger(__name__)
+
+
+# https://bitbucket.org/zzzeek/sqlalchemy/issues/3976/built-in-way-to-convert-an-orm-object-ie
+from sqlalchemy import inspect
+def orm_is_value(a):
+    return hasattr(a, 'key') and \
+        not isinstance(getattr(a, 'impl'), ScalarObjectAttributeImpl)
+def orm_to_dict(obj):
+    attrs = inspect(type(obj)).all_orm_descriptors
+    return {a.key: getattr(obj, a.key) for a in attrs if orm_is_value(a)}
+
+
+@api_view(['GET'])
+@renderer_classes((JSONRenderer,))
+def projects_view(request):
+    try:
+        r = list(map(orm_to_dict, request.db.query(ProjectNode).all()))
+    except DatabaseError as e:
+        logger.debug("Error: %s" % e)
+        r = []
+
+    return Response({'results': r})

gargantext/constants.py

@@ -35,9 +35,10 @@ contents:
 import os
 import re
 import importlib
-from gargantext.util.lists import *
-from gargantext.util import datetime, convert_to_datetime
 from django.conf import settings
+from gargantext.util.lists import *
+from gargantext.utils.dates import datetime, convert_to_datetime
+
 
 # types & models (nodes, lists, hyperdata, resource) ---------------------------------------------
 LISTTYPES = {

gargantext/util/db.py → gargantext/core/db.py

+import logging
+import psycopg2
+import sqlalchemy
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy.orm.session import Session
 from django.conf import settings
-from gargantext.util.json import json_dumps
+from gargantext.utils.json import json_dumps
 
 
-########################################################################
-# get engine, session, etc.
-########################################################################
-from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy import delete
-
-def get_engine():
-    from sqlalchemy import create_engine
-    return create_engine( settings.DATABASES['default']['SECRET_URL']
-                        , use_native_hstore = True
-                        , json_serializer = json_dumps
-                        , pool_size=20, max_overflow=0
-    )
-
-engine = get_engine()
-
-session = scoped_session(sessionmaker(bind=engine))
-
-########################################################################
-# useful for queries
-########################################################################
-from sqlalchemy.orm import aliased
-from sqlalchemy import func, desc
-from sqlalchemy.sql.expression import case
-
-########################################################################
-# bulk insertions
-########################################################################
-import psycopg2
+__all__ = ['session']
+
+
+logger = logging.getLogger(__name__)
+
+
+class ProtectedSession(Session):
+    def close(self):
+        self._logout()
+        super().close()
+
+    logout = close
+
+    def login(self, user):
+        if settings.DEBUG and isinstance(user, str):
+            # For debugging purposes
+            from gargantext.models import User
+            from sqlalchemy.orm.exc import NoResultFound
+            username = user
+            try:
+                user = session.query(User).filter_by(username=username).one()
+            except NoResultFound:
+                raise Exception("User %s not found!" % username)
+
+        self.user_id = user and user.id
+        self.user_name = user and user.username
+        self.role = settings.ROLE_SUPERUSER if user.is_superuser else \
+                    settings.ROLE_STAFF if user.is_staff else \
+                    settings.ROLE_USER if user.id else \
+                    'anon'
+
+        logger.debug("Plug authenticator for: %s (%s, %s)" % (
+            self.role, self.user_name, self.user_id))
+
+        sqlalchemy.event.listen(self, 'after_begin', self._postgres_auth)
+
+    def _logout(self):
+        if sqlalchemy.event.contains(self, 'after_begin', self._postgres_auth):
+            logger.debug("Unplug authenticator for: %s (%s, %s)" % (
+                self.role, self.user_name, self.user_id))
+            sqlalchemy.event.remove(self, 'after_begin', self._postgres_auth)
+
+        self.user_id = self.user_name = self.role = None
 
+    def _postgres_auth(self, *args, **kwargs):
+        logger.debug("Authenticate in postgres as %s (%s, %s)" % (
+            self.role, self.user_name, self.user_id))
+
+        ops = [('set role %s', self.role),
+               ('set local "request.jwt.claim.role" = \'%s\'', self.role),
+               ('set local "request.jwt.claim.sub" = %r', self.user_name),
+               ('set local "request.jwt.claim.user_id" = %d', self.user_id)]
+
+        sql = ';'.join(s % p for s, p in ops if p) + ';'
+
+        self.connection().execute(sql)
+
+
+# FIXME Keeped for backward compatibility but should be removed
+engine = create_engine(settings.DATABASES['default']['SECRET_URL'])
+session = scoped_session(sessionmaker(
+    bind=engine.execution_options(isolation_level='READ COMMITTED'),
+    autoflush=False))
+
+
+protected_engine = create_engine(
+    settings.DATABASES['protected']['SECRET_URL'],
+    use_native_hstore=True,
+    json_serializer=json_dumps,
+    pool_size=20,
+    max_overflow=0)
+
+
+Session = sessionmaker(
+    class_=ProtectedSession,
+    # This is the default postgresql isolation level, we state it here
+    # to be explicit
+    bind=protected_engine.execution_options(isolation_level='READ COMMITTED'),
+    # Disable autoflush to have more control over transactions
+    autoflush=False)
+
+
+# FIXME Should rewrite bulk queries with SQLAlchemy Core.
+# See: http://docs.sqlalchemy.org/en/latest/faq/performance.html#i-m-inserting-400-000-rows-with-the-orm-and-it-s-really-slow
 def get_cursor():
     db_settings = settings.DATABASES['default']
     db = psycopg2.connect(**{
@@ -43,6 +103,7 @@ def get_cursor():
     })
     return db, db.cursor()
 
+
 class bulk_insert:
     def __init__(self, table, fields, data, cursor=None):
         # prepare the iterator
@@ -74,6 +135,7 @@ class bulk_insert:
 
     readline = read
 
+
 def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stats=False):
     """
     Inserts bulk data with an intermediate check on a uniquekey
@@ -157,5 +219,3 @@ def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stat
     cursor.execute('COMMIT WORK;')
 
     cursor.close()
-
-

gargantext/core/task.py

+from celery import shared_task
+from celery.utils.log import get_task_logger
+
+
+__all__ = ['shared_task', 'get_task_logger', 'schedule']
+
+
+def schedule(task, when=None, args=None, kwargs=None):
+    task.apply_async(args=args, kwargs=kwargs)

gargantext/middleware.py

+from gargantext.core.db import Session
+
+
+class DatabaseSessionMiddleware(object):
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        session = Session()
+
+        try:
+            request.db = session
+            response = self.get_response(request)
+            session.commit()
+        except:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+        return response

gargantext/models/nodes.py

-from gargantext.util.db import session
+from gargantext.core.db import session
 from gargantext.constants import NODETYPES, LISTTYPES
 
 from datetime import datetime
@@ -8,7 +8,7 @@ from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index,
                   MutableList, MutableDict, validates, ValidatorMixin, text
 from .users import User
 
-__all__ = ['Node', 'NodeNode', 'CorpusNode']
+__all__ = ['Node', 'NodeNode', 'CorpusNode', 'ProjectNode']
 
 class NodeType(TypeDecorator):
     """Define a new type of column to describe a Node's type.
@@ -39,7 +39,7 @@ class Node(ValidatorMixin, Base):
     <Node(id=None, typename=None, user_id=None, parent_id=None, name='without-type', date=None)>
     >>> Node(typename='CORPUS')
     <CorpusNode(id=None, typename='CORPUS', user_id=None, parent_id=None, name=None, date=None)>
-    >>> from gargantext.util.db import session
+    >>> from gargantext.core.db import g_session as session
     >>> session.query(Node).filter_by(typename='USER').first() # doctest: +ELLIPSIS
     <UserNode(...)>
 
@@ -221,6 +221,18 @@ class Node(ValidatorMixin, Base):
         return self['statuses'][-1]
 
 
+class UserNode(Node):
+    __mapper_args__ = {
+        'polymorphic_identity': 'USER'
+    }
+
+
+class ProjectNode(Node):
+    __mapper_args__ = {
+        'polymorphic_identity': 'PROJECT'
+    }
+
+
 class CorpusNode(Node):
     __mapper_args__ = {
         'polymorphic_identity': 'CORPUS'

gargantext/models/users.py

+from sqlalchemy.orm import aliased
 from django.contrib.auth import models
-from gargantext.util.db import session, aliased
+from gargantext.core.db import session
 
 from datetime import datetime
 
@@ -8,6 +9,7 @@ from .base import DjangoBase, Base, Column, ForeignKey, UniqueConstraint, \
 
 __all__ = ['User', 'Contact']
 
+
 class User(DjangoBase):
     # The properties below are a reflection of Django's auth module's models.
     __tablename__ = models.User._meta.db_table

gargantext/settings.py

@@ -11,8 +11,8 @@ https://docs.djangoproject.com/en/1.11/ref/settings/
 """
 
 import os
-from gargantext.util.config import config
 import datetime
+from gargantext.utils.config import config
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -53,7 +53,7 @@ MIDDLEWARE = [
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'gargantext.middleware.DatabaseSessionMiddleware',
 ]
 
 ROOT_URLCONF = 'gargantext.backend.urls'
@@ -105,12 +105,23 @@ LOGGING = {
             'level': LOG_LEVEL,
             'propagate': True,
         },
+        # All django loggers: https://docs.djangoproject.com/fr/1.11/topics/logging/#id3
         'django.template': {
             # Don't keep debug logs for template module to avoid annoying and
             # useless noise, see:
             # https://github.com/encode/django-rest-framework/issues/3982#issuecomment-325290221
             'level': 'INFO' if LOG_LEVEL == 'DEBUG' else LOG_LEVEL,
         },
+        'django.db.backends': {
+            'level': 'INFO',
+        },
+        'gargantext': {
+            'handlers': ['file'],
+            'level': LOG_LEVEL,
+            # Propagation to True means that this config applies to
+            # 'gargantext' logger and all 'gargantext.*'
+            'propagate': True,
+        },
     },
 }
 
@@ -118,22 +129,24 @@ LOGGING = {
 # Database
 # https://docs.djangoproject.com/en/1.11/ref/settings/#databases
 
-DATABASES = {
-    'default': {
+DEFAULT_DATABASE = {
     'ENGINE': 'django.db.backends.postgresql_psycopg2',
     'NAME': config('DB_NAME', default='gargandb'),
     'USER': config('DB_USER', default='gargantua'),
     'PASSWORD': config('DB_PASS'),
     'HOST': config('DB_HOST', default='127.0.0.1'),
     'PORT': config('DB_PORT', default='5432'),
-        'TEST': {
-            'NAME': 'test_gargandb',
-        },
-    }
 }
-DATABASES['default']['SECRET_URL'] = \
+DATABASES = {
+    'default': DEFAULT_DATABASE,
+    'protected': dict(DEFAULT_DATABASE,
+        USER=config('DB_PROTECTED_USER', default='authenticator'),
+        PASSWORD=config('DB_PROTECTED_PASS'))
+}
+for db in DATABASES:
+    DATABASES[db]['SECRET_URL'] = \
         'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format(
-        **DATABASES['default']
+            **DATABASES[db]
         )
 
 
@@ -211,19 +224,23 @@ REST_FRAMEWORK = {
         'rest_framework.permissions.IsAuthenticated',
     ),
     'DEFAULT_AUTHENTICATION_CLASSES': (
-        'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
+        'gargantext.backend.authentication.JWTAuthentication',
         'rest_framework.authentication.SessionAuthentication',
-        'rest_framework.authentication.BasicAuthentication',
     ),
 }
 
 # See http://getblimp.github.io/django-rest-framework-jwt/
 JWT_AUTH = {
     'JWT_PAYLOAD_HANDLER': 'gargantext.backend.jwt.jwt_payload_handler',
+    'JWT_PAYLOAD_GET_USER_ID_HANDLER':
+        'gargantext.backend.jwt.jwt_get_user_id_from_payload_handler',
+    'JWT_PAYLOAD_GET_USERNAME_HANDLER':
+        'gargantext.backend.jwt.jwt_get_username_from_payload_handler',
     'JWT_VERIFY_EXPIRATION': True,
     'JWT_SECRET_KEY': config('SECRET_KEY'),
     'JWT_EXPIRATION_DELTA': datetime.timedelta(seconds=36000),
     'JWT_AUTH_HEADER_PREFIX': 'Bearer',
+    'JWT_AUTH_COOKIE': 'JWT' if DEBUG else None,
 }
 
 ROLE_SUPERUSER = 'gargantua'

gargantext/util/__init__.py

-from .dates import datetime, convert_to_datetime, MINYEAR

gargantext/util/lists.py

@@ -2,15 +2,13 @@
 """
 
 
-__all__ = ['Translations', 'WeightedMatrix', 'UnweightedList', 'WeightedList', 'WeightedIndex']
-
-
-from gargantext.util.db import session, bulk_insert
-
 from collections import defaultdict
 from math import sqrt
+from gargantext.core.db import session, bulk_insert
 
 
+__all__ = ['Translations', 'WeightedMatrix', 'UnweightedList', 'WeightedList', 'WeightedIndex']
+
 
 class _BaseClass:
 
@@ -303,6 +301,7 @@ class WeightedMatrix(_BaseClass):
                 result.items[key1, key2] = value / sqrt(other.items[key1] * other.items[key2])
         return result
 
+
 # ?TODO rename Wordlist
 class UnweightedList(_BaseClass):
 

gargantext/util/show_nodes.py

@@ -13,8 +13,10 @@ import itertools
 import colorama
 from colorama import Fore
 from sqlalchemy.sql.expression import literal_column
+from sqlalchemy.orm import aliased
+from sqlalchemy import func
 
-from gargantext.util.db import session, func, aliased
+from gargantext.core.db import g_session as session
 from gargantext.models import Node
 
 

gargantext/util/json.py → gargantext/utils/json.py

 import json
-import types
 import datetime
 import traceback
-import inspect
 
 
 __all__ = ['json_encoder', 'json_dumps']
@@ -25,10 +23,11 @@ class JSONEncoder(json.JSONEncoder):
         elif hasattr(obj, '__iter__') and not isinstance(obj, dict):
             return list(obj)
         else:
-            return super(self.__class__, self).default(obj)
+            return super().default(obj)
+
+
+json_encoder = JSONEncoder()
 
-# json_encoder = JSONEncoder(indent=4)
-json_encoder = JSONEncoder()   # compact json
 
 def json_dumps(obj):
     return json.dumps(obj, cls=JSONEncoder)