Commit 12bdce3e authored by sim's avatar sim

[WIP] task & auth

parent ceb2c62b
import logging
from django.contrib.auth.signals import user_logged_in
from django.dispatch import receiver
from rest_framework_jwt.authentication import JSONWebTokenAuthentication
logger = logging.getLogger(__name__)
@receiver(user_logged_in)
def login_handler(sender, user, request, **kwargs):
logger.debug("%r logged in." % user)
request.db.login(user)
class JWTAuthentication(JSONWebTokenAuthentication):
def authenticate(self, request):
result = super().authenticate(request)
if result is None:
return
user, token = result
user_logged_in.send(sender=user.__class__, request=request, user=user)
return (user, token)
from calendar import timegm
from django.conf import settings from django.conf import settings
from rest_framework_jwt.settings import api_settings from rest_framework_jwt.settings import api_settings
from calendar import timegm from gargantext.utils.dates import datetime
from gargantext.util.dates import datetime
def jwt_payload_handler(user): def jwt_payload_handler(user):
username = user.username username = user.username
...@@ -31,3 +31,10 @@ def jwt_payload_handler(user): ...@@ -31,3 +31,10 @@ def jwt_payload_handler(user):
return payload return payload
def jwt_get_user_id_from_payload_handler(payload):
return payload.get('user_id')
def jwt_get_username_from_payload_handler(payload):
return payload.get('sub')
import time
from django.conf import settings
from rest_framework.decorators import api_view, renderer_classes
from rest_framework.response import Response
from rest_framework.renderers import JSONRenderer
from gargantext.core.task import shared_task, get_task_logger, schedule
from gargantext.models import UserNode
logger = get_task_logger(__name__)
@shared_task(bind=True)
def test(self, duration=30):
logger.info('Start %r task (DEBUG=%r): wait %s seconds...' % (
self.name, settings.DEBUG, duration))
time.sleep(duration)
logger.info('End task %r.' % self.name)
@api_view(['GET'])
@renderer_classes((JSONRenderer,))
def dummy_task(request):
schedule(test, args=[15])
return Response({
'me': request.db.query(UserNode).filter_by(user_id=request.user.id).one().name
})
...@@ -18,10 +18,14 @@ from django.contrib import admin ...@@ -18,10 +18,14 @@ from django.contrib import admin
from django.views.generic.base import RedirectView as Redirect from django.views.generic.base import RedirectView as Redirect
from django.contrib.staticfiles.storage import staticfiles_storage as static from django.contrib.staticfiles.storage import staticfiles_storage as static
from rest_framework_jwt.views import obtain_jwt_token from rest_framework_jwt.views import obtain_jwt_token
from .tasks import dummy_task
from .views import projects_view
urlpatterns = [ urlpatterns = [
url(r'^admin/', admin.site.urls), url(r'^admin/', admin.site.urls),
url(r'^favicon.ico$', Redirect.as_view(url=static.url('favicon.ico'), url(r'^favicon.ico$', Redirect.as_view(url=static.url('favicon.ico'),
permanent=False), name="favicon"), permanent=False), name="favicon"),
url(r'^api/auth/token$', obtain_jwt_token), url(r'^api/auth/token$', obtain_jwt_token),
url(r'^projects$', projects_view),
url(r'^dummy$', dummy_task),
] ]
import logging
from sqlalchemy.exc import DBAPIError as DatabaseError
from sqlalchemy.orm.attributes import ScalarObjectAttributeImpl
from rest_framework.decorators import api_view, renderer_classes
from rest_framework.response import Response
from rest_framework.renderers import JSONRenderer
from gargantext.models import ProjectNode
logger = logging.getLogger(__name__)
# https://bitbucket.org/zzzeek/sqlalchemy/issues/3976/built-in-way-to-convert-an-orm-object-ie
from sqlalchemy import inspect
def orm_is_value(a):
return hasattr(a, 'key') and \
not isinstance(getattr(a, 'impl'), ScalarObjectAttributeImpl)
def orm_to_dict(obj):
attrs = inspect(type(obj)).all_orm_descriptors
return {a.key: getattr(obj, a.key) for a in attrs if orm_is_value(a)}
@api_view(['GET'])
@renderer_classes((JSONRenderer,))
def projects_view(request):
try:
r = list(map(orm_to_dict, request.db.query(ProjectNode).all()))
except DatabaseError as e:
logger.debug("Error: %s" % e)
r = []
return Response({'results': r})
...@@ -35,9 +35,10 @@ contents: ...@@ -35,9 +35,10 @@ contents:
import os import os
import re import re
import importlib import importlib
from gargantext.util.lists import *
from gargantext.util import datetime, convert_to_datetime
from django.conf import settings from django.conf import settings
from gargantext.util.lists import *
from gargantext.utils.dates import datetime, convert_to_datetime
# types & models (nodes, lists, hyperdata, resource) --------------------------------------------- # types & models (nodes, lists, hyperdata, resource) ---------------------------------------------
LISTTYPES = { LISTTYPES = {
......
from django.conf import settings import logging
from gargantext.util.json import json_dumps import psycopg2
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.orm.session import Session
from django.conf import settings
from gargantext.utils.json import json_dumps
######################################################################## __all__ = ['session']
# get engine, session, etc.
########################################################################
from sqlalchemy.orm import sessionmaker, scoped_session logger = logging.getLogger(__name__)
from sqlalchemy import delete
def get_engine(): class ProtectedSession(Session):
from sqlalchemy import create_engine def close(self):
return create_engine( settings.DATABASES['default']['SECRET_URL'] self._logout()
, use_native_hstore = True super().close()
, json_serializer = json_dumps
, pool_size=20, max_overflow=0 logout = close
)
def login(self, user):
engine = get_engine() if settings.DEBUG and isinstance(user, str):
# For debugging purposes
session = scoped_session(sessionmaker(bind=engine)) from gargantext.models import User
from sqlalchemy.orm.exc import NoResultFound
######################################################################## username = user
# useful for queries try:
######################################################################## user = session.query(User).filter_by(username=username).one()
from sqlalchemy.orm import aliased except NoResultFound:
from sqlalchemy import func, desc raise Exception("User %s not found!" % username)
from sqlalchemy.sql.expression import case
self.user_id = user and user.id
######################################################################## self.user_name = user and user.username
# bulk insertions self.role = settings.ROLE_SUPERUSER if user.is_superuser else \
######################################################################## settings.ROLE_STAFF if user.is_staff else \
import psycopg2 settings.ROLE_USER if user.id else \
'anon'
logger.debug("Plug authenticator for: %s (%s, %s)" % (
self.role, self.user_name, self.user_id))
sqlalchemy.event.listen(self, 'after_begin', self._postgres_auth)
def _logout(self):
if sqlalchemy.event.contains(self, 'after_begin', self._postgres_auth):
logger.debug("Unplug authenticator for: %s (%s, %s)" % (
self.role, self.user_name, self.user_id))
sqlalchemy.event.remove(self, 'after_begin', self._postgres_auth)
self.user_id = self.user_name = self.role = None
def _postgres_auth(self, *args, **kwargs):
logger.debug("Authenticate in postgres as %s (%s, %s)" % (
self.role, self.user_name, self.user_id))
ops = [('set role %s', self.role),
('set local "request.jwt.claim.role" = \'%s\'', self.role),
('set local "request.jwt.claim.sub" = %r', self.user_name),
('set local "request.jwt.claim.user_id" = %d', self.user_id)]
sql = ';'.join(s % p for s, p in ops if p) + ';'
self.connection().execute(sql)
# FIXME Keeped for backward compatibility but should be removed
engine = create_engine(settings.DATABASES['default']['SECRET_URL'])
session = scoped_session(sessionmaker(
bind=engine.execution_options(isolation_level='READ COMMITTED'),
autoflush=False))
protected_engine = create_engine(
settings.DATABASES['protected']['SECRET_URL'],
use_native_hstore=True,
json_serializer=json_dumps,
pool_size=20,
max_overflow=0)
Session = sessionmaker(
class_=ProtectedSession,
# This is the default postgresql isolation level, we state it here
# to be explicit
bind=protected_engine.execution_options(isolation_level='READ COMMITTED'),
# Disable autoflush to have more control over transactions
autoflush=False)
# FIXME Should rewrite bulk queries with SQLAlchemy Core.
# See: http://docs.sqlalchemy.org/en/latest/faq/performance.html#i-m-inserting-400-000-rows-with-the-orm-and-it-s-really-slow
def get_cursor(): def get_cursor():
db_settings = settings.DATABASES['default'] db_settings = settings.DATABASES['default']
db = psycopg2.connect(**{ db = psycopg2.connect(**{
...@@ -43,6 +103,7 @@ def get_cursor(): ...@@ -43,6 +103,7 @@ def get_cursor():
}) })
return db, db.cursor() return db, db.cursor()
class bulk_insert: class bulk_insert:
def __init__(self, table, fields, data, cursor=None): def __init__(self, table, fields, data, cursor=None):
# prepare the iterator # prepare the iterator
...@@ -74,6 +135,7 @@ class bulk_insert: ...@@ -74,6 +135,7 @@ class bulk_insert:
readline = read readline = read
def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stats=False): def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stats=False):
""" """
Inserts bulk data with an intermediate check on a uniquekey Inserts bulk data with an intermediate check on a uniquekey
...@@ -157,5 +219,3 @@ def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stat ...@@ -157,5 +219,3 @@ def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stat
cursor.execute('COMMIT WORK;') cursor.execute('COMMIT WORK;')
cursor.close() cursor.close()
from celery import shared_task
from celery.utils.log import get_task_logger
__all__ = ['shared_task', 'get_task_logger', 'schedule']
def schedule(task, when=None, args=None, kwargs=None):
task.apply_async(args=args, kwargs=kwargs)
from gargantext.core.db import Session
class DatabaseSessionMiddleware(object):
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
session = Session()
try:
request.db = session
response = self.get_response(request)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return response
from gargantext.util.db import session from gargantext.core.db import session
from gargantext.constants import NODETYPES, LISTTYPES from gargantext.constants import NODETYPES, LISTTYPES
from datetime import datetime from datetime import datetime
...@@ -8,7 +8,7 @@ from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, ...@@ -8,7 +8,7 @@ from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index,
MutableList, MutableDict, validates, ValidatorMixin, text MutableList, MutableDict, validates, ValidatorMixin, text
from .users import User from .users import User
__all__ = ['Node', 'NodeNode', 'CorpusNode'] __all__ = ['Node', 'NodeNode', 'CorpusNode', 'ProjectNode']
class NodeType(TypeDecorator): class NodeType(TypeDecorator):
"""Define a new type of column to describe a Node's type. """Define a new type of column to describe a Node's type.
...@@ -39,7 +39,7 @@ class Node(ValidatorMixin, Base): ...@@ -39,7 +39,7 @@ class Node(ValidatorMixin, Base):
<Node(id=None, typename=None, user_id=None, parent_id=None, name='without-type', date=None)> <Node(id=None, typename=None, user_id=None, parent_id=None, name='without-type', date=None)>
>>> Node(typename='CORPUS') >>> Node(typename='CORPUS')
<CorpusNode(id=None, typename='CORPUS', user_id=None, parent_id=None, name=None, date=None)> <CorpusNode(id=None, typename='CORPUS', user_id=None, parent_id=None, name=None, date=None)>
>>> from gargantext.util.db import session >>> from gargantext.core.db import g_session as session
>>> session.query(Node).filter_by(typename='USER').first() # doctest: +ELLIPSIS >>> session.query(Node).filter_by(typename='USER').first() # doctest: +ELLIPSIS
<UserNode(...)> <UserNode(...)>
...@@ -221,6 +221,18 @@ class Node(ValidatorMixin, Base): ...@@ -221,6 +221,18 @@ class Node(ValidatorMixin, Base):
return self['statuses'][-1] return self['statuses'][-1]
class UserNode(Node):
__mapper_args__ = {
'polymorphic_identity': 'USER'
}
class ProjectNode(Node):
__mapper_args__ = {
'polymorphic_identity': 'PROJECT'
}
class CorpusNode(Node): class CorpusNode(Node):
__mapper_args__ = { __mapper_args__ = {
'polymorphic_identity': 'CORPUS' 'polymorphic_identity': 'CORPUS'
......
from sqlalchemy.orm import aliased
from django.contrib.auth import models from django.contrib.auth import models
from gargantext.util.db import session, aliased from gargantext.core.db import session
from datetime import datetime from datetime import datetime
...@@ -8,6 +9,7 @@ from .base import DjangoBase, Base, Column, ForeignKey, UniqueConstraint, \ ...@@ -8,6 +9,7 @@ from .base import DjangoBase, Base, Column, ForeignKey, UniqueConstraint, \
__all__ = ['User', 'Contact'] __all__ = ['User', 'Contact']
class User(DjangoBase): class User(DjangoBase):
# The properties below are a reflection of Django's auth module's models. # The properties below are a reflection of Django's auth module's models.
__tablename__ = models.User._meta.db_table __tablename__ = models.User._meta.db_table
......
...@@ -11,8 +11,8 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ ...@@ -11,8 +11,8 @@ https://docs.djangoproject.com/en/1.11/ref/settings/
""" """
import os import os
from gargantext.util.config import config
import datetime import datetime
from gargantext.utils.config import config
# Build paths inside the project like this: os.path.join(BASE_DIR, ...) # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
...@@ -53,7 +53,7 @@ MIDDLEWARE = [ ...@@ -53,7 +53,7 @@ MIDDLEWARE = [
'django.middleware.csrf.CsrfViewMiddleware', 'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware', 'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware', 'gargantext.middleware.DatabaseSessionMiddleware',
] ]
ROOT_URLCONF = 'gargantext.backend.urls' ROOT_URLCONF = 'gargantext.backend.urls'
...@@ -105,12 +105,23 @@ LOGGING = { ...@@ -105,12 +105,23 @@ LOGGING = {
'level': LOG_LEVEL, 'level': LOG_LEVEL,
'propagate': True, 'propagate': True,
}, },
# All django loggers: https://docs.djangoproject.com/fr/1.11/topics/logging/#id3
'django.template': { 'django.template': {
# Don't keep debug logs for template module to avoid annoying and # Don't keep debug logs for template module to avoid annoying and
# useless noise, see: # useless noise, see:
# https://github.com/encode/django-rest-framework/issues/3982#issuecomment-325290221 # https://github.com/encode/django-rest-framework/issues/3982#issuecomment-325290221
'level': 'INFO' if LOG_LEVEL == 'DEBUG' else LOG_LEVEL, 'level': 'INFO' if LOG_LEVEL == 'DEBUG' else LOG_LEVEL,
}, },
'django.db.backends': {
'level': 'INFO',
},
'gargantext': {
'handlers': ['file'],
'level': LOG_LEVEL,
# Propagation to True means that this config applies to
# 'gargantext' logger and all 'gargantext.*'
'propagate': True,
},
}, },
} }
...@@ -118,23 +129,25 @@ LOGGING = { ...@@ -118,23 +129,25 @@ LOGGING = {
# Database # Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases # https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DEFAULT_DATABASE = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': config('DB_NAME', default='gargandb'),
'USER': config('DB_USER', default='gargantua'),
'PASSWORD': config('DB_PASS'),
'HOST': config('DB_HOST', default='127.0.0.1'),
'PORT': config('DB_PORT', default='5432'),
}
DATABASES = { DATABASES = {
'default': { 'default': DEFAULT_DATABASE,
'ENGINE': 'django.db.backends.postgresql_psycopg2', 'protected': dict(DEFAULT_DATABASE,
'NAME': config('DB_NAME', default='gargandb'), USER=config('DB_PROTECTED_USER', default='authenticator'),
'USER': config('DB_USER', default='gargantua'), PASSWORD=config('DB_PROTECTED_PASS'))
'PASSWORD': config('DB_PASS'),
'HOST': config('DB_HOST', default='127.0.0.1'),
'PORT': config('DB_PORT', default='5432'),
'TEST': {
'NAME': 'test_gargandb',
},
}
} }
DATABASES['default']['SECRET_URL'] = \ for db in DATABASES:
'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format( DATABASES[db]['SECRET_URL'] = \
**DATABASES['default'] 'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format(
) **DATABASES[db]
)
# Password validation # Password validation
...@@ -211,19 +224,23 @@ REST_FRAMEWORK = { ...@@ -211,19 +224,23 @@ REST_FRAMEWORK = {
'rest_framework.permissions.IsAuthenticated', 'rest_framework.permissions.IsAuthenticated',
), ),
'DEFAULT_AUTHENTICATION_CLASSES': ( 'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework_jwt.authentication.JSONWebTokenAuthentication', 'gargantext.backend.authentication.JWTAuthentication',
'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
), ),
} }
# See http://getblimp.github.io/django-rest-framework-jwt/ # See http://getblimp.github.io/django-rest-framework-jwt/
JWT_AUTH = { JWT_AUTH = {
'JWT_PAYLOAD_HANDLER': 'gargantext.backend.jwt.jwt_payload_handler', 'JWT_PAYLOAD_HANDLER': 'gargantext.backend.jwt.jwt_payload_handler',
'JWT_PAYLOAD_GET_USER_ID_HANDLER':
'gargantext.backend.jwt.jwt_get_user_id_from_payload_handler',
'JWT_PAYLOAD_GET_USERNAME_HANDLER':
'gargantext.backend.jwt.jwt_get_username_from_payload_handler',
'JWT_VERIFY_EXPIRATION': True, 'JWT_VERIFY_EXPIRATION': True,
'JWT_SECRET_KEY': config('SECRET_KEY'), 'JWT_SECRET_KEY': config('SECRET_KEY'),
'JWT_EXPIRATION_DELTA': datetime.timedelta(seconds=36000), 'JWT_EXPIRATION_DELTA': datetime.timedelta(seconds=36000),
'JWT_AUTH_HEADER_PREFIX': 'Bearer', 'JWT_AUTH_HEADER_PREFIX': 'Bearer',
'JWT_AUTH_COOKIE': 'JWT' if DEBUG else None,
} }
ROLE_SUPERUSER = 'gargantua' ROLE_SUPERUSER = 'gargantua'
......
from .dates import datetime, convert_to_datetime, MINYEAR
...@@ -2,15 +2,13 @@ ...@@ -2,15 +2,13 @@
""" """
__all__ = ['Translations', 'WeightedMatrix', 'UnweightedList', 'WeightedList', 'WeightedIndex']
from gargantext.util.db import session, bulk_insert
from collections import defaultdict from collections import defaultdict
from math import sqrt from math import sqrt
from gargantext.core.db import session, bulk_insert
__all__ = ['Translations', 'WeightedMatrix', 'UnweightedList', 'WeightedList', 'WeightedIndex']
class _BaseClass: class _BaseClass:
...@@ -303,6 +301,7 @@ class WeightedMatrix(_BaseClass): ...@@ -303,6 +301,7 @@ class WeightedMatrix(_BaseClass):
result.items[key1, key2] = value / sqrt(other.items[key1] * other.items[key2]) result.items[key1, key2] = value / sqrt(other.items[key1] * other.items[key2])
return result return result
# ?TODO rename Wordlist # ?TODO rename Wordlist
class UnweightedList(_BaseClass): class UnweightedList(_BaseClass):
......
...@@ -13,8 +13,10 @@ import itertools ...@@ -13,8 +13,10 @@ import itertools
import colorama import colorama
from colorama import Fore from colorama import Fore
from sqlalchemy.sql.expression import literal_column from sqlalchemy.sql.expression import literal_column
from sqlalchemy.orm import aliased
from sqlalchemy import func
from gargantext.util.db import session, func, aliased from gargantext.core.db import g_session as session
from gargantext.models import Node from gargantext.models import Node
......
import json import json
import types
import datetime import datetime
import traceback import traceback
import inspect
__all__ = ['json_encoder', 'json_dumps'] __all__ = ['json_encoder', 'json_dumps']
...@@ -25,10 +23,11 @@ class JSONEncoder(json.JSONEncoder): ...@@ -25,10 +23,11 @@ class JSONEncoder(json.JSONEncoder):
elif hasattr(obj, '__iter__') and not isinstance(obj, dict): elif hasattr(obj, '__iter__') and not isinstance(obj, dict):
return list(obj) return list(obj)
else: else:
return super(self.__class__, self).default(obj) return super().default(obj)
json_encoder = JSONEncoder()
# json_encoder = JSONEncoder(indent=4)
json_encoder = JSONEncoder() # compact json
def json_dumps(obj): def json_dumps(obj):
return json.dumps(obj, cls=JSONEncoder) return json.dumps(obj, cls=JSONEncoder)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment