Merge branch 'romain-goodies' into unstable

gargantext/settings.py

@@ -42,6 +42,9 @@ CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
 CELERY_IMPORTS = ("gargantext.util.toolchain")
 
 
+# garg's custom unittests runner (adapted to our db models)
+TEST_RUNNER = 'unittests.framework.GargTestRunner'
+
 # Application definition
 
 INSTALLED_APPS = [
@@ -123,6 +126,9 @@ DATABASES = {
         'PASSWORD': 'C8kdcUrAQy66U',
         'HOST': '127.0.0.1',
         'PORT': '5432',
+        'TEST': {
+            'NAME': 'test_gargandb',
+        },
     }
 }
 
@@ -164,6 +170,3 @@ USE_I18N = True
 USE_L10N = True
 
 USE_TZ = True
-
-
-

unittests/README.md

+UNIT TESTS
+==========
+
+Prerequisite
+------------
+Running unit tests will involve creating a **temporary test DB** !
+ + it implies **CREATEDB permssions** for settings.DATABASES.user  
+   (this has security consequences)
+ + for instance in gargantext you would need to run this in psql as postgres:  
+   `# ALTER USER gargantua CREATEDB;`
+
+A "principe de précaution" could be to allow gargantua the CREATEDB rights on the **dev** machines (to be able to run tests) and not give it on the **prod** machines (no testing but more protection just in case).
+
+Usage
+------
+```
+./manage.py test unittests/ -v 2            # in django root container directory
+
+# or for a single module
+./manage.py test unittests.tests_010_basic  -v 2
+```
+
+( `-v 2` is the verbosity level )
+
+
+Tests
+------
+  1. **tests_010_basic**
+  2. ** tests ??? **
+  3. ** tests ??? **
+  4. ** tests ??? **
+  5. ** tests ??? **
+  6. ** tests ??? **
+  7. **tests_070_routes**  
+     Checks the response types from the app url routes:  
+    - "/"
+    - "/api/nodes"
+    - "/api/nodes/<ID>"
+
+
+GargTestRunner
+---------------
+Most of the tests will interact with a DB but we don't want to touch the real one so we provide a customized test_runner class in `unittests/framework.py` that creates a test database.
+
+It must be referenced in django's `settings.py` like this:
+```
+TEST_RUNNER = 'unittests.framework.GargTestRunner'
+```
+
+(This way the `./manage.py test` command will be using GargTestRunner.)
+
+
+Using a DB session
+------------------
+To emulate a session the way we usually do it in gargantext, our `unittests.framework` also
+provides a session object to the test database via `GargTestRunner.testdb_session`
+
+To work correctly, it needs to be read *inside the test setup.*
+
+**Example**
+```
+from unittests.framework import GargTestRunner
+
+class MyTestRecipes(TestCase):
+    def setUp(self):
+        # -------------------------------------
+        session = GargTestRunner.testdb_session
+        # -------------------------------------
+        new_project = Node(
+            typename = 'PROJECT',
+            name = "hello i'm a project",
+        )
+        session.add(new_project)
+        session.commit()
+```
+
+
+Accessing the URLS
+------------------
+Django tests provide a client to browse the urls
+
+
+**Example**
+```
+from django.test import Client
+
+class MyTestRecipes(TestCase):
+    def setUp(self):
+        self.client = Client()
+
+    def test_001_get_front_page(self):
+        ''' get the about page localhost/about '''
+        # --------------------------------------
+        the_response = self.client.get('/about')
+        # --------------------------------------
+        self.assertEqual(the_response.status_code, 200)
+```
+
+Logging in
+-----------
+Most of our functionalities are only available on login so we provide a fake user at the initialization of the test DB.
+
+His login in 'pcorser' and password is 'peter'
+
+**Example**
+```
+from django.test import Client
+
+class MyTestRecipes(TestCase):
+    def setUp(self):
+        self.client = Client()
+        # login ---------------------------------------------------
+        response = self.client.post(
+                      '/auth/login/',
+                      {'username': 'pcorser', 'password': 'peter'}
+                   )
+        # ---------------------------------------------------------
+
+    def test_002_get_to_a_restricted_page(self):
+        ''' get the projects page /projects '''
+        the_response = self.client.get('/projects')
+        self.assertEqual(the_response.status_code, 200)
+```
+
+*Si vous aimez les aventures de Peter Corser, lisez l'album précédent ["Doors"](https://gogs.iscpif.fr/leclaire/doors)* (Scénario M. Leclaire, Dessins R. Loth) (disponible dans toutes les bonnes librairies)
+
+
+FIXME
+-----
+
+url client get will still give read access to original DB ?
+      cf. http://stackoverflow.com/questions/19714521
+      cf. http://stackoverflow.com/questions/11046039
+      cf. test_073_get_api_one_node

unittests/framework.py

+"""
+A test runner derived from default (DiscoverRunner) but adapted to our custom DB
+
+cf. docs.djangoproject.com/en/1.9/topics/testing/advanced/#using-different-testing-frameworks
+cf. gargantext/settings.py => TEST_RUNNER
+cf. dbmigrate.py
+
+FIXME url get will still give read access to original DB ?
+      cf. http://stackoverflow.com/questions/19714521
+      cf. http://stackoverflow.com/questions/11046039
+      cf. test_073_get_api_one_node
+"""
+
+# basic elements
+from django.test.runner  import DiscoverRunner, get_unique_databases_and_mirrors
+from sqlalchemy          import create_engine
+from gargantext.settings import DATABASES
+
+# things needed to create a user
+from django.contrib.auth.models import User
+
+# here we setup a minimal django so as to load SQLAlchemy models ---------------
+# and then be able to import models and Base.metadata.tables
+from os import environ
+from django import setup
+environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
+setup()   # models can now be imported
+from gargantext import models # Base is now filled
+from gargantext.util.db  import Base  # contains metadata.tables
+# ------------------------------------------------------------------------------
+
+# things needed to provide a session
+from sqlalchemy.orm import sessionmaker, scoped_session
+
+
+class GargTestRunner(DiscoverRunner):
+    """
+    We use the default test runner but we just add
+    our own dbmigrate elements at db creation
+
+    => we let django.test.runner do the test db creation + auto migrations
+    => we retrieve the test db name from django.test.runner
+    => we create a test engine like in gargantext.db.create_engine but with the test db name
+    => we create tables for our models like in dbmigrate with the test engine
+
+    TODO: list of tables to be created are hard coded in self.models
+    """
+
+    # we'll also expose a session as GargTestRunner.testdb_session
+    testdb_session = None
+
+    def __init__(self, *args, **kwargs):
+        # our custom tables to be created (in correct order)
+        self.models = ['ngrams', 'nodes', 'contacts', 'nodes_nodes',  'nodes_ngrams', 'nodes_nodes_ngrams',  'nodes_ngrams_ngrams',  'nodes_hyperdata']
+        self.testdb_engine = None
+
+        # and execute default django init
+        old_config = super(GargTestRunner, self).__init__(*args, **kwargs)
+
+
+    def setup_databases(self, *args, **kwargs):
+        """
+        Complement the database creation
+        by our own "models to tables" migration
+        """
+
+        # default django setup performs base creation + auto migrations
+        old_config = super(GargTestRunner, self).setup_databases(*args, **kwargs)
+
+        # retrieve the testdb_name set by DiscoverRunner
+        testdb_names = []
+        for db_infos in get_unique_databases_and_mirrors():
+            # a key has the form: (IP, port, backend, dbname)
+            for key in db_infos:
+                # db_infos[key] has the form (dbname, {'default'})
+                testdb_names.append(db_infos[key][0])
+
+        # /!\ hypothèse d'une database unique /!\
+        testdb_name = testdb_names[0]
+
+        # now we use a copy of our normal db config...
+        db_params = DATABASES['default']
+
+        # ...just changing the name
+        db_params['NAME'] = testdb_name
+
+        # connect to this test db
+        testdb_url = 'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format_map(db_params)
+        self.testdb_engine = create_engine( testdb_url )
+        print("TESTDB INIT: opened connection to database **%s**" % db_params['NAME'])
+
+        # we retrieve real tables declarations from our loaded Base
+        sqla_models = (Base.metadata.tables[model_name] for model_name in self.models)
+
+        # example: Base.metadata.tables['ngrams']
+        # ---------------------------------------
+        # Table('ngrams', Column('id', Integer(), table=<ngrams>, primary_key=True),
+        #                 Column('terms', String(length=255), table=<ngrams>),
+        #                 Column('n', Integer(), table=<ngrams>),
+        #                 schema=None)
+
+
+        # and now creation of each table in our test db (like dbmigrate)
+        for model in sqla_models:
+            try:
+                model.create(self.testdb_engine)
+                print('TESTDB INIT: created model: `%s`' % model)
+            except Exception as e:
+                print('TESTDB INIT ERROR: could not create model: `%s`, %s' % (model, e))
+
+
+        # we also create a session to provide it the way we usually do in garg
+        # (it's a class based static var to be able to share it with our tests)
+        GargTestRunner.testdb_session = scoped_session(sessionmaker(bind=self.testdb_engine))
+
+        # and let's create a user too otherwise we'll never be able to login
+        user = User.objects.create_user(username='pcorser', password='peter')
+
+        # old_config will be used by DiscoverRunner
+        # (to remove everything at the end)
+        return old_config
+
+
+    def teardown_databases(self, old_config, *args, **kwargs):
+        """
+        After all tests
+        """
+        # close the session
+        GargTestRunner.testdb_session.close()
+
+        # free the connection
+        self.testdb_engine.dispose()
+
+        # default django teardown performs destruction of the test base
+        super(GargTestRunner, self).teardown_databases(old_config, *args, **kwargs)
+
+
+
+
+# snippets if we choose direct model building instead of setup() and Base.metadata.tables[model_name]
+# from sqlalchemy.types import Integer, String, DateTime, Text, Boolean, Float
+# from gargantext.models.nodes import NodeType
+# from gargantext.models.hyperdata import HyperdataKey
+# from sqlalchemy.schema import Table, Column, ForeignKey, UniqueConstraint, MetaData
+# from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION
+# from sqlalchemy.ext.mutable import MutableDict, MutableList
+# Double = DOUBLE_PRECISION
+
+# sqla_models = [i for i in sqla_models]
+# print (sqla_models)
+# sqla_models = [Table('ngrams', MetaData(bind=None), Column('id', Integer(), primary_key=True, nullable=False), Column('terms', String(length=255)), Column('n', Integer()), schema=None), Table('nodes', MetaData(bind=None), Column('id', Integer(), primary_key=True, nullable=False), Column('typename', NodeType()), Column('user_id', Integer(), ForeignKey('auth_user.id')), Column('parent_id', Integer(), ForeignKey('nodes.id')), Column('name', String(length=255)), Column('date', DateTime()), Column('hyperdata', JSONB(astext_type=Text())), schema=None), Table('contacts', MetaData(bind=None), Column('id', Integer(), primary_key=True, nullable=False), Column('user1_id', Integer(), primary_key=True, nullable=False), Column('user2_id', Integer(), primary_key=True, nullable=False), Column('is_blocked', Boolean()), Column('date_creation', DateTime()), schema=None), Table('nodes_nodes', MetaData(bind=None), Column('node1_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('node2_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('score', Float(precision=24)), schema=None), Table('nodes_ngrams', MetaData(bind=None), Column('node_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('ngram_id', Integer(), ForeignKey('ngrams.id'), primary_key=True, nullable=False), Column('weight', Float()), schema=None), Table('nodes_nodes_ngrams', MetaData(bind=None), Column('node1_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('node2_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('ngram_id', Integer(), ForeignKey('ngrams.id'), primary_key=True, nullable=False), Column('score', Float(precision=24)), schema=None), Table('nodes_ngrams_ngrams', MetaData(bind=None), Column('node_id', Integer(), ForeignKey('nodes.id'), primary_key=True, nullable=False), Column('ngram1_id', Integer(), ForeignKey('ngrams.id'), primary_key=True, nullable=False), Column('ngram2_id', Integer(), ForeignKey('ngrams.id'), primary_key=True, nullable=False), Column('weight', Float(precision=24)), schema=None), Table('nodes_hyperdata', MetaData(bind=None), Column('id', Integer(), primary_key=True, nullable=False), Column('node_id', Integer(), ForeignKey('nodes.id')), Column('key', HyperdataKey()), Column('value_int', Integer()), Column('value_flt', DOUBLE_PRECISION()), Column('value_utc', DateTime(timezone=True)), Column('value_str', String(length=255)), Column('value_txt', Text()), schema=None)]

unittests/tests_010_basic.py

+"""
+BASIC UNIT TESTS FOR GARGANTEXT IN DJANGO
+=========================================
+"""
+from django.test import TestCase
+
+
+class NodeTestCase(TestCase):
+    def setUp(self):
+        from gargantext.models import nodes
+        self.node_1000 = nodes.Node(id=1000)
+        self.new_node = nodes.Node()
+
+    def test_010_node_has_id(self):
+        '''new_node.id'''
+        self.assertEqual(self.node_1000.id, 1000)
+
+    def test_011_node_write(self):
+        '''write new_node to DB and commit'''
+        from gargantext.util.db import session
+        self.assertFalse(self.new_node._sa_instance_state._attached)
+        session.add(self.new_node)
+        session.commit()
+        self.assertTrue(self.new_node._sa_instance_state._attached)

unittests/tests_070_routes.py

+"""
+ROUTE UNIT TESTS
+================
+"""
+from django.test import TestCase
+from django.test import Client
+
+# to be able to create Nodes
+from gargantext.models import Node
+
+# to be able to compare in test_073_get_api_one_node()
+from gargantext.constants import NODETYPES
+
+# provides GargTestRunner.testdb_session
+from unittests.framework import GargTestRunner
+
+
+class RoutesChecker(TestCase):
+    def setUp(self):
+        """
+        Will be run before each test
+        """
+        self.client = Client()
+
+        # login with our fake user
+        response = self.client.post(
+                            '/auth/login/',
+                            {'username': 'pcorser', 'password': 'peter'}
+                            )
+        print(response.status_code)
+
+        session = GargTestRunner.testdb_session
+
+        new_project = Node(
+            typename = 'PROJECT',
+            name = "hello i'm a project",
+        )
+        session.add(new_project)
+        session.commit()
+        self.a_node_id = new_project.id
+        print("created a project with id: %i" % new_project.id)
+
+    def test_071_get_front_page(self):
+        ''' get the front page / '''
+        front_response = self.client.get('/')
+        self.assertEqual(front_response.status_code, 200)
+        self.assertIn('text/html', front_response.get('Content-Type'))
+        # on suppose que la page contiendra toujours ce titre
+        self.assertIn(b'<h1>Gargantext</h1>', front_response.content)
+
+    def test_072_get_api_nodes(self):
+        ''' get "/api/nodes" '''
+        api_response = self.client.get('/api/nodes')
+        self.assertEqual(api_response.status_code, 200)
+
+        # 1) check the type is json
+        self.assertTrue(api_response.has_header('Content-Type'))
+        self.assertIn('application/json', api_response.get('Content-Type'))
+
+        # 2) let's try to get things in the json
+        json_content = api_response.json()
+        json_count = json_content['count']
+        json_nodes = json_content['records']
+        self.assertEqual(type(json_count), int)
+        self.assertEqual(type(json_nodes), list)
+        print("\ntesting nodecount: %i " % json_count)
+
+
+    def test_073_get_api_one_node(self):
+        ''' get "api/nodes/<node_id>" '''
+
+        # we first get one node id by re-running this bit from test_072
+        a_node_id = self.client.get('/api/nodes').json()['records'][0]['id']
+
+        one_node_route = '/api/nodes/%i' % a_node_id
+        # print("\ntesting node route: %s" % one_node_route)
+        api_response = self.client.get(one_node_route)
+        self.assertTrue(api_response.has_header('Content-Type'))
+        self.assertIn('application/json', api_response.get('Content-Type'))
+
+        json_content = api_response.json()
+        nodetype = json_content['typename']
+        nodename = json_content['name']
+        print("\ntesting nodename:", nodename)
+        print("\ntesting nodetype:", nodetype)
+        self.assertIn(nodetype, NODETYPES)
+
+    # TODO http://localhost:8000/api/nodes?types[]=CORPUS
+
+    # £TODO test request.*
+        # print ("request")
+        # print ("user.id", request.user.id)
+        # print ("user.name", request.user.username)
+        # print ("path", request.path)
+        # print ("path_info", request.path_info)