unittests: fixed framework to be able to use gargantext.util.db.session...

unittests: fixed framework to be able to use gargantext.util.db.session seemlessly on the test db as if it was the normal one (no need anymore for GargTestRunner.testdb_session)

unittests/framework.py

@@ -24,13 +24,18 @@ from django.contrib.auth.models import User
 from os import environ
 from django import setup
 environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
+
+# hack to make our minimal django use the same test DB settings as DiscoverRunner
+# (more details: cf ./session_and_db_remarks.md)
+DATABASES['default']['NAME'] = DATABASES['default']['TEST']['NAME']
+
 setup()   # models can now be imported
 from gargantext import models # Base is now filled
 from gargantext.util.db  import Base  # contains metadata.tables
 # ------------------------------------------------------------------------------
 
-# things needed to provide a session
-from sqlalchemy.orm import sessionmaker, scoped_session
+# thanks to our hack, util.db.engine and util.db.session already use the test DB
+from gargantext.util.db import engine, session
 
 
 class GargTestRunner(DiscoverRunner):
@@ -43,18 +48,18 @@ class GargTestRunner(DiscoverRunner):
     => we create a test engine like in gargantext.db.create_engine but with the test db name
     => we create tables for our models like in dbmigrate with the test engine
 
-    TODO: list of tables to be created are hard coded in self.models
+    POSSIBLE: definitions of tables to be created should be fully hard coded like the list in self.models
+    => then remove django.setup() used to import models and DATABASES renaming to prevent its secondary effects
     """
 
-    # we'll also expose a session as GargTestRunner.testdb_session
-    testdb_session = None
-
     def __init__(self, *args, **kwargs):
-        # our custom tables to be created (in correct order)
+        # our custom tablenames to be created (in correct order)
         self.models = ['ngrams', 'nodes', 'contacts', 'nodes_nodes',  'nodes_ngrams', 'nodes_nodes_ngrams',  'nodes_ngrams_ngrams',  'nodes_hyperdata']
-        self.testdb_engine = None
 
-        # and execute default django init
+        # POSSIBLE: hard-code here our custom table declarations
+        # self.tables = [Table('ngrams', MetaData(bind=None)....)]
+
+        # and execute default django unittests init
         old_config = super(GargTestRunner, self).__init__(*args, **kwargs)
 
 
@@ -67,28 +72,6 @@ class GargTestRunner(DiscoverRunner):
         # default django setup performs base creation + auto migrations
         old_config = super(GargTestRunner, self).setup_databases(*args, **kwargs)
 
-        # retrieve the testdb_name set by DiscoverRunner
-        testdb_names = []
-        for db_infos in get_unique_databases_and_mirrors():
-            # a key has the form: (IP, port, backend, dbname)
-            for key in db_infos:
-                # db_infos[key] has the form (dbname, {'default'})
-                testdb_names.append(db_infos[key][0])
-
-        # /!\ hypothèse d'une database unique /!\
-        testdb_name = testdb_names[0]
-
-        # now we use a copy of our normal db config...
-        db_params = DATABASES['default']
-
-        # ...just changing the name
-        db_params['NAME'] = testdb_name
-
-        # connect to this test db
-        testdb_url = 'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format_map(db_params)
-        self.testdb_engine = create_engine( testdb_url )
-        print("TESTDB INIT: opened connection to database **%s**" % db_params['NAME'])
-
         # we retrieve real tables declarations from our loaded Base
         sqla_models = (Base.metadata.tables[model_name] for model_name in self.models)
 
@@ -103,16 +86,11 @@ class GargTestRunner(DiscoverRunner):
         # and now creation of each table in our test db (like dbmigrate)
         for model in sqla_models:
             try:
-                model.create(self.testdb_engine)
+                model.create(engine)
                 print('TESTDB INIT: created model: `%s`' % model)
             except Exception as e:
                 print('TESTDB INIT ERROR: could not create model: `%s`, %s' % (model, e))
 
-
-        # we also create a session to provide it the way we usually do in garg
-        # (it's a class based static var to be able to share it with our tests)
-        GargTestRunner.testdb_session = scoped_session(sessionmaker(bind=self.testdb_engine))
-
         # and let's create a user too otherwise we'll never be able to login
         user = User.objects.create_user(username='pcorser', password='peter')
 
@@ -126,10 +104,10 @@ class GargTestRunner(DiscoverRunner):
         After all tests
         """
         # close the session
-        GargTestRunner.testdb_session.close()
+        session.close()
 
         # free the connection
-        self.testdb_engine.dispose()
+        engine.dispose()
 
         # default django teardown performs destruction of the test base
         super(GargTestRunner, self).teardown_databases(old_config, *args, **kwargs)

unittests/session_and_db_remarks.md

+# About the DB settings during the tests
+
+rloth 2016-08-22
+
+#### Correct ordering strategies
+
+Our specific database model causes a problem for the correct order of doing things
+
+  - the good practice in creating the test framework is:
+    1. create a child class of DiscoverRunner
+    2. define a 'TEST' key in `settings.DATABASES.default`
+    3. let the DiscoverRunner create the tables in his `__init__()` by calling `super.__init__()` from the child class  
+     (cf. https://docs.djangoproject.com/en/1.10/topics/testing/advanced/)
+
+
+  - but we have tables not in the migrations... so creating our full database model (with the `nodes_*` tables) implies either to hard-code their definitions or to:
+     1. do a `django.setup()` first so we can load the SQLAlchemy models (`import gargantext.models`)
+     2. from there use `util.db.Base` as the table definitions
+      - Table('nodes', Column('id', Integer()...)
+      - Table('ngrams' ...)
+      - etc.
+     3. Use those definitions to create the tables:  `table_definition.create(engine)`  
+     *(cf. db_migrate.py)*
+
+
+
+#### But we see these two ordering strategies are contradictory!
+
+**Explanation**: Doing the `django.setup()` to get the models will load the app modules before using the test database created by `DiscoverRunner.__init__()`
+
+**Consequence**: `util.db.session` will use the native settings for the "real DB" instead of the "test DB".
+
+
+
+#### So we need to "cheat" a little bit...
+
+**Solution 1**   *(=> will be better in the long run when the tables stop changing)*  
+We could hard-code the list of tables and columns to create in the test DB. Then there would be no need to load the models to do the migration, so therefore no need to do a `django.setup()` before the `DiscoverRunner.__init__()`  
+
+
+**Solution 2**  *(=> used now)*  
+We do the `django.setup()` but we modify its `gargantext.settings.DATABASES` on-the-fly with this line:
+```
+DATABASES['default']['NAME'] = DATABASES['default']['TEST']['NAME']
+```
+
+This is a dirty hack because changing settings at runtime makes final values difficult to track, but this way, the setup part and the DiscoverRunner part will share the same DB name (`test_gargandb`)
+
+
+
+### To inspect the testdb
+
+Run tests with:
+```
+./manage.py test unittests/ --keepdb
+```
+And after the tests, connect to it as gargantua with `psql test_gargandb`

unittests/tests_070_routes.py

@@ -11,9 +11,8 @@ from gargantext.models import Node
 # to be able to compare in test_073_get_api_one_node()
 from gargantext.constants import NODETYPES
 
-# provides GargTestRunner.testdb_session
-from unittests.framework import GargTestRunner
-
+from gargantext.util.db   import session
+from gargantext.settings import DATABASES
 
 class RoutesChecker(TestCase):
     def setUp(self):
@@ -27,13 +26,12 @@ class RoutesChecker(TestCase):
                             '/auth/login/',
                             {'username': 'pcorser', 'password': 'peter'}
                             )
-        print(response.status_code)
-
-        session = GargTestRunner.testdb_session
+        # print(response.status_code) # expected: 302 FOUND
 
         new_project = Node(
             typename = 'PROJECT',
             name = "hello i'm a project",
+            user_id = 1                   # todo make sure it's the same user as login
         )
         session.add(new_project)
         session.commit()
@@ -47,7 +45,7 @@ class RoutesChecker(TestCase):
         self.assertIn('text/html', front_response.get('Content-Type'))
         # on suppose que la page contiendra toujours ce titre
         self.assertIn(b'<h1>Gargantext</h1>', front_response.content)
-    
+
     def test_071b_get_inexisting_page(self):
         ''' get the inexisting page /foo '''
         front_response = self.client.get('/foo')
@@ -64,20 +62,15 @@ class RoutesChecker(TestCase):
 
         # 2) let's try to get things in the json
         json_content = api_response.json()
+        print(json_content)
         json_count = json_content['count']
         json_nodes = json_content['records']
         self.assertEqual(type(json_count), int)
         self.assertEqual(type(json_nodes), list)
-        print("\ntesting nodecount: %i " % json_count)
-
 
     def test_073_get_api_one_node(self):
         ''' get "api/nodes/<node_id>" '''
-
-        # we first get one node id by re-running this bit from test_072
-        a_node_id = self.client.get('/api/nodes').json()['records'][0]['id']
-
-        one_node_route = '/api/nodes/%i' % a_node_id
+        one_node_route = '/api/nodes/%i' % self.a_node_id
         # print("\ntesting node route: %s" % one_node_route)
         api_response = self.client.get(one_node_route)
         self.assertTrue(api_response.has_header('Content-Type'))
@@ -89,6 +82,7 @@ class RoutesChecker(TestCase):
         print("\ntesting nodename:", nodename)
         print("\ntesting nodetype:", nodetype)
         self.assertIn(nodetype, NODETYPES)
+        self.assertEqual(nodename, "hello i'm a project")
 
     # TODO http://localhost:8000/api/nodes?types[]=CORPUS
 

unittests/tests_090_toolchain.py

@@ -8,10 +8,7 @@ from django.test import TestCase, Client, RequestFactory
 from gargantext.models import Node, User
 from gargantext.util.db import session
 
-
 from gargantext.constants import RESOURCETYPES, NODETYPES, get_resource
-# provides GargTestRunner.testdb_session
-#from unittests.framework import GargTestRunner
 
 from gargantext.util.toolchain.main import *
 DATA_SAMPLE_DIR = "/srv/gargantext_lib/test_samples/"