Update to latest celery version

NOTE: Don't forget to `./manage.py migrate djcelery zero` before `pipenv
update` and `./manage.py migrate` after, to get an up-to-date database.

Pipfile

@@ -17,13 +17,13 @@ colorama = "*"
 
 django = ">=1.11,<1.12"
 dateutils = "*"
-celery = "==3.1.25"
+celery = "*"
 sqlalchemy = "*"
 psycopg2-binary = "*"
 sqlalchemy-utils = "*"
-django-celery = "*"
 djangorestframework = "*"
 djangorestframework-jwt = "*"
+django-celery-beat = "*"
 python-decouple = "*"
 alembic = "*"
 

Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "e4b1596465acb906727fbadd90970f8136e845112da9a0b828fc886db45ed845"
+            "sha256": "644c4a04bf5c7390da40b6d7650df1f12e03213a9c8c979283283b5b7cc8196d"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,16 +24,10 @@
         },
         "amqp": {
             "hashes": [
-                "sha256:2dea4d16d073c902c3b89d9b96620fb6729ac0f7a923bbc777cb4ad827c0c61a",
-                "sha256:e0ed0ce6b8ffe5690a2e856c7908dc557e0e605283d6885dd1361d79f2928908"
+                "sha256:4e28d3ea61a64ae61830000c909662cb053642efddbe96503db0e7783a6ee85b",
+                "sha256:cba1ace9d4ff6049b190d8b7991f9c1006b443a5238021aca96dd6ad2ac9da22"
             ],
-            "version": "==1.4.9"
-        },
-        "anyjson": {
-            "hashes": [
-                "sha256:37812d863c9ad3e35c0734c42e0bf0320ce8c3bed82cd20ad54cb34d158157ba"
-            ],
-            "version": "==0.3.3"
+            "version": "==2.2.2"
         },
         "argparse": {
             "hashes": [
@@ -44,23 +38,17 @@
         },
         "billiard": {
             "hashes": [
-                "sha256:204e75d390ef8f839c30a93b696bd842c3941916e15921745d05edc2a83868ab",
-                "sha256:23cb71472712e96bff3e0d45763b7b8a99e5040385fffb96816028352c255682",
-                "sha256:692a2a5a55ee39a42bcb7557930e2541da85df9ea81c6e24827f63b80cd39d0b",
-                "sha256:82041dbaa62f7fde1464d7ab449978618a38b241b40c0d31dafabb36446635dc",
-                "sha256:958fc9f8fd5cc9b936b2cb9d96f02aa5ec3613ba13ee7f089c77ff0bcc368fac",
-                "sha256:c0cbe8d45ba8d8213ad68ef9a1881002a151569c9424d551634195a18c3a4160",
-                "sha256:ccfe0419eb5e49f27ad35cf06e75360af903df6d576c66cb8073246d4e023e5c",
-                "sha256:d4d2fed1a251ea58eed47b48db3778ebb92f5ff4407dc91869c6f41c3a9249d0"
+                "sha256:1d7b22bdc47aa52841120fcd22a74ae4fc8c13e9d3935643098184f5788c3ce6",
+                "sha256:abd9ce008c9a71ccde2c816f8daa36246e92a21e6a799831b887d88277187ecd"
             ],
-            "version": "==3.3.0.23"
+            "version": "==3.5.0.3"
         },
         "celery": {
             "hashes": [
-                "sha256:1954a224805f3835e5b6f5998ec9fe51db3413cc49e59fc720d314c7913427cf",
-                "sha256:6ced63033bc663e60c992564954dbb5c84c43899f7f1a04b739957350f6b55f3"
+                "sha256:77ff3730198d6a17b3c1f05579ebe570b579efb35f6d7e13dba3b1368d068b35",
+                "sha256:81a67f0d53a688ec2bc8557bd5d6d7218f925a6f2e6df80e01560de9e28997ec"
             ],
-            "version": "==3.1.25"
+            "version": "==4.1.0"
         },
         "dateutils": {
             "hashes": [
@@ -75,12 +63,12 @@
             ],
             "version": "==1.11.11"
         },
-        "django-celery": {
+        "django-celery-beat": {
             "hashes": [
-                "sha256:1450264db5ec58e45f3f20d8d361e696920352f62481bf56047288cdb38bcc0b",
-                "sha256:aaba492bf7777f231ec6b02c80aa3ea68758c39f4723864dd4164589b99ad703"
+                "sha256:1f770a936f070fd6b2ceac123ef4951fbe9e941e39b52ae9eea5ac0efe57d51d",
+                "sha256:d40e2c48aeed7043fff6064f53bc157f6a1f55b45d984eb2b52bf1b88ae96026"
             ],
-            "version": "==3.2.2"
+            "version": "==1.1.1"
         },
         "djangorestframework": {
             "hashes": [
@@ -98,10 +86,10 @@
         },
         "kombu": {
             "hashes": [
-                "sha256:7ceab743e3e974f3e5736082e8cc514c009e254e646d6167342e0e192aee81a6",
-                "sha256:e064a00c66b4d1058cd2b0523fb8d98c82c18450244177b6c0f7913016642650"
+                "sha256:01f0da9fe222a2183345004243d1518c0fbe5875955f1b24842f2d9c65709ade",
+                "sha256:4249d9dd9dbf1fcec471d1c2def20653c9310dd1a217272d77e4844f9d5273cb"
             ],
-            "version": "==3.0.37"
+            "version": "==4.1.0"
         },
         "mako": {
             "hashes": [
@@ -205,6 +193,13 @@
                 "sha256:e416a4a882d97082f9cbd8373b185aa858b57a012f113421c6dc031af74067e4"
             ],
             "version": "==0.33.0"
+        },
+        "vine": {
+            "hashes": [
+                "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72",
+                "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b"
+            ],
+            "version": "==1.1.4"
         }
     },
     "develop": {

gargantext/settings.py

@@ -42,7 +42,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
-    'djcelery',
+    'django_celery_beat',
     'gargantext.backend',
 ]
 
@@ -204,15 +204,12 @@ MEDIA_URL = '/media/'
 
 # Asynchronous tasks
 
-import djcelery
-djcelery.setup_loader()
-
 BROKER_URL = config('BROKER_URL', default='amqp://guest:guest@localhost:5672/')
 
 CELERY_APP = config('CELERY_APP', default='gargantext.backend')
 CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
 CELERY_TIMEZONE = TIME_ZONE
-CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
+CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
 CELERY_IMPORTS = ()
 
 

tools/init.d/gargantext-celery

@@ -10,7 +10,7 @@
 # Description:       starts gargantext task queue using celery
 ### END INIT INFO
 
-DAEMON=$(pipenv --py)
+DAEMON=${CELERY:-$(which celery)}
 NAME=gargantext-celery
 DESC=gargantext-celery
 
@@ -34,14 +34,14 @@ celery() {
 
 	if [ "$1" = "start" ]; then
 		# Detect errors in python code
-		$DAEMON $PWD/manage.py celery worker -A "$CELERY_APP" 2> $ERROR_LOG 1> /dev/null &
+		$DAEMON worker -A "$CELERY_APP" 2> $ERROR_LOG 1> /dev/null &
 		# Wait 1 second and kill process, if it was already exited we have an
 		# error
 		sleep 1 && kill $! > /dev/null 2>&1 || \
 			( log_progress_msg "[$(tail -1 $ERROR_LOG)]" && return 2 )
 	fi
 
-	$DAEMON $PWD/manage.py celery multi $1 1 -A "$CELERY_APP" -B -E \
+	$DAEMON multi $1 1 -A "$CELERY_APP" -B -E \
 		--pidfile="$CELERYD_PID_FILE" \
 		--logfile="$CELERYD_LOG_FILE" \
 		--loglevel="$CELERYD_LOG_LEVEL" 2> $ERROR_LOG 1> /dev/null