Update to latest celery version

NOTE: Don't forget to `./manage.py migrate djcelery zero` before `pipenv
update` and `./manage.py migrate` after, to get an up-to-date database.

Pipfile

@@ -17,15 +17,15 @@ colorama = "*"
 
 Django = ">=1.11,<1.12"
 dateutils = "*"
-celery = "==3.1.25"
+celery = "*"
 SQLAlchemy = "*"
 "psycopg2" = "*"
 SQLAlchemy-Utils = "*"
-django-celery = "*"
 djangorestframework = "*"
 djangorestframework-jwt = "*"
 python-decouple = "*"
 alembic = "*"
+django_celery_beat = "*"
 
 
 [requires]

Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "44ce89065245fce07b4f14911f128a110b8f7cca24207cc23a49a73aed1ff5b8"
+            "sha256": "3f498c4c7f782ca20571dfac36bb531d6e161d9b2b00a5c7026d782823a5d0f8"
         },
         "host-environment-markers": {
             "implementation_name": "cpython",
@@ -37,16 +37,10 @@
         },
         "amqp": {
             "hashes": [
-                "sha256:e0ed0ce6b8ffe5690a2e856c7908dc557e0e605283d6885dd1361d79f2928908",
-                "sha256:2dea4d16d073c902c3b89d9b96620fb6729ac0f7a923bbc777cb4ad827c0c61a"
+                "sha256:4e28d3ea61a64ae61830000c909662cb053642efddbe96503db0e7783a6ee85b",
+                "sha256:cba1ace9d4ff6049b190d8b7991f9c1006b443a5238021aca96dd6ad2ac9da22"
             ],
-            "version": "==1.4.9"
-        },
-        "anyjson": {
-            "hashes": [
-                "sha256:37812d863c9ad3e35c0734c42e0bf0320ce8c3bed82cd20ad54cb34d158157ba"
-            ],
-            "version": "==0.3.3"
+            "version": "==2.2.2"
         },
         "argparse": {
             "hashes": [
@@ -57,23 +51,17 @@
         },
         "billiard": {
             "hashes": [
-                "sha256:c0cbe8d45ba8d8213ad68ef9a1881002a151569c9424d551634195a18c3a4160",
-                "sha256:82041dbaa62f7fde1464d7ab449978618a38b241b40c0d31dafabb36446635dc",
-                "sha256:958fc9f8fd5cc9b936b2cb9d96f02aa5ec3613ba13ee7f089c77ff0bcc368fac",
-                "sha256:204e75d390ef8f839c30a93b696bd842c3941916e15921745d05edc2a83868ab",
-                "sha256:d4d2fed1a251ea58eed47b48db3778ebb92f5ff4407dc91869c6f41c3a9249d0",
-                "sha256:23cb71472712e96bff3e0d45763b7b8a99e5040385fffb96816028352c255682",
-                "sha256:ccfe0419eb5e49f27ad35cf06e75360af903df6d576c66cb8073246d4e023e5c",
-                "sha256:692a2a5a55ee39a42bcb7557930e2541da85df9ea81c6e24827f63b80cd39d0b"
+                "sha256:abd9ce008c9a71ccde2c816f8daa36246e92a21e6a799831b887d88277187ecd",
+                "sha256:1d7b22bdc47aa52841120fcd22a74ae4fc8c13e9d3935643098184f5788c3ce6"
             ],
-            "version": "==3.3.0.23"
+            "version": "==3.5.0.3"
         },
         "celery": {
             "hashes": [
-                "sha256:1954a224805f3835e5b6f5998ec9fe51db3413cc49e59fc720d314c7913427cf",
-                "sha256:6ced63033bc663e60c992564954dbb5c84c43899f7f1a04b739957350f6b55f3"
+                "sha256:81a67f0d53a688ec2bc8557bd5d6d7218f925a6f2e6df80e01560de9e28997ec",
+                "sha256:77ff3730198d6a17b3c1f05579ebe570b579efb35f6d7e13dba3b1368d068b35"
             ],
-            "version": "==3.1.25"
+            "version": "==4.1.0"
         },
         "dateutils": {
             "hashes": [
@@ -88,12 +76,12 @@
             ],
             "version": "==1.11.9"
         },
-        "django-celery": {
+        "django-celery-beat": {
             "hashes": [
-                "sha256:1450264db5ec58e45f3f20d8d361e696920352f62481bf56047288cdb38bcc0b",
-                "sha256:aaba492bf7777f231ec6b02c80aa3ea68758c39f4723864dd4164589b99ad703"
+                "sha256:b5fa8f956ca9a0aa2bf81dcfb246542620b8b17a2a61b6c77e43a2cac5252826",
+                "sha256:2371f6d23b63a09aff8dbd294665becd0f3bfa1e05e59bb6fbe2fee3abbdfa9d"
             ],
-            "version": "==3.2.2"
+            "version": "==1.1.0"
         },
         "djangorestframework": {
             "hashes": [
@@ -109,12 +97,26 @@
             ],
             "version": "==1.11.0"
         },
+        "ephem": {
+            "hashes": [
+                "sha256:6a2e445ba3a1e6bd9d6dedcafa4dda83957f4f9b0efac3d642974c55faebcfa4",
+                "sha256:9ea5c8d9b407fe151cece238d13e3ca12114ac5c73269ef6541bf65b208048a3",
+                "sha256:4bcd9899863ef04f4e75d894a6973dce4b4d16baeb8c2e96fb66bd3c677491a2",
+                "sha256:7af6d726c3d903087c284e3dd72c5cda2b5438e84f2d564314469f0fb7494fab",
+                "sha256:3884de133045d2f12784ef456c0f5557139a247b88d2c26097f7bd420803ed7f",
+                "sha256:bb3e04e981352ab8c6049325533944b882d9be1bf13c19be5d85b918ba75723f",
+                "sha256:fd15421938cac27cd87c3b73c81e4695ab7a22cd37d43e05090140f5d48392d8",
+                "sha256:f19a380f83f36e56e6e499bf673a43c42ed28c766c9cafb4326b5defcca0a116",
+                "sha256:7a4c82b1def2893e02aec0394f108d24adb17bd7b0ca6f4bc78eb7120c0212ac"
+            ],
+            "version": "==3.7.6.0"
+        },
         "kombu": {
             "hashes": [
-                "sha256:7ceab743e3e974f3e5736082e8cc514c009e254e646d6167342e0e192aee81a6",
-                "sha256:e064a00c66b4d1058cd2b0523fb8d98c82c18450244177b6c0f7913016642650"
+                "sha256:01f0da9fe222a2183345004243d1518c0fbe5875955f1b24842f2d9c65709ade",
+                "sha256:4249d9dd9dbf1fcec471d1c2def20653c9310dd1a217272d77e4844f9d5273cb"
             ],
-            "version": "==3.0.37"
+            "version": "==4.1.0"
         },
         "mako": {
             "hashes": [
@@ -213,15 +215,22 @@
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:7dda3e0b1b12215e3bb05368d1abbf7d747112a43738e0a4e6deb466b83fd88e"
+                "sha256:9ede7070d6fd18f28058be88296ed67893e2637465516d6a596cd9afea97b154"
             ],
-            "version": "==1.2.0"
+            "version": "==1.2.1"
         },
         "sqlalchemy-utils": {
             "hashes": [
                 "sha256:e35431b0e57c4f7030ff598c23813c8b7b04b508ce10e8e9ebe448645b38d6d7"
             ],
             "version": "==0.32.21"
+        },
+        "vine": {
+            "hashes": [
+                "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b",
+                "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72"
+            ],
+            "version": "==1.1.4"
         }
     },
     "develop": {

gargantext/settings.py

@@ -42,7 +42,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
-    'djcelery',
+    'django_celery_beat',
     'gargantext.backend',
 ]
 
@@ -204,15 +204,12 @@ MEDIA_URL = '/media/'
 
 # Asynchronous tasks
 
-import djcelery
-djcelery.setup_loader()
-
 BROKER_URL = config('BROKER_URL', default='amqp://guest:guest@localhost:5672/')
 
 CELERY_APP = config('CELERY_APP', default='gargantext.backend')
 CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
 CELERY_TIMEZONE = TIME_ZONE
-CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
+CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
 CELERY_IMPORTS = ()
 
 

tools/init.d/gargantext-celery

@@ -10,7 +10,7 @@
 # Description:       starts gargantext task queue using celery
 ### END INIT INFO
 
-DAEMON=$(pipenv --py)
+DAEMON=${CELERY:-$(which celery)}
 NAME=gargantext-celery
 DESC=gargantext-celery
 
@@ -34,14 +34,14 @@ celery() {
 
 	if [ "$1" = "start" ]; then
 		# Detect errors in python code
-		$DAEMON $PWD/manage.py celery worker -A "$CELERY_APP" 2> $ERROR_LOG 1> /dev/null &
+		$DAEMON worker -A "$CELERY_APP" 2> $ERROR_LOG 1> /dev/null &
 		# Wait 1 second and kill process, if it was already exited we have an
 		# error
 		sleep 1 && kill $! > /dev/null 2>&1 || \
 			( log_progress_msg "[$(tail -1 $ERROR_LOG)]" && return 2 )
 	fi
 
-	$DAEMON $PWD/manage.py celery multi $1 1 -A "$CELERY_APP" -B -E \
+	$DAEMON multi $1 1 -A "$CELERY_APP" -B -E \
 		--pidfile="$CELERYD_PID_FILE" \
 		--logfile="$CELERYD_LOG_FILE" \
 		--loglevel="$CELERYD_LOG_LEVEL" 2> $ERROR_LOG 1> /dev/null