Skip to content

gargantext
gargantext
Commit 9641f8c5 authored by Administrator's avatar Administrator
Browse files
Options

[FEATURE] Asynchronous task for parsing! OK :)

parent f0766f59
Hide whitespace changes
Inline Side-by-side
Showing with 59 additions and 36 deletions
AlexandreTests.ipynb
View file @ 9641f8c5
{ {
"metadata": { "metadata": {
"name": "", "name": "",
"signature": "sha256:2fb7f3051bdf9ffb1f88229c3a00227ce26c8394f4db8de3de2e59e220d93bbf" "signature": "sha256:0e63832a6b33d476c8b284b72b0740bd9ade357e5ebb1f73bdc399bbd2824a16"
}, },
"nbformat": 3, "nbformat": 3,
"nbformat_minor": 0, "nbformat_minor": 0,
...@@ -2506,11 +2506,7 @@ ...@@ -2506,11 +2506,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
"input": [ "input": [],
"from celery import Celery\n",
"\n",
"app = Celery('tasks', broker='amqp://guest@localhost:5672//', backend='amqp://guest@localhost:5672//',)"
],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
...@@ -2519,16 +2515,7 @@ ...@@ -2519,16 +2515,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
"input": [ "input": [],
"app.conf.CELERY_TASK_SERIALIZER = 'json'\n",
"app.conf.update(\n",
" CELERY_TASK_SERIALIZER='json',\n",
" CELERY_ACCEPT_CONTENT=['json'], # Ignore other content\n",
" CELERY_RESULT_SERIALIZER='json',\n",
" CELERY_TIMEZONE='Europe/Oslo',\n",
" CELERY_ENABLE_UTC=True,\n",
")"
],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
...@@ -2537,27 +2524,52 @@ ...@@ -2537,27 +2524,52 @@
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
"input": [ "input": [],
"@app.task\n",
"def addition(x, y):\n",
" return x + y"
],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 147 "prompt_number": 147
}, },
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
"input": [ "input": [
"from node.models import addition" "from celerytest.tasks import add"
], ],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 1
}, },
{
"cell_type": "code",
"collapsed": false,
"input": [
"add."
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 3,
"text": [
"<AsyncResult: c7df5232-b80a-4dd4-b615-432a6fb206e4>"
]
}
],
"prompt_number": 3
},
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
...@@ -2567,7 +2579,7 @@ ...@@ -2567,7 +2579,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 5
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -2578,18 +2590,18 @@ ...@@ -2578,18 +2590,18 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 2 "prompt_number": 6
}, },
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
"input": [ "input": [
"res = t.addition.delay((),)" "res = t.addition.delay((2,2))"
], ],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 4 "prompt_number": 7
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -2600,7 +2612,7 @@ ...@@ -2600,7 +2612,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 5 "prompt_number": 8
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -2613,17 +2625,17 @@ ...@@ -2613,17 +2625,17 @@
"outputs": [ "outputs": [
{ {
"ename": "TypeError", "ename": "TypeError",
"evalue": "addition() missing 1 required positional argument: 'y'", "evalue": "addition() takes 2 positional arguments but 3 were given",
"output_type": "pyerr", "output_type": "pyerr",
"traceback": [ "traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-4-8bb969b0b8af>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[1;32m<ipython-input-5-8bb969b0b8af>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/celery/result.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout, propagate, interval, no_ack, follow_parents, EXCEPTION_STATES, PROPAGATE_STATES)\u001b[0m\n\u001b[0;32m 173\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'status'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 174\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mPROPAGATE_STATES\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mpropagate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 175\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 176\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mEXCEPTION_STATES\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 177\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/celery/result.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout, propagate, interval, no_ack, follow_parents, EXCEPTION_STATES, PROPAGATE_STATES)\u001b[0m\n\u001b[0;32m 173\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'status'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 174\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mPROPAGATE_STATES\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mpropagate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 175\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 176\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mEXCEPTION_STATES\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 177\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mTypeError\u001b[0m: addition() missing 1 required positional argument: 'y'" "\u001b[1;31mTypeError\u001b[0m: addition() takes 2 positional arguments but 3 were given"
] ]
} }
], ],
"prompt_number": 4 "prompt_number": 5
}, },
{ {
"cell_type": "code", "cell_type": "code",
......
celerytest/task_simple.py 0 → 100644
View file @ 9641f8c5
from celery import task
@task()
def add(x, y):
return x + y
celerytest/tasks.py
View file @ 9641f8c5
...@@ -5,8 +5,11 @@ from celery import current_app ...@@ -5,8 +5,11 @@ from celery import current_app
class Test(object): class Test(object):
@current_app.task(filter=task_method) @current_app.task(filter=task_method)
def addition(self): def addition(self, x, y):
#return "hello" #return "hello"
#return int(x) + int(y) return int(x) + int(y)
@current_app.task()
def add(x, y):
return x + y
gargantext_web/settings.py
View file @ 9641f8c5
...@@ -17,7 +17,7 @@ PROJECT_PATH = os.path.abspath(PROJECT_PATH) ...@@ -17,7 +17,7 @@ PROJECT_PATH = os.path.abspath(PROJECT_PATH)
import djcelery import djcelery
djcelery.setup_loader() djcelery.setup_loader()
BROKER_URL = 'amqp://guest:guest@localhost:5672/' BROKER_URL = 'amqp://guest:guest@localhost:5672/'
CELERY_IMPORTS=("node.models","celerytest.tasks") CELERY_IMPORTS=("node.models","celerytest.tasks", "celerytest.task_simple")
# Quick-start development settings - unsuitable for production # Quick-start development settings - unsuitable for production
......
node/models.py
View file @ 9641f8c5
...@@ -15,7 +15,9 @@ import hashlib ...@@ -15,7 +15,9 @@ import hashlib
from gargantext_web.settings import MEDIA_ROOT from gargantext_web.settings import MEDIA_ROOT
from celery.contrib.methods import task from celery.contrib.methods import task_method
from celery import current_app
# Some usefull functions # Some usefull functions
# TODO: start the function name with an underscore (private) # TODO: start the function name with an underscore (private)
...@@ -122,7 +124,7 @@ class Node(CTENode): ...@@ -122,7 +124,7 @@ class Node(CTENode):
node_resource.save() node_resource.save()
return resource return resource
@task() @current_app.task(filter=task_method)
def parse_resources(self): def parse_resources(self):
# parse all resources into a list of metadata # parse all resources into a list of metadata
metadata_list = [] metadata_list = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment