Skip to content

gargantext
gargantext
Commit f0766f59 authored by Administrator's avatar Administrator
Browse files
Options

[FEATURE] asynchronous methods (still buggy)

parent afb5f90b
Hide whitespace changes
Inline Side-by-side
Showing with 182 additions and 8 deletions
AlexandreTests.ipynb
View file @ f0766f59
{
"metadata": {
"name": "",
"signature": "sha256:395704b4d4eb714ff9d76fa24d7fbb0b926e58cb7f734a64d3558232ea342e75"
"signature": "sha256:2fb7f3051bdf9ffb1f88229c3a00227ce26c8394f4db8de3de2e59e220d93bbf"
},
"nbformat": 3,
"nbformat_minor": 0,
......@@ -2496,21 +2496,165 @@
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Asynchrone"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"input": [
"from celery import Celery\n",
"\n",
"app = Celery('tasks', broker='amqp://guest@localhost:5672//', backend='amqp://guest@localhost:5672//',)"
],
"language": "python",
"metadata": {},
"outputs": []
"outputs": [],
"prompt_number": 145
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"input": [
"app.conf.CELERY_TASK_SERIALIZER = 'json'\n",
"app.conf.update(\n",
" CELERY_TASK_SERIALIZER='json',\n",
" CELERY_ACCEPT_CONTENT=['json'], # Ignore other content\n",
" CELERY_RESULT_SERIALIZER='json',\n",
" CELERY_TIMEZONE='Europe/Oslo',\n",
" CELERY_ENABLE_UTC=True,\n",
")"
],
"language": "python",
"metadata": {},
"outputs": []
"outputs": [],
"prompt_number": 146
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"@app.task\n",
"def addition(x, y):\n",
" return x + y"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 147
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from node.models import addition"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from celerytest.tasks import Test"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"t = Test()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"res = t.addition.delay((),)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"res = t.addition.apply_async((2,2), countdown=2)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"res.get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "addition() missing 1 required positional argument: 'y'",
"output_type": "pyerr",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-4-8bb969b0b8af>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/celery/result.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout, propagate, interval, no_ack, follow_parents, EXCEPTION_STATES, PROPAGATE_STATES)\u001b[0m\n\u001b[0;32m 173\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'status'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 174\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mPROPAGATE_STATES\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mpropagate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 175\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 176\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mEXCEPTION_STATES\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 177\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mTypeError\u001b[0m: addition() missing 1 required positional argument: 'y'"
]
}
],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from celery.contrib.methods import current_app"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"app."
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 6,
"text": [
"False"
]
}
],
"prompt_number": 6
}
],
"metadata": {}
......
celerytest/admin.py 0 → 100644
View file @ f0766f59
from django.contrib import admin
# Register your models here.
celerytest/models.py 0 → 100644
View file @ f0766f59
from django.db import models
# Create your models here.
celerytest/tasks.py 0 → 100644
View file @ f0766f59
from celery import task
from celery.contrib.methods import task_method
from celery import current_app
class Test(object):
@current_app.task(filter=task_method)
def addition(self):
#return "hello"
#return int(x) + int(y)
celerytest/tests.py 0 → 100644
View file @ f0766f59
from django.test import TestCase
# Create your tests here.
celerytest/views.py 0 → 100644
View file @ f0766f59
from django.shortcuts import render
# Create your views here.
gargantext_web/settings.py
View file @ f0766f59
......@@ -14,6 +14,10 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
PROJECT_PATH = os.path.join(BASE_DIR, os.pardir)
PROJECT_PATH = os.path.abspath(PROJECT_PATH)
import djcelery
djcelery.setup_loader()
BROKER_URL = 'amqp://guest:guest@localhost:5672/'
CELERY_IMPORTS=("node.models","celerytest.tasks")
# Quick-start development settings - unsuitable for production
......@@ -64,6 +68,7 @@ INSTALLED_APPS = (
'node',
'ngram',
'django_hstore',
'djcelery',
)
MIDDLEWARE_CLASSES = (
......@@ -143,7 +148,5 @@ TEMPLATE_CONTEXT_PROCESSORS = (
)
# grappelli custom
GRAPPELLI_ADMIN_TITLE = "Gargantext"
gargantext_web/views.py
View file @ f0766f59
......@@ -228,7 +228,7 @@ def project(request, project_id):
)
try:
corpus.parse_resources()
corpus.parse_resources.apply_async((), countdown=10)
except Exception as error:
print(error)
......
node/models.py
View file @ f0766f59
......@@ -15,6 +15,8 @@ import hashlib
from gargantext_web.settings import MEDIA_ROOT
from celery.contrib.methods import task
# Some usefull functions
# TODO: start the function name with an underscore (private)
def _upload_to(instance, filename):
......@@ -120,6 +122,7 @@ class Node(CTENode):
node_resource.save()
return resource
@task()
def parse_resources(self):
# parse all resources into a list of metadata
metadata_list = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment