Commit bd991379 authored by Mathieu Rodic's avatar Mathieu Rodic

[FEAT] implemented 3 different kinds of scheduling

 - immediate
 - threaded
 - managed with Celery
(see `gargantext.util.schedule`)
parent 1757301b
from gargantext.util.db import *
from gargantext.util.files import upload
from gargantext.util import workflow
from gargantext.constants import *
from datetime import datetime
......@@ -71,5 +70,4 @@ class Node(Base):
})
session.add(corpus)
session.commit()
workflow.parse(corpus)
return corpus
"""This module defines three distinct decorators for scheduling.
"""
def scheduled_now(func):
"""Provides a decorator to execute the task right away.
Mostly useful for debugging purpose.
"""
return func
import threading
def scheduled_thread(func):
"""Provides a decorator to schedule a task as a new thread.
Problem: a shutdown may lose the task forever...
"""
def go(*args, **kwargs):
thread = threading.Thread(target=func, args=args, kwargs=kwargs)
thread.start()
return go
from celery import shared_task
def scheduled_celery(func):
"""Provides a decorator to schedule a task with Celery.
"""
@shared_task
def _func(*args, **kwargs):
func(*args, **kwargs)
def go(*args, **kwargs):
_func.apply_async(args=args, kwargs=kwargs)
return go
# scheduled = scheduled_now
# scheduled = scheduled_thread
scheduled = scheduled_celery
from celery import shared_task
from gargantext.util.db import *
from gargantext.models import *
from gargantext.util.schedule import scheduled
from time import sleep
@shared_task
def _parse(corpus_id):
print('ABOUT TO PARSE CORPUS #%d' % corpus_id)
@scheduled
def parse(corpus_id):
print('CORPUS #%d...' % (corpus_id, ))
corpus = session.query(Node).filter(Node.id == corpus_id).first()
sleep(2)
print('PARSED CORPUS #%d' % corpus_id)
def parse(corpus):
print('ABOUT TO PLAN PARSING')
_parse.apply_async((corpus.id,),)
print('PLANNED PARSING')
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
print('CORPUS #%d: %s' % (corpus_id, corpus, ))
from gargantext.util import workflow
from gargantext.util.http import *
from gargantext.util.db import *
from gargantext.util.db_cache import cache
......@@ -85,6 +86,7 @@ def project(request, project_id):
resource_type = request.POST['type'],
resource_upload = request.FILES['file'],
)
workflow.parse(corpus.id)
# corpora within this project
corpora = project.children('CORPUS').all()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment