Skip to content

gargantext
gargantext
Commit c1d89f3d authored by delanoe's avatar delanoe
Browse files
Options

[FEAT] Progress bar, worflow ok. TODO: ajax to update the progress bar without reloading page.

parent f8927c17
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 36 deletions
gargantext/util/toolchain/__init__.py
View file @ c1d89f3d
...@@ -50,11 +50,26 @@ def parse_extract_indexhyperdata(corpus): ...@@ -50,11 +50,26 @@ def parse_extract_indexhyperdata(corpus):
if corpus is None: if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id) print('NO SUCH CORPUS: #%d' % corpus_id)
return return
# Instantiate status
corpus.status('Workflow', progress=1)
corpus.save_hyperdata()
session.commit()
# apply actions # apply actions
print('CORPUS #%d' % (corpus.id)) print('CORPUS #%d' % (corpus.id))
parse(corpus) parse(corpus)
print('CORPUS #%d: parsed' % (corpus.id)) print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus) extract_ngrams(corpus)
# Preparing Databse
# Indexing
#
corpus.status('Index', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: extracted ngrams' % (corpus.id)) print('CORPUS #%d: extracted ngrams' % (corpus.id))
index_hyperdata(corpus) index_hyperdata(corpus)
print('CORPUS #%d: indexed hyperdata' % (corpus.id)) print('CORPUS #%d: indexed hyperdata' % (corpus.id))
...@@ -67,9 +82,21 @@ def parse_extract_indexhyperdata(corpus): ...@@ -67,9 +82,21 @@ def parse_extract_indexhyperdata(corpus):
session.commit() session.commit()
print('CORPUS #%d: [%s] new favorites node #%i' % (corpus.id, t(), favs.id)) print('CORPUS #%d: [%s] new favorites node #%i' % (corpus.id, t(), favs.id))
corpus.status('Index', progress=1, complete=True)
corpus.save_hyperdata()
session.commit()
# ------------------------------- # -------------------------------
# temporary ngram lists workflow # temporary ngram lists workflow
# ------------------------------- # -------------------------------
corpus.status('Lists', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: [%s] starting ngram lists computation' % (corpus.id, t())) print('CORPUS #%d: [%s] starting ngram lists computation' % (corpus.id, t()))
# -> stoplist: filter + write (to Node and NodeNgram) # -> stoplist: filter + write (to Node and NodeNgram)
...@@ -116,11 +143,20 @@ def parse_extract_indexhyperdata(corpus): ...@@ -116,11 +143,20 @@ def parse_extract_indexhyperdata(corpus):
print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id)) print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id))
print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t())) print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t()))
corpus.status('Lists', progress=0, complete=True)
corpus.save_hyperdata()
session.commit()
if DEBUG is False: if DEBUG is False:
print('CORPUS #%d: [%s] FINISHED Sendind email notification' % (corpus.id, t())) print('CORPUS #%d: [%s] FINISHED Sendind email notification' % (corpus.id, t()))
notify_owner(corpus) notify_owner(corpus)
corpus.status('Workflow', progress=10, complete=True)
corpus.save_hyperdata()
session.commit()
def t(): def t():
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
gargantext/util/toolchain/ngrams_extraction.py
View file @ c1d89f3d
...@@ -85,16 +85,16 @@ def extract_ngrams(corpus, keys=('title', 'abstract', )): ...@@ -85,16 +85,16 @@ def extract_ngrams(corpus, keys=('title', 'abstract', )):
nodes_ngrams_count.clear() nodes_ngrams_count.clear()
ngrams_data.clear() ngrams_data.clear()
if documents_count % BATCH_NGRAMSEXTRACTION_SIZE == 0: if documents_count % BATCH_NGRAMSEXTRACTION_SIZE == 0:
corpus.status('ngrams_extraction', progress=documents_count+1) corpus.status('Ngrams', progress=documents_count+1)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
# integrate ngrams and nodes-ngrams # integrate ngrams and nodes-ngrams
_integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor) _integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor)
corpus.status('ngrams_extraction', progress=documents_count+1, complete=True) corpus.status('Ngrams', progress=documents_count+1, complete=True)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
except Exception as error: except Exception as error:
corpus.status('ngrams_extraction', error=error) corpus.status('Ngrams', error=error)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
raise error raise error
......
gargantext/util/toolchain/parsing.py
View file @ c1d89f3d
...@@ -8,7 +8,7 @@ from re import sub ...@@ -8,7 +8,7 @@ from re import sub
def parse(corpus): def parse(corpus):
try: try:
documents_count = 0 documents_count = 0
corpus.status('parsing', progress=0) corpus.status('Docs', progress=0)
# will gather info about languages # will gather info about languages
observed_languages = defaultdict(int) observed_languages = defaultdict(int)
...@@ -46,7 +46,7 @@ def parse(corpus): ...@@ -46,7 +46,7 @@ def parse(corpus):
# logging # logging
if documents_count % BATCH_PARSING_SIZE == 0: if documents_count % BATCH_PARSING_SIZE == 0:
corpus.status('parsing', progress=documents_count) corpus.status('Docs', progress=documents_count)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
documents_count += 1 documents_count += 1
...@@ -57,11 +57,11 @@ def parse(corpus): ...@@ -57,11 +57,11 @@ def parse(corpus):
# ...with a special key inside for skipped languages at ngrams_extraction # ...with a special key inside for skipped languages at ngrams_extraction
corpus.hyperdata['languages']['__skipped__'] = {} corpus.hyperdata['languages']['__skipped__'] = {}
# commit all changes # commit all changes
corpus.status('parsing', progress=documents_count, complete=True) corpus.status('Docs', progress=documents_count, complete=True)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
except Exception as error: except Exception as error:
corpus.status('parsing', error=error) corpus.status('Docs', error=error)
corpus.save_hyperdata() corpus.save_hyperdata()
session.commit() session.commit()
raise error raise error
......
templates/pages/menu.html
View file @ c1d89f3d
...@@ -94,7 +94,7 @@ ...@@ -94,7 +94,7 @@
{% for state in corpus.hyperdata.statuses %} {% for state in corpus.hyperdata.statuses %}
{% if state.action == "ngrams_extraction" %} {% if state.action == "Workflow" %}
{% if state.complete %} {% if state.complete %}
<li> <li>
......
templates/pages/projects/project.html
View file @ c1d89f3d
...@@ -110,19 +110,20 @@ ...@@ -110,19 +110,20 @@
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span> <span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</button> </button>
</div> </div>
<div class="col-md-3 content"> <div class="col-md-4 content">
{% for state in corpus.hyperdata.statuses %} {% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "ngrams_extraction" %} {% ifequal state.action "Workflow" %}
{% if state.complete %} {% if state.complete %}
<span class="glyphicon glyphicon-ok" aria-hidden="true"></span> <span class="glyphicon glyphicon-ok" aria-hidden="true"></span>
{% else %} {% else %}
{% if state.error %} {% if state.error %}
<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
{{ state.error }}
{% else %} {% else %}
<div class="progress"> <div class="progress">
{% for state in corpus.hyperdata.statuses %} {% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "parsing" %} {% if state.action != "Workflow" %}
<div class=" progress-bar progress-bar-striped <div class=" progress-bar progress-bar-striped
{% if state.complete %} {% if state.complete %}
progress-bar-success progress-bar-success
...@@ -130,39 +131,18 @@ ...@@ -130,39 +131,18 @@
active active
{% endif %} {% endif %}
" "
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 50%"> role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 25%">
<span> <span>
Parsing {{ state.action }}
{% if state.complete %} {% if state.complete %}
Complete Ok
{% else %} {% else %}
Processing Processing
{% endif %} {% endif %}
</span> </span>
</div> </div>
{% endifequal %} {% endif %}
{% ifequal state.action "ngrams_extraction" %}
<div class="progress-bar progress-bar-striped
{% if state.complete %}
progress-bar-success
{% else %}
active
{% endif %}
"
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 50%">
<span>
Ngrams extraction
{% if state.complete %}
Complete
{% else %}
Processing
{% endif %}
</span>
</div>
{% endifequal %}
{% endfor %} {% endfor %}
</div> </div>
{% endif %} {% endif %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment