Skip to content

gargantext
gargantext
Commit c1d89f3d authored by delanoe's avatar delanoe
Browse files
Options

[FEAT] Progress bar, worflow ok. TODO: ajax to update the progress bar without reloading page.

parent f8927c17
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 36 deletions
gargantext/util/toolchain/__init__.py
View file @ c1d89f3d
......@@ -50,11 +50,26 @@ def parse_extract_indexhyperdata(corpus):
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
# Instantiate status
corpus.status('Workflow', progress=1)
corpus.save_hyperdata()
session.commit()
# apply actions
print('CORPUS #%d' % (corpus.id))
parse(corpus)
print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus)
# Preparing Databse
# Indexing
#
corpus.status('Index', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: extracted ngrams' % (corpus.id))
index_hyperdata(corpus)
print('CORPUS #%d: indexed hyperdata' % (corpus.id))
......@@ -67,9 +82,21 @@ def parse_extract_indexhyperdata(corpus):
session.commit()
print('CORPUS #%d: [%s] new favorites node #%i' % (corpus.id, t(), favs.id))
corpus.status('Index', progress=1, complete=True)
corpus.save_hyperdata()
session.commit()
# -------------------------------
# temporary ngram lists workflow
# -------------------------------
corpus.status('Lists', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: [%s] starting ngram lists computation' % (corpus.id, t()))
# -> stoplist: filter + write (to Node and NodeNgram)
......@@ -116,11 +143,20 @@ def parse_extract_indexhyperdata(corpus):
print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id))
print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t()))
corpus.status('Lists', progress=0, complete=True)
corpus.save_hyperdata()
session.commit()
if DEBUG is False:
print('CORPUS #%d: [%s] FINISHED Sendind email notification' % (corpus.id, t()))
notify_owner(corpus)
corpus.status('Workflow', progress=10, complete=True)
corpus.save_hyperdata()
session.commit()
def t():
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
gargantext/util/toolchain/ngrams_extraction.py
View file @ c1d89f3d
......@@ -85,16 +85,16 @@ def extract_ngrams(corpus, keys=('title', 'abstract', )):
nodes_ngrams_count.clear()
ngrams_data.clear()
if documents_count % BATCH_NGRAMSEXTRACTION_SIZE == 0:
corpus.status('ngrams_extraction', progress=documents_count+1)
corpus.status('Ngrams', progress=documents_count+1)
corpus.save_hyperdata()
session.commit()
# integrate ngrams and nodes-ngrams
_integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor)
corpus.status('ngrams_extraction', progress=documents_count+1, complete=True)
corpus.status('Ngrams', progress=documents_count+1, complete=True)
corpus.save_hyperdata()
session.commit()
except Exception as error:
corpus.status('ngrams_extraction', error=error)
corpus.status('Ngrams', error=error)
corpus.save_hyperdata()
session.commit()
raise error
......
gargantext/util/toolchain/parsing.py
View file @ c1d89f3d
......@@ -8,7 +8,7 @@ from re import sub
def parse(corpus):
try:
documents_count = 0
corpus.status('parsing', progress=0)
corpus.status('Docs', progress=0)
# will gather info about languages
observed_languages = defaultdict(int)
......@@ -46,7 +46,7 @@ def parse(corpus):
# logging
if documents_count % BATCH_PARSING_SIZE == 0:
corpus.status('parsing', progress=documents_count)
corpus.status('Docs', progress=documents_count)
corpus.save_hyperdata()
session.commit()
documents_count += 1
......@@ -57,11 +57,11 @@ def parse(corpus):
# ...with a special key inside for skipped languages at ngrams_extraction
corpus.hyperdata['languages']['__skipped__'] = {}
# commit all changes
corpus.status('parsing', progress=documents_count, complete=True)
corpus.status('Docs', progress=documents_count, complete=True)
corpus.save_hyperdata()
session.commit()
except Exception as error:
corpus.status('parsing', error=error)
corpus.status('Docs', error=error)
corpus.save_hyperdata()
session.commit()
raise error
......
templates/pages/menu.html
View file @ c1d89f3d
......@@ -94,7 +94,7 @@
{% for state in corpus.hyperdata.statuses %}
{% if state.action == "ngrams_extraction" %}
{% if state.action == "Workflow" %}
{% if state.complete %}
<li>
......
templates/pages/projects/project.html
View file @ c1d89f3d
......@@ -110,19 +110,20 @@
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</button>
</div>
<div class="col-md-3 content">
<div class="col-md-4 content">
{% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "ngrams_extraction" %}
{% ifequal state.action "Workflow" %}
{% if state.complete %}
<span class="glyphicon glyphicon-ok" aria-hidden="true"></span>
{% else %}
{% if state.error %}
<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
{{ state.error }}
{% else %}
<div class="progress">
{% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "parsing" %}
{% if state.action != "Workflow" %}
<div class=" progress-bar progress-bar-striped
{% if state.complete %}
progress-bar-success
......@@ -130,39 +131,18 @@
active
{% endif %}
"
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 50%">
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 25%">
<span>
Parsing
{{ state.action }}
{% if state.complete %}
Complete
Ok
{% else %}
Processing
{% endif %}
</span>
</div>
{% endifequal %}
{% ifequal state.action "ngrams_extraction" %}
<div class="progress-bar progress-bar-striped
{% if state.complete %}
progress-bar-success
{% else %}
active
{% endif %}
"
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 50%">
<span>
Ngrams extraction
{% if state.complete %}
Complete
{% else %}
Processing
{% endif %}
</span>
</div>
{% endifequal %}
{% endif %}
{% endfor %}
</div>
{% endif %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment