Skip to content

gargantext
gargantext
Commit 30b3c871 authored by Romain Loth's avatar Romain Loth
Browse files
Options

[FEAT] project view: show in real time the progression of active corpora... 

[FEAT] project view: show in real time the progression of active corpora workflow progressbars (use status API at regular increasing intervals 10 times, then stop trying, parameters are configurable)
parent 2d5a6ac0
Hide whitespace changes
Inline Side-by-side
Showing with 223 additions and 7 deletions
gargantext/constants.py
View file @ 30b3c871
......@@ -382,6 +382,9 @@ BATCH_NGRAMSEXTRACTION_SIZE = 3000 # how many new node-ngram relations before
QUERY_SIZE_N_MAX = 1000
QUERY_SIZE_N_DEFAULT = 1000
# Refresh corpora workflow status for project view's progressbar
PROJECT_VIEW_REFRESH_INTERVAL = 3000 # in ms
PROJECT_VIEW_MAX_REFRESH_ATTEMPTS = 5 # how many times before we give up
# ------------------------------------------------------------------------------
# Graph <=> nodes API parameters
......
gargantext/views/pages/projects.py
View file @ 30b3c871
......@@ -54,6 +54,9 @@ def overview(request):
# projects owned by the user's contacts
'common_users': (contact for contact, projects in contacts_projects),
'common_projects': sum((projects for contact, projects in contacts_projects), []),
# status refreshing params (when active workflows)
'status_refresh_initial_interval': PROJECT_VIEW_REFRESH_INTERVAL,
'status_refresh_max_attempts': PROJECT_VIEW_MAX_REFRESH_ATTEMPTS,
},
)
......
templates/pages/projects/project.html
View file @ 30b3c871
......@@ -88,14 +88,14 @@
{{ key }}
</h2>
{% for corpus in corpora %}
<div id="corpus_{{corpus.id}}">
<div id="corpus_{{corpus.id}}" class="corpusElt">
<div class="row">
<h4>
<div class="col-md-1 content"></div>
<div class="col-md-5 content">
<a href="/projects/{{project.id}}/corpora/{{corpus.id}}">
<span class="glyphicon glyphicon-file" aria-hidden="true"></span>
{{corpus.name}}, {{ corpus.count }} documents {{ corpus.status_message }}
{{corpus.name}}, {{ corpus.count }} documents <span id="corpus_{{corpus.id}}_msg">{{ corpus.status_message }}</span>
</a>
</div>
<div class="col-md-3 content" id="corpus_{{corpus.id}}_tools">
......@@ -146,7 +146,9 @@
{% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "Workflow" %}
{% if state.complete %}
<span class="glyphicon glyphicon-ok" aria-hidden="true"></span>
<span id="corpus_{{corpus.id}}_status_ok"
class="glyphicon glyphicon-ok"
aria-hidden="true"></span>
{% else %}
{% if state.error %}
......@@ -172,7 +174,8 @@
active
{% endif %}
"
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 20%">
role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100"
id="corpus_{{corpus.id}}_status_{{state.action}}" style="width: 20%">
<span>
{{ state.action }}
{% if not state.complete %}
......@@ -305,6 +308,31 @@
</div><!-- /.modal -->
<script type="text/javascript" src="{% static "lib/jquery/1.11.2/jquery-ui.js" %}"></script>
<script type="text/javascript">
var corporaDivs = document.getElementsByClassName('corpusElt')
// all corpora ids ======================================
var corporaIds = []
for (var i = 0 ; i < corporaDivs.length ; i++) {
// ex: corpus_1198
divId = corporaDivs[i].id
if (divId) {
var corpusId = divId.match(/[0-9]+$/).pop()
corporaIds.push(corpusId)
}
}
var activeCorporaIds = testActiveCorpora()
if (activeCorporaIds.length) {
// initial checks if page reloaded with active corpora
keepCheckingProjectStatus()
}
// cookie ajax helper ==================================
function getCookie(name) {
var cookieValue = null;
if (document.cookie && document.cookie != '') {
......@@ -321,6 +349,9 @@
return cookieValue;
}
// =====================================================
// search queries and crawling =========================
// =====================================================
var thequeries = [] ;
// load the template's value for N scan size
......@@ -426,6 +457,9 @@
}
}
// schedule periodic checks of status of active corpora
keepCheckingProjectStatus()
}
function getGlobalResults(value){
......@@ -772,9 +806,6 @@
}
{% if donut %}
// Morris Donut Chart
Morris.Donut({
......@@ -793,10 +824,189 @@
});
{% endif %}
// =====================================================
$('#wait').on('hidden.bs.modal', function (e) {
// reload page when dismiss the info box
window.location.reload()
})
// =====================================================
// corpus-status checking ==============================
// =====================================================
// ------------------------------
// 1) helper progressbar function
// -------------------------------
function updateCorpusProgressbar(crid, statuses, the_status_url) {
if (statuses && statuses[0]) {
// 0 is status of whole WORKFLOW
var statusW = statuses[0]
if (statusW.complete) {
$('#corpus_'+crid+'_status').html(
'<span id="corpus_'+crid+'_status_ok" '
+ ' class="glyphicon glyphicon-ok"></span>'
)
}
// workflow incomplete: we check each action in [1,4]
else {
var subStatuses = statuses.slice(1,5)
console.warn(subStatuses)
for (var j in subStatuses) {
// stepName parmi 'Docs','Ngrams','Index','Lists'
var stepOk = subStatuses[j]['complete']
var stepError = subStatuses[j]['error']
var stepName = subStatuses[j]['action']
// debug
// console.warn(stepName)
var pgbarId = 'corpus_'+crid+'_status_'+stepName
// if error
if (stepError && stepError != 'null') {
$('#corpus_'+crid+'_status').html(
'<p class="workflow_error">'
+ 'Error in corpus parsing at step '
+ j +' (' + stepName + ')'
+ JSON.stringify(stepError) +
+ ' <a href="https://www.iscpif.fr/gargantext-feedback-and-bug-report/">'
+'(bug report here)'
+'</a></p>'
)
}
// normal cases: update progressbar ------------
else {
progressbar = document.getElementById(pgbarId)
if (progressbar) {
// console.log('updating '+pgbarId)
// A: done
if (stepOk || stepOk == "true") {
// make progressbar segment green
if (progressbar.class && progressbar.class.match('active')) {
progressbar.classList.remove("active")
progressbar.classList.add("progress-bar-success")
}
// remove message if any
document.getElementById('corpus_'+crid+'_msg').innerHTML = ""
}
// B: active
else {
progressbar.classList.add("active")
}
}
// C: new action => new bar segment
else {
// console.log('creating '+pgbarId)
barSegmentHtml = '<div class="progress-bar progress-bar-striped'
barSegmentHtml += (stepOk ? ' progress-bar-success' : ' active') + '"'
barSegmentHtml += 'role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100"'
barSegmentHtml += 'id="corpus_'+crid+'_status_'+stepName+'" style="width: 20%">'
barSegmentHtml += '<span>'+stepName
barSegmentHtml += (stepOk ? 'Processing' : '')
barSegmentHtml += '</span></div>'
$('#corpus_'+crid+'_status > .progress')
.append(barSegmentHtml)
}
}
// ---------------------------------------------
} // end detailed check
}
} // end if statuses array
else {
console.error("Wrong status API return format "
+ "for url" + the_status_url)
}
} // end function
// ---------------------------------------------------
// 2 - main status check function on activeCorporaIds
// ---------------------------------------------------
function updateCorporaStates(someCorporaIds) {
for (var i in someCorporaIds) {
var crid = someCorporaIds[i]
var the_status_url = "/api/nodes/"+crid+"/status?format=json"
// iterate ajax checks
$.ajax({
type: 'GET',
url: the_status_url,
success: function(data) {
var statuses = data['statuses']
updateCorpusProgressbar(crid, statuses, the_status_url)
},
error: function(data, s) {
console.warning("status GET !! ajax err !! " + s)
console.log(data)
},
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
}
}) // ajax: did 1 corpus
} // did all corpora
}
// -----------------------------------------------------
// 3 - for testing on client side which need refresh
// -----------------------------------------------------
function testActiveCorpora() {
var activeCorporaIds = []
for (var i in corporaIds) {
var crid = corporaIds[i]
if (! document.getElementById(
'corpus_'+crid+'_status_ok')
) {
activeCorporaIds.push(crid)
}
}
return activeCorporaIds
}
// -----------------------------------------------------
// 4 - running the check at regular intervals until done
// -----------------------------------------------------
var nChecks = 0
function keepCheckingProjectStatus() {
console.log("checking status", nChecks)
nChecks ++
// local check for active corpora
var newActiveCorporaIds = testActiveCorpora()
if (newActiveCorporaIds.length) {
// start remote calls
updateCorporaStates(newActiveCorporaIds)
if (nChecks > 5) {
// we abandon after 5 checks
console.warn("stopping status checks for corpora:",
newActiveCorporaIds)
nChecks = 0
return null
}
else {
// decreasing intervals (preserving DB while "loosing interest")
nextTime = nChecks * 3000
// schedule next check
setTimeout(keepCheckingProjectStatus, nextTime)
console.log("next status check in", nextTime/1000, "s" )
return false
}
}
else {
console.warn("OK, all corpora ingestion complete")
nChecks = 0
return true
}
}
</script>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment