Skip to content

gargantext
gargantext
Commit 8fcfc803 authored by delanoe's avatar delanoe
Browse files
Options

Merge branch 'romain-refactoring' into unstable

parents a01472c3 8ba25bfd
Hide whitespace changes
Inline Side-by-side
Showing with 817 additions and 494 deletions
gargantext/constants.py
View file @ 8fcfc803
......@@ -13,10 +13,12 @@ LISTTYPES = {
'MAINLIST' : UnweightedList,
'MAPLIST' : UnweightedList,
'SPECIFICITY' : WeightedList,
'OCCURRENCES' : WeightedIndex, # todo replace by WeightedList
'OCCURRENCES' : WeightedIndex, # could be WeightedList
'COOCCURRENCES': WeightedMatrix,
'TFIDF-CORPUS' : WeightedIndex, # todo split -> WeightedList for ti_rank and WeightedIndex for tfidf
'TFIDF-GLOBAL' : WeightedIndex, # todo split -> WeightedList for ti_rank and WeightedIndex for tfidf
'TFIDF-CORPUS' : WeightedIndex,
'TFIDF-GLOBAL' : WeightedIndex,
'TIRANK-LOCAL' : WeightedIndex, # could be WeightedList
'TIRANK-GLOBAL' : WeightedIndex # could be WeightedList
}
NODETYPES = [
......@@ -40,8 +42,11 @@ NODETYPES = [
'TFIDF-CORPUS', # 13
'TFIDF-GLOBAL', # 14
# docs subset
'FAVORITES' # 15
# TODO add ti RANK
'FAVORITES', # 15
# more scores (sorry!)
'TIRANK-LOCAL', # 16
'TIRANK-GLOBAL', # 17
]
INDEXED_HYPERDATA = {
......
gargantext/util/toolchain/__init__.py
View file @ 8fcfc803
from gargantext.settings import DEBUG
from .parsing import parse
from .ngrams_extraction import extract_ngrams
from .hyperdata_indexing import index_hyperdata
# in usual run order
from .list_stop import do_stoplist
from .ngram_groups import compute_groups
from .metric_tfidf import compute_occs, compute_tfidf_local, compute_ti_ranking
from .list_main import do_mainlist
from .ngram_coocs import compute_coocs
from .metric_specificity import compute_specificity
from .list_map import do_maplist # TEST
from .mail_notification import notify_owner
from gargantext.util.db import session
from gargantext.models import Node
from datetime import datetime
from celery import shared_task
#@shared_task
def parse_extract(corpus):
# retrieve corpus from database from id
if isinstance(corpus, int):
corpus_id = corpus
corpus = session.query(Node).filter(Node.id == corpus_id).first()
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
# apply actions
print('CORPUS #%d' % (corpus.id))
parse(corpus)
# was there an error in the process ?
if corpus.status()['error']:
print("ERROR: aborting parse_extract for corpus #%i" % corpus_id)
return None
print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus)
print('CORPUS #%d: extracted ngrams' % (corpus.id))
@shared_task
def parse_extract_indexhyperdata(corpus):
# retrieve corpus from database from id
if isinstance(corpus, int):
corpus_id = corpus
corpus = session.query(Node).filter(Node.id == corpus_id).first()
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
# Instantiate status
corpus.status('Workflow', progress=1)
corpus.save_hyperdata()
session.commit()
# FIXME: 'Workflow' will still be uncomplete when 'Index' and 'Lists' will
# get stacked into hyperdata['statuses'], but doing corpus.status()
# will return only the 1st uncomplete action (corpus.status() doesn't
# understand "subactions")
# apply actions
print('CORPUS #%d' % (corpus.id))
parse(corpus)
print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus)
# Preparing Databse
# Indexing
#
corpus.status('Index', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: extracted ngrams' % (corpus.id))
index_hyperdata(corpus)
print('CORPUS #%d: indexed hyperdata' % (corpus.id))
# -> 'favorites' node
favs = corpus.add_child(
typename='FAVORITES', name='favorite docs in "%s"' % corpus.name
)
session.add(favs)
session.commit()
print('CORPUS #%d: [%s] new favorites node #%i' % (corpus.id, t(), favs.id))
corpus.status('Index', progress=1, complete=True)
corpus.save_hyperdata()
session.commit()
# -------------------------------
# temporary ngram lists workflow
# -------------------------------
corpus.status('Lists', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: [%s] starting ngram lists computation' % (corpus.id, t()))
# -> stoplist: filter + write (to Node and NodeNgram)
stop_id = do_stoplist(corpus)
print('CORPUS #%d: [%s] new stoplist node #%i' % (corpus.id, t(), stop_id))
# -> write groups to Node and NodeNgramNgram
group_id = compute_groups(corpus, stoplist_id = None)
print('CORPUS #%d: [%s] new grouplist node #%i' % (corpus.id, t(), group_id))
# ------------
# -> write occurrences to Node and NodeNodeNgram # (todo: NodeNgram)
occ_id = compute_occs(corpus, groupings_id = group_id)
print('CORPUS #%d: [%s] new occs node #%i' % (corpus.id, t(), occ_id))
# -> write cumulated ti_ranking (tfidf ranking vector) to Node and NodeNodeNgram (todo: NodeNgram)
tirank_id = compute_ti_ranking(corpus,
groupings_id = group_id,
count_scope="global")
print('CORPUS #%d: [%s] new ti ranking node #%i' % (corpus.id, t(), tirank_id))
# -> mainlist: filter + write (to Node and NodeNgram)
mainlist_id = do_mainlist(corpus,
ranking_scores_id = tirank_id,
stoplist_id = stop_id)
print('CORPUS #%d: [%s] new mainlist node #%i' % (corpus.id, t(), mainlist_id))
# -> write local tfidf similarities to Node and NodeNodeNgram
ltfidf_id = compute_tfidf_local(corpus,
on_list_id=mainlist_id,
groupings_id = group_id)
print('CORPUS #%d: [%s] new localtfidf node #%i' % (corpus.id, t(), ltfidf_id))
# => used for doc <=> ngram association
# ------------
# -> cooccurrences on mainlist: compute + write (=> Node and NodeNgramNgram)
coocs = compute_coocs(corpus,
on_list_id = mainlist_id,
groupings_id = group_id,
just_pass_result = True)
print('CORPUS #%d: [%s] computed mainlist coocs for specif rank' % (corpus.id, t()))
# -> specificity: compute + write (=> NodeNodeNgram)
spec_id = compute_specificity(corpus,cooc_matrix = coocs)
# no need here for subforms because cooc already counted them in mainform
print('CORPUS #%d: [%s] new specificity node #%i' % (corpus.id, t(), spec_id))
# maplist: compute + write (to Node and NodeNgram)
map_id = do_maplist(corpus,
mainlist_id = mainlist_id,
specificity_id=spec_id,
grouplist_id=group_id
)
print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id))
print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t()))
corpus.status('Lists', progress=0, complete=True)
corpus.save_hyperdata()
session.commit()
if DEBUG is False:
print('CORPUS #%d: [%s] FINISHED Sending email notification' % (corpus.id, t()))
notify_owner(corpus)
corpus.status('Workflow', progress=10, complete=True)
corpus.save_hyperdata()
session.commit()
def t():
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
from .main import parse_extract_indexhyperdata
gargantext/util/toolchain/main.py 0 → 100644
View file @ 8fcfc803
from gargantext.settings import DEBUG
from .parsing import parse
from .ngrams_extraction import extract_ngrams
from .hyperdata_indexing import index_hyperdata
# in usual run order
from .list_stop import do_stoplist
from .ngram_groups import compute_groups
from .metric_tfidf import compute_occs, compute_tfidf_local, compute_ti_ranking
from .list_main import do_mainlist
from .ngram_coocs import compute_coocs
from .metric_specificity import compute_specificity
from .list_map import do_maplist # TEST
from .mail_notification import notify_owner
from gargantext.util.db import session
from gargantext.models import Node
from datetime import datetime
from celery import shared_task
#@shared_task
def parse_extract(corpus):
# retrieve corpus from database from id
if isinstance(corpus, int):
corpus_id = corpus
corpus = session.query(Node).filter(Node.id == corpus_id).first()
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
# apply actions
print('CORPUS #%d' % (corpus.id))
parse(corpus)
# was there an error in the process ?
if corpus.status()['error']:
print("ERROR: aborting parse_extract for corpus #%i" % corpus_id)
return None
print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus)
print('CORPUS #%d: extracted ngrams' % (corpus.id))
@shared_task
def parse_extract_indexhyperdata(corpus):
# retrieve corpus from database from id
if isinstance(corpus, int):
corpus_id = corpus
corpus = session.query(Node).filter(Node.id == corpus_id).first()
if corpus is None:
print('NO SUCH CORPUS: #%d' % corpus_id)
return
# Instantiate status
corpus.status('Workflow', progress=1)
corpus.save_hyperdata()
session.commit()
# FIXME: 'Workflow' will still be uncomplete when 'Index' and 'Lists' will
# get stacked into hyperdata['statuses'], but doing corpus.status()
# will return only the 1st uncomplete action (corpus.status() doesn't
# understand "subactions")
# apply actions
print('CORPUS #%d' % (corpus.id))
parse(corpus)
print('CORPUS #%d: parsed' % (corpus.id))
extract_ngrams(corpus)
# Preparing Databse
# Indexing
#
corpus.status('Index', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: extracted ngrams' % (corpus.id))
index_hyperdata(corpus)
print('CORPUS #%d: indexed hyperdata' % (corpus.id))
# -> 'favorites' node
favs = corpus.add_child(
typename='FAVORITES', name='favorite docs in "%s"' % corpus.name
)
session.add(favs)
session.commit()
print('CORPUS #%d: [%s] new favorites node #%i' % (corpus.id, t(), favs.id))
corpus.status('Index', progress=1, complete=True)
corpus.save_hyperdata()
session.commit()
# -------------------------------
# temporary ngram lists workflow
# -------------------------------
corpus.status('Lists', progress=0)
corpus.save_hyperdata()
session.commit()
print('CORPUS #%d: [%s] starting ngram lists computation' % (corpus.id, t()))
# -> stoplist: filter + write (to Node and NodeNgram)
stop_id = do_stoplist(corpus)
print('CORPUS #%d: [%s] new stoplist node #%i' % (corpus.id, t(), stop_id))
# -> write groups to Node and NodeNgramNgram
group_id = compute_groups(corpus, stoplist_id = None)
print('CORPUS #%d: [%s] new grouplist node #%i' % (corpus.id, t(), group_id))
# ------------
# -> write occurrences to Node and NodeNodeNgram
occ_id = compute_occs(corpus, groupings_id = group_id)
print('CORPUS #%d: [%s] new occs node #%i' % (corpus.id, t(), occ_id))
# -> write cumulated ti_ranking (tfidf ranking vector) to Node and NodeNodeNgram
tirank_id = compute_ti_ranking(corpus,
groupings_id = group_id,
count_scope="global")
print('CORPUS #%d: [%s] new ti ranking node #%i' % (corpus.id, t(), tirank_id))
# -> mainlist: filter + write (to Node and NodeNgram)
mainlist_id = do_mainlist(corpus,
ranking_scores_id = tirank_id,
stoplist_id = stop_id)
print('CORPUS #%d: [%s] new mainlist node #%i' % (corpus.id, t(), mainlist_id))
# -> write local tfidf similarities to Node and NodeNodeNgram
ltfidf_id = compute_tfidf_local(corpus,
on_list_id=mainlist_id,
groupings_id = group_id)
print('CORPUS #%d: [%s] new localtfidf node #%i' % (corpus.id, t(), ltfidf_id))
# => used for doc <=> ngram association
# ------------
# -> cooccurrences on mainlist: compute + write (=> Node and NodeNgramNgram)
coocs = compute_coocs(corpus,
on_list_id = mainlist_id,
groupings_id = group_id,
just_pass_result = True)
print('CORPUS #%d: [%s] computed mainlist coocs for specif rank' % (corpus.id, t()))
# -> specificity: compute + write (=> NodeNodeNgram)
spec_id = compute_specificity(corpus,cooc_matrix = coocs)
# no need here for subforms because cooc already counted them in mainform
print('CORPUS #%d: [%s] new specificity node #%i' % (corpus.id, t(), spec_id))
# maplist: compute + write (to Node and NodeNgram)
map_id = do_maplist(corpus,
mainlist_id = mainlist_id,
specificity_id=spec_id,
grouplist_id=group_id
)
print('CORPUS #%d: [%s] new maplist node #%i' % (corpus.id, t(), map_id))
print('CORPUS #%d: [%s] FINISHED ngram lists computation' % (corpus.id, t()))
corpus.status('Lists', progress=0, complete=True)
corpus.save_hyperdata()
session.commit()
if DEBUG is False:
print('CORPUS #%d: [%s] FINISHED Sending email notification' % (corpus.id, t()))
notify_owner(corpus)
corpus.status('Workflow', progress=10, complete=True)
corpus.save_hyperdata()
session.commit()
@shared_task
def recount(corpus):
"""
Recount essential metrics of the toolchain after group modifications.
==> updates all scores in terms table
==> updates tfidf relationship b/w term and doc
When groups change, the metrics need to be updated because subforms must be
added to their new mainform aggregate values:
- occurrences
- ndocs
- ti_rank
- coocs
- specificity
- tfidf
NB: no new extraction, no list change, just the metrics
"""
# 1) we'll need the new groups and mainlist as basis
group_id = corpus.children("GROUPLIST").first().id
mainlist_id = corpus.children("MAINLIST").first().id
# 2) and we're going to overwrite the previous metric nodes
try:
old_occ_id = corpus.children("OCCURRENCES").first().id
except:
old_occ_id = None
try:
old_tirank_id = corpus.children("TIRANK-GLOBAL").first().id
except:
old_tirank_id = None
try:
old_spec_id = corpus.children("SPECIFICITY").first().id
except:
old_spec_id = None
try:
old_ltfidf_id = corpus.children("TFIDF-CORPUS").first().id
except:
old_ltfidf_id = None
# 3) we redo the required toolchain parts
# -------------------------------------------
# Instantiate status
corpus.status('Recounting mini-workflow', progress=1)
corpus.save_hyperdata()
session.commit()
# -> overwrite occurrences (=> NodeNodeNgram)
occ_id = compute_occs(corpus,
groupings_id = group_id,
overwrite_id=old_occ_id)
print('RECOUNT #%d: [%s] updated occs node #%i' % (corpus.id, t(), occ_id))
# -> write cumulated ti_ranking (tfidf ranking vector) (=> NodeNodeNgram)
tirank_id = compute_ti_ranking(corpus,
groupings_id = group_id,
count_scope="global",
overwrite_id=old_tirank_id)
print('RECOUNT #%d: [%s] updated ti ranking node #%i' % (corpus.id, t(), tirank_id))
# -> write local tfidf similarities to (=> NodeNodeNgram)
ltfidf_id = compute_tfidf_local(corpus,
on_list_id = mainlist_id,
groupings_id = group_id,
overwrite_id = old_ltfidf_id)
print('RECOUNT #%d: [%s] updated localtfidf node #%i' % (corpus.id, t(), ltfidf_id))
# => used for doc <=> ngram association
# ------------
# -> cooccurrences on mainlist: compute + write (=> NodeNgramNgram)
coocs = compute_coocs(corpus,
on_list_id = mainlist_id,
groupings_id = group_id,
just_pass_result = True)
print('RECOUNT #%d: [%s] updated mainlist coocs for specif rank' % (corpus.id, t()))
# -> specificity: compute + write (=> NodeNgram)
spec_id = compute_specificity(corpus,cooc_matrix = coocs, overwrite_id = old_spec_id)
print('RECOUNT #%d: [%s] updated specificity node #%i' % (corpus.id, t(), spec_id))
print('RECOUNT #%d: [%s] FINISHED metric recounts' % (corpus.id, t()))
corpus.status('Recounting mini-workflow', progress=10, complete=True)
corpus.save_hyperdata()
session.commit()
def t():
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
gargantext/util/toolchain/metric_specificity.py
View file @ 8fcfc803
......@@ -99,7 +99,7 @@ def compute_specificity(corpus, cooc_id=None, cooc_matrix=None, overwrite_id = N
if overwrite_id:
# overwrite pre-existing id
the_id = overwrite_id
session.query(NodeNodeNgram).filter(NodeNodeNgram.node1_id==the_id).delete()
session.query(NodeNgram).filter(NodeNgram.node_id==the_id).delete()
session.commit()
else:
specnode = corpus.add_child(
......
gargantext/util/toolchain/metric_tfidf.py
View file @ 8fcfc803
......@@ -345,11 +345,11 @@ def compute_ti_ranking(corpus,
# create the new TFIDF-XXXX node to get an id
tir_nd = corpus.add_child()
if count_scope == "local":
tir_nd.typename = "TFIDF-CORPUS"
tir_nd.typename = "TIRANK-CORPUS"
tir_nd.name = "ti rank (%i ngforms in corpus:%s)" % (
total_ngramforms, corpus_id)
elif count_scope == "global":
tir_nd.typename = "TFIDF-GLOBAL"
tir_nd.typename = "TIRANK-GLOBAL"
tir_nd.name = "ti rank (%i ngforms %s in corpora of sourcetype:%s)" % (
total_ngramforms,
("from corpus %i" % corpus_id) if (termset_scope == "local") else "" ,
......
gargantext/views/api/metrics.py 0 → 100644
View file @ 8fcfc803
from gargantext.util.db_cache import cache
from gargantext.util.http import ValidationException, APIView \
, HttpResponse, JsonHttpResponse
from gargantext.util.toolchain.main import recount
from datetime import datetime
class CorpusMetrics(APIView):
def patch(self, request, corpusnode_id):
"""
PATCH triggers recount of metrics for the specified corpus.
ex PATCH http://localhost:8000/api/metrics/14072
-----
corpus_id
"""
print("==> update metrics request on ", corpusnode_id)
if not request.user.is_authenticated():
# can't use @requires_auth because of positional 'self' within class
return HttpResponse('Unauthorized', status=401)
try:
corpus = cache.Node[int(corpusnode_id)]
except:
corpus = None
if corpus is None:
raise ValidationException("%s is not a valid corpus node id."
% corpusnode_id)
else:
t_before = datetime.now()
# =============
recount(corpus)
# =============
t_after = datetime.now()
return JsonHttpResponse({
'corpus_id' : corpusnode_id,
'took': "%f s." % (t_after - t_before).total_seconds()
})
gargantext/views/api/ngramlists.py
View file @ 8fcfc803
......@@ -34,24 +34,36 @@ def _query_list(list_id,
"""
if not details:
# simple contents
query = session.query(NodeNgram.ngram_id)
query = session.query(NodeNgram.ngram_id).filter(NodeNgram.node_id == list_id)
else:
# detailed contents (terms and some NodeNodeNgram for score)
# NB: score can be undefined (eg ex-subform that now became free)
# ==> we need outerjoin
# and the filter needs to have scoring_metric_id so we do it before
ScoresTable = (session
.query(NodeNodeNgram.score, NodeNodeNgram.ngram_id)
.filter(NodeNodeNgram.node1_id == scoring_metric_id)
.subquery()
)
query = (session
.query(
NodeNgram.ngram_id,
Ngram.terms,
NodeNodeNgram.score
ScoresTable.c.score
)
.join(Ngram, NodeNgram.ngram_id == Ngram.id)
.join(NodeNodeNgram, NodeNgram.ngram_id == NodeNodeNgram.ngram_id)
.filter(NodeNodeNgram.node1_id == scoring_metric_id)
.order_by(desc(NodeNodeNgram.score))
)
# main filter
# -----------
query = query.filter(NodeNgram.node_id == list_id)
# main filter ----------------------
.filter(NodeNgram.node_id == list_id)
# scores if possible
.outerjoin(ScoresTable,
ScoresTable.c.ngram_id == NodeNgram.ngram_id)
.order_by(desc(ScoresTable.c.score))
)
if pagination_limit:
query = query.limit(pagination_limit)
......@@ -128,13 +140,18 @@ class GroupChange(APIView):
}
Chained effect:
any previous group under mainformA or B will be overwritten
The DELETE HTTP method also works, with same url
(and simple array in the data)
NB: request.user is also checked for current authentication status
"""
def initial(self, request):
"""
Before dispatching to post()
Before dispatching to post() or delete()
Checks current user authentication to prevent remote DB manipulation
"""
......@@ -150,28 +167,29 @@ class GroupChange(APIView):
=> removes couples where newly reconnected ngrams where involved
=> adds new couples from GroupsBuffer of terms view
TODO recalculate scores after new groups
TODO see use of util.lists.Translations
TODO benchmark selective delete compared to entire list rewrite
POST data:
<QueryDict: {'1228[]': ['891', '1639']}> => creates 1228 - 891
and 1228 - 1639
request.POST.lists() iterator where each elt is like :('1228[]',['891','1639'])
"""
group_node = get_parameters(request)['node']
all_nodes_involved = []
all_mainforms = []
links = []
for (mainform_key, subforms_ids) in request.POST.lists():
mainform_id = mainform_key[:-2] # remove brackets '543[]' -> '543'
all_nodes_involved.append(mainform_id)
all_mainforms.append(mainform_id)
for subform_id in subforms_ids:
links.append((mainform_id,subform_id))
all_nodes_involved.append(subform_id)
# remove selectively all groupings with these nodes involved
# TODO benchmark
# remove selectively all groupings with these mainforms
# using IN is correct in this case: list of ids is short and external
# see stackoverflow.com/questions/444475/
old_links = (session.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id == group_node)
.filter(or_(
NodeNgramNgram.ngram1_id.in_(all_nodes_involved),
NodeNgramNgram.ngram2_id.in_(all_nodes_involved)))
.filter(NodeNgramNgram.ngram1_id.in_(all_mainforms))
)
n_removed = old_links.delete(synchronize_session=False)
session.commit()
......@@ -189,6 +207,40 @@ class GroupChange(APIView):
}, 200)
def delete(self, request):
"""
Deletes some groups from the group node
Send in data format is simply a json { 'keys':'["11492","16438"]' }
==> it means removing any synonym groups having these 2 as mainform
(within the url's groupnode_id)
NB: At reception here it becomes like:
<QueryDict: {'keys[]': ['11492', '16438']}>
"""
# from the url
group_node = get_parameters(request)['node']
print(request.POST)
# from the data in body
all_mainforms = request.POST.getlist('keys[]')
links_to_remove = (session.query(NodeNgramNgram)
.filter(NodeNgramNgram.node_id == group_node)
.filter(NodeNgramNgram.ngram1_id.in_(all_mainforms))
)
n_removed = links_to_remove.delete(synchronize_session=False)
session.commit()
return JsonHttpResponse({
'count_removed': n_removed
}, 200)
class ListChange(APIView):
"""
......
gargantext/views/api/nodes.py
View file @ 8fcfc803
......@@ -89,7 +89,7 @@ class NodeListResource(APIView):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="Gargantext_Corpus.csv"'
writer = csv.writer(response, delimiter='\t')
writer = csv.writer(response, delimiter='\t', quoting=csv.QUOTE_MINIMAL)
keys = [ 'title' , 'journal'
, 'publication_year', 'publication_month', 'publication_day'
......
gargantext/views/api/urls.py
View file @ 8fcfc803
from django.conf.urls import url
from . import nodes
from . import metrics
from . import ngramlists
from . import analytics
......@@ -19,6 +20,14 @@ urlpatterns = [ url(r'^nodes$' , nodes.NodeListResource.as_view()
, url(r'^nodes/(\d+)/favorites$', nodes.CorpusFavorites.as_view() )
# in these two routes the node is supposed to be a *corpus* node
, url(r'^metrics/(\d+)$', metrics.CorpusMetrics.as_view() )
# update all metrics for a corpus
# ex: PUT metrics/123
# \
# corpus id
, url(r'^ngramlists/change$', ngramlists.ListChange.as_view() )
# add or remove ngram from a list
# ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
......
static/lib/gargantext/NGrams_dyna_chart_and_table.js
View file @ 8fcfc803
......@@ -22,6 +22,30 @@
* The stateIds are described by the System object.
* - columns use stateId [0..2] (miam aka normal, map aka keep, stop aka delete)
*
*
* These states are stored in 3 places :
* Original (in *OriginalNG*)
* Current (in *AjaxRecords*)
* Changes to make (in "FlagsBuffer")
*
* (the crud operations create FlagsBuffer at the last moment by
* inspecting the diff of original vs current and infering changes)
*
* For the groups it's a little different with only 2 main vars:
* Current (in CurrentGroups)
* Changes to make (in "GroupsBuffer")
*
* (each user action is enacted in Current and at the same time carried
* over to GroupsBuffer)
*
* NB for groups removal: there is a case when user clicks "cancel" in group modifs
* but some modifications will already be triggered:
* it happens when an entire group A is attached to a new
* group B, and then later an ex-element of A is removed
* from the new group
* => even if the new group B is canceled, it will mark
* the entire old group A for destruction
*
* @author
* Samuel Castillo (original 2015 work)
* Romain Loth
......@@ -44,12 +68,19 @@
// =============================================================================
// ngram infos (<-> row data)
// --------------------------
// current ngram infos (<-> row data)
// ----------------------------------
// from /api/ngramlists/lexmodel?corpus=312
// with some expanding in AfterAjax
var AjaxRecords = [] ;
// current groups
// --------------
// links and subforms (reverse index)
var CurrentGroups = {"links":{}, "subs":{}}
// table element (+config +events)
// -------------------------------
var MyTable ;
......@@ -108,7 +139,15 @@ for(var i in System[GState]["states"] ) {
var FlagsBuffer = {}
// + 1 for groups
GroupsBuffer = {}
var GroupsBuffer = {'_to_add':{}, '_to_del':{}}
// to_add will have structure like this { mainformid1 : [array1 of subformids],
// mainformid2 : [array2 of subformids]}
// to_del is simple array of mainforms : [mainform3, mainform4]
// because deleting "the group of mainform3" is deleting all DB rows of
// couples having mainform3 on the left-hand-side ( <=> no need to know the subform)
// GROUP "WINDOWS"
......@@ -239,22 +278,22 @@ function printCorpuses() {
if(Object.keys( results ).length>0) {
var sub_ngrams_data = {
"ngrams":[],
"scores": $.extend({}, NGrams["main"].scores)
"scores": $.extend({}, OriginalNG.scores)
}
for(var i in NGrams["main"].ngrams) {
if( results[ NGrams["main"].ngrams[i].id] ) {
var a_ngram = NGrams["main"].ngrams[i]
sub_ngrams_data["ngrams"].push( a_ngram )
for(var i in OriginalNG["records"].ngrams) {
if( results[ OriginalNG["records"].ngrams[i].id] ) {
var a_ngram = OriginalNG["records"].ngrams[i]
sub_ngrams_data["records"].push( a_ngram )
}
// if( results[ NGrams["main"].ngrams[i].id] && NGrams["main"].ngrams[i].name.split(" ").length==1 ) {
// if( NGrams["map"][ NGrams["main"].ngrams[i].id] ) {
// var a_ngram = NGrams["main"].ngrams[i]
// if( results[ OriginalNG["records"][i].id] && OriginalNG["records"][i].name.split(" ").length==1 ) {
// if( OriginalNG["map"][ i ] ) {
// var a_ngram = OriginalNG["records"][i]
// // a_ngram["state"] = System[0]["statesD"]["delete"]
// sub_ngrams_data["ngrams"].push( a_ngram )
// }
// }
}
var result = MainTableAndCharts(sub_ngrams_data , NGrams["main"].scores.initial , "filter_all")
var result = MainTableAndCharts(sub_ngrams_data , OriginalNG.scores.initial , "filter_all")
}
});
}
......@@ -362,40 +401,30 @@ function saveActiveGroup() {
var mainform = activeGroup.now_mainform_id
// pr("saving mainform to GroupsBuffer: " + mainform)
// the new array to save is in now_links -------------
GroupsBuffer[mainform] = activeGroup.now_links
// ---------------------------------------------------
// the new array of tgt forms to save is in now_links
addInCurrentGroups(mainform, activeGroup.now_links) // -> current status
GroupsBuffer._to_add[mainform] = activeGroup.now_links // -> changes to make
// console.log(AjaxRecords[mainform])
// -----------------------------------------------------
// various consequences
// also we prefix "*" to the name if not already there
if (AjaxRecords[mainform].name[0] != '*') {
AjaxRecords[mainform].name = "*" + AjaxRecords[mainform].name
}
// the previous mainforms that became subforms can't stay in the main records
for (downgradedNgramId in activeGroup.were_mainforms) {
if (downgradedNgramId != mainform) {
AjaxRecords[downgradedNgramId].state = -1
// they go to nodesmemory
// NGrams.group.nodesmemory = AjaxRecords[downgradedNgramId]
// delete AjaxRecords[downgradedNgramId]
}
// all subforms old and new become deactivated in AjaxRecords
for (var i in activeGroup.now_links) {
subformId = activeGroup.now_links[i]
AjaxRecords[subformId].state = -1
}
// TODO posttest
// the previous "old" links are now in GroupsBuffer so from now on
// they'll be searched in AjaxRecords by updateActiveGroupInfo()
delete NGrams.group.links[mainform]
for (i in activeGroup.now_links) {
newLink = activeGroup.now_links[i] ;
if (activeGroup.ngraminfo[newLink].origin == 'old' || activeGroup.ngraminfo[newLink].origin == 'oldnew') {
// new AjaxRecords entry from nodesmemory
AjaxRecords[newLink] = NGrams.group.nodesmemory[newLink]
delete NGrams.group.nodesmemory[newLink]
// console.log('oldLinkThatBecameNew: '+AjaxRecords[newLink].name)
// the previous mainforms have old groupings to be marked for deletion
for (downgradedNgramId in activeGroup.were_mainforms) {
// (except the one we just saved)
if (downgradedNgramId != mainform) {
GroupsBuffer._to_del[downgradedNgramId] = true
deleteInCurrentGroups(downgradedNgramId, false) // "false" <=> no need to change subform states
}
}
......@@ -404,6 +433,9 @@ function saveActiveGroup() {
}
function removeActiveGroupFrame() {
// no need to restore records: everything from the frame
// was in temporary var activeGroup
// erases now_links and restores empty activeGroup global cache
activeGroup = {'now_mainform_id':undefined, 'were_mainforms':{}} ;
......@@ -447,7 +479,7 @@ function toggleSeeGroup(plusicon, ngramId) {
* @param ngramId (of the mainform)
*/
function seeGroup ( ngramId , allowChangeFlag) {
// 1/7 create new element container
// 1/6 create new element container
var subNgramHtml = $('<p class="note">') ;
subNgramHtml.attr("id", "subforms-"+ngramId) ;
subNgramHtml.css("line-height", 1.2) ;
......@@ -455,32 +487,24 @@ function seeGroup ( ngramId , allowChangeFlag) {
subNgramHtml.css("margin-top", '.5em') ;
// 2/7 attach flag open to global state register
// 2/6 attach flag open to global state register
vizopenGroup[ngramId] = true ;
// 3/7 retrieve names of the untouched (from DB) grouped ngrams (aka "old")
var oldlinksNames = [] ;
if( ngramId in NGrams.group.links ) {
for (var i in NGrams.group.links[ngramId]) {
var subNgramId = NGrams.group.links[ngramId][i] ;
oldlinksNames[i] = NGrams.group.nodesmemory[subNgramId].name
}
}
// 4/7 retrieve names of the newly created grouped ngrams (aka "new" + "oldnew")
var newlinksNames = [] ;
if( ngramId in GroupsBuffer ) {
for(var i in GroupsBuffer[ ngramId ] ) {
var subNgramId = GroupsBuffer[ ngramId ][i] ;
newlinksNames[i] = AjaxRecords[subNgramId].name
}
// 3/6 retrieve names of the grouped ngrams
var linksNames = [] ;
if( ngramId in CurrentGroups["links"] ) {
var thisGroup = CurrentGroups["links"][ngramId]
for (var i in thisGroup) {
var subNgramId = thisGroup[i] ;
linksNames[i] = AjaxRecords[subNgramId].name
}
}
// 5/7 create the "tree" from the names, as html lines
var htmlMiniTree = drawSublist(oldlinksNames.concat(newlinksNames))
// 4/6 create the "tree" from the names, as html lines
var htmlMiniTree = drawSublist(linksNames)
subNgramHtml.append(htmlMiniTree)
// 6/7 add a "modify group" button
// 5/6 add a "modify group" button
if (allowChangeFlag) {
var changeGroupsButton = '<button style="float:right"' ;
changeGroupsButton += ' title="add/remove contents of groups"' ;
......@@ -490,7 +514,7 @@ function seeGroup ( ngramId , allowChangeFlag) {
subNgramHtml.append(changeGroupsButton) ;
}
// 7/7 return html snippet (ready for rendering)
// 6/6 return html snippet (ready for rendering)
return(subNgramHtml)
}
......@@ -562,27 +586,21 @@ function subformSpan( subNgramInfo ) {
span = $('<span/>', {
text: subNgramInfo.name,
title: subNgramInfo.id,
id: 'active-subform-' + subNgramInfo.id
id: 'active-subform-' + subNgramInfo.id,
class: 'subform'
})
if (subNgramInfo.origin == 'old') {
span.addClass("oldsubform")
}
else if (subNgramInfo.origin == 'new' || subNgramInfo.origin == 'oldnew'){
span.addClass("usersubform")
}
// remove button
// var removeButton = '&nbsp;<span class="note glyphicon glyphicon-minus-sign"'
// removeButton += ' title="remove from group (/!\\ bug: will be unattached if was previously a subform)"' ;
// removeButton += ' onclick="removeSubform('+ subNgramInfo.id +')"></span>'
// span.append(removeButton)
var removeButton = '<span class="note glyphicon glyphicon-minus-sign"'
removeButton += ' title="remove from group"' ;
removeButton += ' onclick="removeSubform('+ subNgramInfo.id +')"></span> &nbsp;'
span.prepend(removeButton)
// makes this subform become the mainform
// var mainformButton = '&nbsp;<span class="note glyphicon glyphicon-circle-arrow-up"'
// var mainformButton = '<span class="note glyphicon glyphicon-circle-arrow-up"'
// mainformButton += ' title="upgrade to mainform of this group"'
// mainformButton += ' onclick="makeMainform('+ subNgramInfo.id +')"></span>'
// span.append(mainformButton)
// mainformButton += ' onclick="makeMainform('+ subNgramInfo.id +')"></span>&nbsp;'
// span.prepend(mainformButton)
return(span[0].outerHTML)
}
......@@ -610,7 +628,6 @@ function makeMainform(ngramId) {
activeGroup.now_links[i] = previousMainformId
// if it was previously a subform then:
// -> it had no entry in AjaxRecords
// -> it was not in any of the lists
if (! (mainform in activeGroup.were_mainforms)) {
// update records
......@@ -640,21 +657,47 @@ function makeMainform(ngramId) {
function removeSubform(ngramId) {
// no need to restore AjaxRecords[ngramId] because it's untouched
// NB removeSubform has a side effect for subforms that were previously
// in another group see comment at the top of script
// special case: if removed form already was a subform it becomes independant
//
// (because the old mainform may be remaining in the new group, we set a
// convention: entire previous group goes "to_delete" at 1st sub remove)
//
if (CurrentGroups["subs"][ngramId]) {
var oldMainFormId = CurrentGroups["subs"][ngramId]
// it must break the old group, mark for deletion
GroupsBuffer._to_del[oldMainFormId] = true
// consequences:
deleteInCurrentGroups(oldMainFormId, true)
// => they are all removed from CurrentGroups
// => the removed subform and all others from the old group
// get a state (map/del/normal)
}
// ==========================================
// core of the function for any type of ngram
// ==========================================
$('#active-subform-'+ngramId).remove()
if (activeGroup.now_links.length == 1) {
// close the frame if last subform
removeActiveGroupFrame()
}
else {
// clean were_mainforms dict
delete activeGroup.were_mainforms[ngramId]
// clean now_links array
var i = activeGroup.now_links.indexOf(ngramId)
activeGroup.now_links.splice(i,1)
// if (activeGroup.ngraminfo[ngramId].origin == 'new') {
// AjaxRecords[ngramId].state = 0 ;
// }
// clean were_mainforms dict (if key existed)
delete activeGroup.were_mainforms[ngramId]
// redraw active group_box_content
drawActiveGroup(
......@@ -668,6 +711,59 @@ function removeSubform(ngramId) {
}
}
/**
* Effects of deleting a mainform from the current groups
*
* => updates the global var CurrentGroups
*
* @param ngramId of a mainform
* @param inheritState boolean => updates the AjaxRecords[subformId].state
*/
function deleteInCurrentGroups(ngramId, inheritState) {
if (inheritState) {
// ex-subforms can inherit state from their previous mainform
var implicitState = AjaxRecords[ngramId].state
}
if (CurrentGroups.links[ngramId]) {
var oldGroup = CurrentGroups.links[ngramId]
// deleting in reverse index
for (var i in oldGroup) {
var subformId = oldGroup[i]
delete CurrentGroups.subs[subformId]
if (inheritState) {
AjaxRecords[subformId].state = implicitState
// consequence:
// now OriginalNG.records[subformId].state
// is != AjaxRecords[subformId].state
// therefore the newly independant forms
// will be added to their new wordlist
}
}
// deleting in "links"
delete CurrentGroups.links[ngramId]
}
}
/**
* Adding links to the current groups
*
* => updates the global var CurrentGroups
*
* @param mainformId
* @param subforms array of subNgramIds
*/
function addInCurrentGroups(mainformId, subforms) {
console.log("addInCurrentGroups: "+mainformId+"("+JSON.stringify(subforms)+")")
CurrentGroups["links"][mainformId] = subforms
for (var i in subforms) {
var subformId = subforms[i]
CurrentGroups["subs"][subformId] = mainformId
}
}
function modifyGroup ( mainFormNgramId ) {
// create modification container
//
......@@ -706,9 +802,9 @@ function modifyGroup ( mainFormNgramId ) {
// add new ngramid (and any present subforms) to currently modified group
function add2group ( ngramId ) {
function addToGroup ( ngramId ) {
// console.log("FUN add2group(" + AjaxRecords[ngramId].name + ")")
// console.log("FUN addToGroup(" + AjaxRecords[ngramId].name + ")")
var toOther = true ;
activeGroup.were_mainforms[ngramId] = true ;
......@@ -718,7 +814,6 @@ function add2group ( ngramId ) {
// add this mainform as a new subform
activeGroup.now_links.push(ngramId)
activeGroup.ngraminfo[ngramId] = AjaxRecords[ngramId]
activeGroup.ngraminfo[ngramId].origin = 'new'
// also add all its subforms as new subforms
updateActiveGroupInfo (ngramId, toOther)
......@@ -736,14 +831,13 @@ function add2group ( ngramId ) {
else {
console.warn("ADD2GROUP but no active group")
}
}
/**
* subforms from DB have their info in a separate NGrams.group.nodesmemory
* so here and in saveActiveGroup we need to take it into account
* subforms from DB have their info in AjaxRecords like everyone, and state = -1
*
* TODO: remove this mecanism
* here all current infos are copied to activeGroup temporary var
* (copy will remain until user cancel/finishes group modif)
*
* @param ngramId
* @param toOtherMainform = flag if ngram was a subform of another mainform
......@@ -754,23 +848,11 @@ function updateActiveGroupInfo (ngramId, toOtherMainform) {
// console.log(activeGroup)
// fill active link info
if( ngramId in NGrams.group.links ) {
for (var i in NGrams.group.links[ngramId]) {
var subId = NGrams.group.links[ngramId][i] ;
// ----------- old links (already in DB)
activeGroup.now_links.push(subId)
activeGroup.ngraminfo[subId] = NGrams.group.nodesmemory[subId]
activeGroup.ngraminfo[subId].origin = toOtherMainform ? 'oldnew' : 'old'
}
}
if( ngramId in GroupsBuffer ) {
for(var i in GroupsBuffer[ ngramId ] ) {
var subId = GroupsBuffer[ ngramId ][i] ;
// ----------- new links (not in DB)
activeGroup.now_links.push(subId)
activeGroup.ngraminfo[subId] = AjaxRecords[subId]
activeGroup.ngraminfo[subId].origin = 'new'
}
for(var i in CurrentGroups["links"][ ngramId ] ) {
var subId = CurrentGroups["links"][ ngramId ][i] ;
// ----------- links (old and new)
activeGroup.now_links.push(subId)
activeGroup.ngraminfo[subId] = AjaxRecords[subId]
}
}
......@@ -881,12 +963,12 @@ function transformContent(ngramId) {
}
// GState = 1 situation: button allows to add to active group
// (if previously had add2group button clicked)
// (if previously had addToGroup button clicked)
if(GState==1 ) {
if(ngram_info.state!=System[0]["statesD"]["delete"] && ! GroupsBuffer[ngramId]) { // if deleted and already group, no Up button
if(ngram_info.state!=System[0]["statesD"]["delete"] && ! GroupsBuffer._to_add[ngramId]) { // if deleted and already group, no Up button
plus_event = '<span class="note glyphicon glyphicon-plus"'
plus_event += ' color="#FF530D"'
plus_event += ' onclick="add2group('+ ngramId +')"></span>'
plus_event += ' onclick="addToGroup('+ ngramId +')"></span>'
}
}
......@@ -1219,6 +1301,19 @@ function InferCRUDFlags(id, oldState, desiredState, registry) {
}
}
}
// (if previously was under a group)
else if (oldState === state_skip) {
if (desiredState === state_main || desiredState === state_map) {
registry["inmain"][id] = true
// (... and one more action only if is now desired to be in MAP)
if(desiredState === state_map) {
registry["inmap"][ id ] = true
}
}
else if(desiredState === state_stop) {
registry["indel"][id] = true
}
}
// (if previously was in MAIN)
else {
if(desiredState === state_map) {
......@@ -1230,6 +1325,8 @@ function InferCRUDFlags(id, oldState, desiredState, registry) {
}
}
}
console.log("registry")
console.log(registry)
return registry
}
......@@ -1260,13 +1357,13 @@ function SaveLocalChanges() {
// LOOP on all mainforms + subforms
// --------------------------------
// we use 2 globals to evaluate change-of-state
// => NGrams for old states (as in DB)
// => OriginalNG for old states (as in DB)
// => AjaxRecords for current (desired) states
for(var id in AjaxRecords) {
var oldState = 0 ;
if (NGrams["map"][ id ] ) oldState = 1
else if (NGrams["stop"][ id ]) oldState = 2
if (OriginalNG["map"][ id ] ) oldState = 1
else if (OriginalNG["stop"][ id ]) oldState = 2
var mainNewState = AjaxRecords[id]["state"] ;
......@@ -1275,26 +1372,22 @@ function SaveLocalChanges() {
FlagsBuffer = InferCRUDFlags(id, oldState, mainNewState, FlagsBuffer)
}
console.log( "dont do anything" )
return "dont do anything"
// [ = = = = propagating to subforms = = = = ]
// if change in mainform list or change in groups
if(oldState != mainNewState || GroupsBuffer[id]) {
// linked nodes
var linkedNodes ;
// a) retrieve the untouched (from DB) grouped ngrams (aka "old")
if(NGrams.group.links[id]) linkedNodes = NGrams.group.links[id]
// b) or retrieve the new linked nodes (aka "new" + "oldnew")
else if( GroupsBuffer[id] ) linkedNodes = GroupsBuffer[id]
if(oldState != mainNewState || GroupsBuffer._to_add[id]) {
for (var i in linkedNodes) {
var subNgramId = linkedNodes[i] ;
// linked nodes
for (var i in CurrentGroups["links"][id]) {
var subNgramId = CurrentGroups["links"][id][i] ;
// todo check (if undefined old state, should add to main too...)
var subOldState = undefined ;
if (NGrams["map"][ subNgramId ] ) subOldState = System[0]["statesD"]["keep"]
else if (NGrams["stop"][ subNgramId ]) subOldState = System[0]["statesD"]["delete"]
if (OriginalNG["map"][ subNgramId ] ) subOldState = System[0]["statesD"]["keep"]
else if (OriginalNG["stop"][ subNgramId ]) subOldState = System[0]["statesD"]["delete"]
else {
subOldState = System[0]["statesD"]["normal"] ;
// (special legacy case: subforms can have oldStates == undefined,
......@@ -1405,7 +1498,7 @@ function SaveLocalChanges() {
console.log("===> AJAX CRUD6 RmStop <===\n") ;
CRUD( stoplist_id , Object.keys(FlagsBuffer["outdel"]), "DELETE" , function(success) {
if (success) {
CRUD_7_groups() // chained AJAX 6 -> 7
CRUD_7_AddGroups() // chained AJAX 6 -> 7
}
else {
console.warn('CRUD error on ngrams remove from stoplist ('+stoplist_id+')')
......@@ -1413,14 +1506,28 @@ function SaveLocalChanges() {
});
}
// add to groups reading data from GroupsBuffer
function CRUD_7_groups() {
console.log("===> AJAX CRUD7 RewriteGroups <===\n") ;
GROUPCRUD(groupnode_id, GroupsBuffer, function(success) {
// (also removes previous groups with same mainforms!)
function CRUD_7_AddGroups() {
console.log("===> AJAX CRUD7 AddGroups <===\n") ;
GROUPCRUDS(groupnode_id, GroupsBuffer._to_add, "POST" , function(success) {
if (success) {
CRUD_8_RmGroups() // chained AJAX 7 -> 8
}
else {
console.warn('CRUD error on groups modification ('+groupnode_id+')')
}
}) ;
}
// add to groups reading data from GroupsBuffer
function CRUD_8_RmGroups() {
console.log("===> AJAX CRUD8 RmGroups <===\n") ;
GROUPCRUDS(groupnode_id, GroupsBuffer._to_del, "DELETE", function(success) {
if (success) {
window.location.reload() // all 7 CRUDs OK => refresh whole page
window.location.reload() // all 8 CRUDs OK => refresh whole page
}
else {
console.warn('CRUD error on ngrams add to group node ('+groupings_id+')')
console.warn('CRUD error on groups removal ('+groupnode_id+')')
}
}) ;
}
......@@ -1464,36 +1571,54 @@ function CRUD( list_id , ngram_ids , http_method , callback) {
// For group modifications (POST: {mainformA: [subformsA1,A2,A3], mainformB:..})
function GROUPCRUD( groupnode_id , post_data , callback) {
// ngramlists/change?node_id=42&ngram_ids=1,2
// (adds new groups for mainformA and mainformB)
// (also deletes any old group under A or B)
//
// (DEL: {"keys": [mainformX, mainformY]})
// (just deletes the old groups of X and Y)
function GROUPCRUDS( groupnode_id , send_object, http_method , callback) {
// ngramlists/groups?node=9
var the_url = window.location.origin+"/api/ngramlists/groups?node="+groupnode_id;
// debug
// console.log(" ajax target: " + the_url + " (" + http_method + ")")
// array of the keys
var mainformIds = Object.keys(send_object)
$.ajax({
method: 'POST',
url: the_url,
data: post_data, // currently all data explicitly in the url (like a GET)
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data){
console.log("-- GROUPCRUD ----------")
console.log("POST ok!!")
console.log(JSON.stringify(data))
console.log("-----------------------")
callback(true);
},
error: function(result) {
console.log("-- GROUPCRUD ----------")
console.log("AJAX Error on POST " + the_url);
console.log(result)
console.log("-----------------------")
callback(false);
}
});
var send_data ;
// if DELETE we need only them keys
if (http_method == "DELETE") {
send_data = {"keys": mainformIds} ;
}
else {
send_data = send_object ;
}
if(mainformIds.length > 0) {
$.ajax({
method: http_method,
url: the_url,
data: send_data,
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data){
console.log("-- GROUPCRUD ----------")
console.log(http_method + " ok!!")
console.log(JSON.stringify(data))
console.log("-----------------------")
callback(true);
},
error: function(result) {
console.log("-- GROUPCRUD ----------")
console.log("AJAX Error on " + http_method + " " + the_url);
console.log(result)
console.log("------------------")
callback(false);
}
});
} else callback(true);
}
......@@ -1509,7 +1634,7 @@ function GROUPCRUD( groupnode_id , post_data , callback) {
* 3. Creates the scores distribution chart over table
* 4. Set up Search div
*
* @param ngdata: a response from the api/node/CID/ngrams/list/ routes
* @param ngdata: OriginalNG['records']
* @param initial: initial score type "occs" or "tfidf"
* @param search_filter: value among {0,1,2,'reset'} (see #picklistmenu options)
*/
......@@ -1518,24 +1643,24 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
// debug
// alert("refresh main")
console.log("")
console.log(" = = = = MainTableAndCharts: = = = = ")
console.log("ngdata:")
console.log(ngdata)
console.log("initial:") //
console.log(initial)
console.log("search_filter:") // eg 'filter_all'
console.log(search_filter)
console.log(" = = = = / MainTableAndCharts: = = = = ")
console.log("")
// Expected infos in "ngdata.ngrams" should have the form:
// console.log("")
// console.log(" = = = = MainTableAndCharts: = = = = ")
// console.log("ngdata:")
// console.log(ngdata)
// console.log("initial:") //
// console.log(initial)
// console.log("search_filter:") // eg 'filter_all'
// console.log(search_filter)
// console.log(" = = = = / MainTableAndCharts: = = = = ")
// console.log("")
// Expected infos in "ngdata" should have the form:
// { "1": { id: "1", name: "réalité", score: 36 },
// "9": { id: "9", name: "pdg", score: 116 },
// "10": { id:"10", name: "infrastructure", score: 12 } etc. }
// (see filling of rec_info below)
// console.log(ngdata.ngrams)
// console.log(ngdata)
var DistributionDict = {}
for(var i in DistributionDict)
......@@ -1616,21 +1741,21 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
$('#delAll').data("columnSelection", 'SOME')
$('#mapAll').data("columnSelection", 'SOME')
var div_stats = "<p>";
for(var i in ngdata.scores) {
var value = (!isNaN(Number(ngdata.scores[i])))? Number(ngdata.scores[i]).toFixed(1) : ngdata.scores[i];
div_stats += i+": "+value+" | "
}
div_stats += "</p>"
$("#stats").html(div_stats)
// var div_stats = "<p>";
// for(var i in ngscores) {
// var value = (!isNaN(Number(ngscores[i])))? Number(ngscores[i]).toFixed(1) : ngscores[i];
// div_stats += i+": "+value+" | "
// }
// div_stats += "</p>"
// $("#stats").html(div_stats)
AjaxRecords = {}
for(var id in ngdata.ngrams) {
for(var id in ngdata) {
// console.log(i)
// console.log(ngdata.ngrams[i])
var le_ngram = ngdata.ngrams[id] ;
// console.log(ngdata[i])
var le_ngram = ngdata[id] ;
// INIT records
// one record <=> one line in the table + ngram states
......@@ -1643,7 +1768,7 @@ function MainTableAndCharts( ngdata , initial , search_filter) {
"state": (le_ngram.state)?le_ngram.state:0,
// properties enabling to see old and new groups
"group_exists": (le_ngram.id in NGrams.group.links || le_ngram.id in GroupsBuffer),
"group_exists": (le_ngram.id in CurrentGroups["links"])
}
// AjaxRecords.push(rec_info)
AjaxRecords[id] = rec_info
......@@ -1960,12 +2085,12 @@ function GET_( url , callback ) {
// [ = = = = = = = = = = INIT = = = = = = = = = = ]
// http://localhost:8000/api/node/84592/ngrams?format=json&score=tfidf,occs&list=miam
var corpus_id = getIDFromURL( "corpora" )
var NGrams = {
"group" : {},
var OriginalNG = {
"records" : {},
"stop" : {},
"main" : {},
"map" : {},
"scores" : {}
"scores" : {},
"links" : {}
}
......@@ -1982,51 +2107,47 @@ var final_url = window.location.origin+"/api/ngramlists/family?corpus="+corpus_i
GET_(final_url, HandleAjax)
function HandleAjax(res, sourceUrl) {
//£TODO unify with AfterAjax
if (res && res.ngraminfos) {
main_ngrams_objects = {}
// = = = = MIAM = = = = //
OriginalNG["records"] = {}
for (var ngram_id in res.ngraminfos) {
var ngram_tuple = res.ngraminfos[ngram_id]
main_ngrams_objects[ngram_id] = {
OriginalNG["records"][ngram_id] = {
'id' : ngram_id, // redundant but for backwards compat
'name' : ngram_tuple[0],
'score' : ngram_tuple[1]
}
}
// = = = = MIAM = = = = //
NGrams["main"] = {
"ngrams": main_ngrams_objects,
"scores": {
OriginalNG["scores"] = {
"initial":"occs",
"nb_ngrams":Object.keys(main_ngrams_objects).length,
"nb_ngrams":Object.keys(OriginalNG["records"]).length,
}
} ;
// = = MAP ALSO STOP = = //
// 2x(array of ids) ==> 2x(lookup hash)
NGrams["map"] = {} ;
OriginalNG["map"] = {} ;
for (var i in res.listmembers.maplist) {
var map_ng_id = res.listmembers.maplist[i] ;
NGrams["map"][map_ng_id] = true ;
OriginalNG["map"][map_ng_id] = true ;
}
NGrams["stop"] = {} ;
OriginalNG["stop"] = {} ;
for (var i in res.listmembers.stoplist) {
var stop_ng_id = res.listmembers.stoplist[i] ;
NGrams["stop"][stop_ng_id] = true ;
OriginalNG["stop"][stop_ng_id] = true ;
}
// = = = = GROUP = = = = //
NGrams["group"] = {
"links" : res.links ,
// "nodesmemory" will be filled from "links" in AfterAjax()
"nodesmemory" : {}
};
// they go directly to "Current" var (we need not keep the original situation)
CurrentGroups["links"] = res.links ;
}
// console.log('after init NGrams["main"].ngrams')
// console.log(NGrams["main"].ngrams)
// console.log('after init OriginalNG["records"]')
// console.log(OriginalNG["records"])
// cache all DB node_ids
$("input#mainlist_id").val(res.nodeids['mainlist'])
......@@ -2039,86 +2160,73 @@ function HandleAjax(res, sourceUrl) {
function AfterAjax(sourceUrl) {
// -------------------------------------------------------------------
// console.log(JSON.stringify(NGrams))
// console.log(JSON.stringify(OriginalNG))
// -------------------------------------------------------------------
state_skip = -1 // -1
state_main = System[0]["statesD"]["normal"] // 0
state_map = System[0]["statesD"]["keep"] // 1
state_stop = System[0]["statesD"]["delete"] // 2
// ----------------------------------------- MAPLIST
// keepstateId = 1
keepstateId = System[0]["statesD"]["keep"]
if( Object.keys(NGrams["map"]).length>0 ) {
for(var ngram_id in NGrams["map"]) {
myNgramInfo = NGrams["main"].ngrams[ngram_id]
// initialize state of maplist items
myNgramInfo["state"] = keepstateId ;
if( Object.keys(OriginalNG["map"]).length>0 ) {
for(var ngram_id in OriginalNG["map"]) {
myNgramInfo = OriginalNG["records"][ngram_id]
if (typeof myNgramInfo == "undefined") {
console.error("record of ngram " + ngram_id + " is undefined")
}
else {
// initialize state of maplist items
myNgramInfo["state"] = state_map ;
}
}
}
// ----------------------------------------- STOPLIST
// delstateId = 2
delstateId = System[0]["statesD"]["delete"]
if( Object.keys(NGrams["stop"]).length>0 ) {
for(var ngram_id in NGrams["stop"]) {
console.log('stopping ' + ngram_id)
myNgramInfo = NGrams["main"].ngrams[ngram_id]
// initialize state of stoplist items
myNgramInfo["state"] = delstateId ;
if( Object.keys(OriginalNG["stop"]).length>0 ) {
for(var ngram_id in OriginalNG["stop"]) {
myNgramInfo = OriginalNG["records"][ngram_id]
if (typeof myNgramInfo == "undefined") {
console.error("record of ngram " + ngram_id + " is undefined")
}
else {
// initialize state of stoplist items
myNgramInfo["state"] = state_stop ;
}
}
}
// Deleting subforms from the ngrams-table, clean start baby!
if( Object.keys(NGrams["group"].links).length>0 ) {
// subforms inventory { "main":{ all mainform ids } , "sub":{ all subform ids} }
var _forms = { "main":{} , "sub":{} }
for(var ngramId in NGrams["group"].links) {
_forms["main"][ngramId] = true
for(var i in NGrams["group"].links[ngramId]) {
var subformId = NGrams["group"].links[ngramId][i]
// for each subform: true
_forms["sub"][ subformId ] = true
// Deactivating subforms from the ngrams-table, clean start baby!
if( Object.keys(CurrentGroups["links"]).length>0 ) {
// init global actualized subform inventory (reverse index of links)
// (very useful to find what to change if group is split)
for(var ngramId in CurrentGroups["links"]) {
for(var i in CurrentGroups["links"][ngramId]) {
var subformId = CurrentGroups["links"][ngramId][i]
// for each subform: mainform
CurrentGroups["subs"][ subformId ] = ngramId
}
}
// ------------------------------------------- MAINLIST
// ngrams_data_ will update NGrams.main.ngrams (with subforms removed)
var ngrams_data_ = {}
for(var ngram_id in NGrams["main"].ngrams) {
// if ngram is subform of another
if(_forms["sub"][ngram_id]) {
// move subform info into NGrams.group.nodesmemory
// ------------------------------------------
// (subform goes away from new list but info preserved)
// (useful if we want to see/revive subforms in future)
NGrams.group.nodesmemory[ngram_id] = NGrams["main"].ngrams[ngram_id]
// debug:
// console.log(ngram_id + " ("+NGrams["main"].ngrams[ngram_id].name+") is a subform")
}
// normal case
else {
// we keep the info untouched in the new obj
ngrams_data_[ngram_id] = NGrams["main"].ngrams[ngram_id]
}
}
// use it to deactivate <=> hidden state for all them subforms
for (var subNgramId in CurrentGroups["subs"]) {
// the new hash of ngrams replaces the old main
NGrams["main"].ngrams = ngrams_data_;
// will allow us to distinguish it from mainlist items that
// have original state undefined (in InferCRUDFlags)
OriginalNG['records'][subNgramId]['state'] = state_skip
}
}
// NB: this miamlist will eventually become AjaxRecords
// debug:
// console.log('NGrams["main"]')
// console.log( NGrams["main"] )
// Building the Score-Selector //NGrams["scores"]
var FirstScore = NGrams["main"].scores.initial
// Building the Score-Selector //OriginalNG["scores"]
var FirstScore = OriginalNG.scores.initial
// TODO scores_div
// Recreate possible_scores from some constants (tfidf, occs)
// and not from ngrams[0], to keep each ngram's info smaller
// var possible_scores = Object.keys( NGrams["main"].ngrams[0].scores );
// var possible_scores = Object.keys( OriginalNG["main"].ngrams[0].scores );
// var scores_div = '<br><select style="font-size:25px;" class="span1" id="scores_selector">'+"\n";
// scores_div += "\t"+'<option value="'+FirstScore+'">'+FirstScore+'</option>'+"\n"
// for( var i in possible_scores ) {
......@@ -2131,22 +2239,11 @@ function AfterAjax(sourceUrl) {
termsfilter = (sourceUrl == final_url) ? "reset" : "1"
// Initializing the Charts and Table ---------------------------------------
var result = MainTableAndCharts( NGrams["main"] , FirstScore , termsfilter) ;
var result = MainTableAndCharts(OriginalNG["records"], FirstScore , termsfilter) ;
console.log( result ) // OK
// -------------------------------------------------------------------------
// see TODO scores_div
// Listener for onchange Score-Selector
// scores_div += "<select>"+"\n";
// $("#ScoresBox").html(scores_div)
// $("#scores_selector").on('change', function() {
// console.log( this.value )
// var result = MainTableAndCharts( NGrams["main"] , this.value , "filter_all")
// console.log( result )
//
// });
$("#content_loader").remove()
$(".nav-tabs a").click(function(e){
......
static/lib/gargantext/garganrest.js
View file @ 8fcfc803
......@@ -69,10 +69,11 @@ var Resource = function(url_path) {
});
};
// change an item
this.change = this.update = function(item, callback) {
this.change = this.update = function(id, callback) {
$.ajax({
url: url_path + '/' + item.id,
url: url_path + '/' + id,
type: 'PATCH',
success: callback
});
};
......@@ -84,14 +85,18 @@ var Resource = function(url_path) {
$.ajax({
url: url_path + '/' + id,
type: 'DELETE',
success: callback
});
};
// add an item
this.add = this.append = function(value, callback) {
$.ajax({
// todo define id
url: url_path + '/' + id,
type: 'POST',
success: callback
});
};
......@@ -99,12 +104,12 @@ var Resource = function(url_path) {
var GarganRest = function(base_path, path_list) {
var that = this;
$.each(path_list, function(p, path){
$.each(path_list, function(i, path){
that[path] = new Resource(base_path + path);
});
};
garganrest = new GarganRest('/api/', ['nodes']);
garganrest = new GarganRest('/api/', ['nodes', 'metrics']);
// var log = function(result){console.log(result);};
......
static/lib/gargantext/tables.css
View file @ 8fcfc803
......@@ -61,6 +61,7 @@ span.note {
span.note.glyphicon {
color: #555;
top:0;
}
p.note {
......@@ -129,14 +130,10 @@ tr:hover {
margin-bottom: 1em;
}
.oldsubform {
.subform {
color: #777 ;
}
.usersubform {
color: blue ;
}
.dynatable-record-count {
font-size: 0.7em;
}
......
templates/pages/projects/project.html
View file @ 8fcfc803
......@@ -88,13 +88,29 @@
{{corpus.name}}, {{ corpus.count }} documents {{ corpus.status_message }}
</a>
</div>
<div class="col-md-2 content">
<a href="/projects/{{project.id}}/corpora/{{corpus.id}}">
<div class="col-md-3 content">
<a href="/projects/{{project.id}}/corpora/{{corpus.id}}"
title="View the corpus">
<button type="button" class="btn btn-default" aria-label="Left Align">
<span class="glyphicon glyphicon-eye-open" aria-hidden="true"></span>
</button>
</a>
<!-- -->
<button type="button" class="btn btn-default yopla" data-container="body" data-toggle="popover" data-placement="bottom" data-trigger="focus"
data-content="
<ul>
<li
onclick=&quot;
garganrest.metrics.update({{corpus.id}}, function(){alert('The corpus ({{corpus.name|escapejs}}) was updated')});
&quot;>
<a href='#'>Recalculate ngram metrics</a> <br/> (can take a little while)
</li>
</ul>
">
<span class="glyphicon glyphicon-dashboard" aria-hidden="true"
title='Recalculate ngram scores and similarities'></span>
</button>
<button type="button" class="btn btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
data-content="
<ul>
......@@ -103,14 +119,15 @@
garganrest.nodes.delete({{corpus.id}}, function(){$('#corpus_'+{{corpus.id}}).remove()});
$(this).parent().parent().remove();
&quot;>
<a href=&quot;#&quot;>Delete this</a>
<a href='#'>Delete this</a>
</li>
</ul>
">
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
<span class="glyphicon glyphicon-trash" aria-hidden="true"
title='Delete this corpus'></span>
</button>
</div>
<div class="col-md-4 content">
<div class="col-md-3 content">
{% for state in corpus.hyperdata.statuses %}
{% ifequal state.action "Workflow" %}
{% if state.complete %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment