1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
from admin.env import *
from ngram.tfidf import compute_tfidf, compute_tfidf_global
from ngram.cvalue import compute_cvalue
from ngram.specificity import compute_specificity
from ngram.stop import compute_stop
from ngram.group import compute_groups
from gargantext_web.db import get_or_create_node
from ngram.mapList import compute_mapList
from gargantext_web.db import NodeNgram
#from gargantext_web.celery import update_processing
def ngram_workflow(corpus, n=5000):
'''
All the workflow to filter the ngrams.
'''
compute_stop(corpus)
compute_tfidf_global(corpus)
part = round(n * 0.9)
compute_cvalue(corpus,limit=1000) # size
part = round(part * 0.8)
print('spec part:', part)
compute_specificity(corpus,limit=part)
part = round(part * 0.8)
limit_inf = round(part * 1)
limit_sup = round(part * 5)
print(limit_inf,limit_sup)
compute_groups(corpus,limit_inf=limit_inf, limit_sup=limit_sup)
compute_mapList(corpus,limit=1000) # size
compute_tfidf(corpus)
node_id = 1427298
#corpus=session.query(Node).filter(Node.id==540420).first()
corpus=session.query(Node).filter(Node.id==node_id).first()
ngram_workflow(corpus)
#update_processing(corpus, 0)
check_stop = False
if check_stop:
stop = get_or_create_node(corpus=corpus,nodetype='StopList')
#session.query(NodeNgram).filter(NodeNgram.node_id==stop.id).delete()
#session.commit()
stop_ngrams = (session.query(Ngram)
.join(NodeNgram, NodeNgram.ngram_id == Ngram.id)
.filter(NodeNgram.node_id==stop.id)
.all()
)
print([n for n in stop_ngrams])