[FIX] Ngram workflow parameters and cooc for specificities (for tests on dev.gargantext.org).

analysis/cooccurrences.py

@@ -27,7 +27,7 @@ def cooc(corpus=None
     stop_id :: Int
     group_id :: Int
 
-    For the moment, start and ens are simple, only year is implemented yet
+    For the moment, start and end are simple, only year is implemented yet
     start :: TimeStamp -- example: '2010-05-30 02:00:00+02'
     end   :: TimeStamp
     limit :: Int
@@ -109,7 +109,8 @@ def cooc(corpus=None
 # Cooc is symetric, take only the main cooccurrences and cut at the limit
     cooc_query = (cooc_query
              .filter(NodeNgramX.ngram_id < NodeNgramY.ngram_id)
-             .having(cooc_score > 1)
+             .having(cooc_score > 2)
+             #.having(cooc_score > 1)
              
              .group_by(NodeNgramX.ngram_id, NodeNgramY.ngram_id)
              .order_by(desc('cooc_score'))

analysis/functions.py

@@ -21,7 +21,8 @@ import networkx as nx
 from networkx.readwrite import json_graph
 from rest_v1_0.api import JsonHttpResponse
 
-from analysis.louvain import best_partition
+from analysis.louvain import best_partition, generate_dendogram, partition_at_level
+
 from ngram.lists import listIds
 
 
@@ -229,10 +230,10 @@ def get_cooc(request=None, corpus=None, cooc_id=None, type='node_link', size=siz
     #print(n)
     #print(m)
 
-    nodes_included = 200 #int(round(size/20,0))
+    nodes_included = 300 #int(round(size/20,0))
     #nodes_excluded = int(round(size/10,0))
 
-    nodes_specific = 200 #int(round(size/10,0))
+    nodes_specific = 300 #int(round(size/10,0))
     #nodes_generic = int(round(size/10,0))
 
     # TODO user the included score for the node size
@@ -263,11 +264,11 @@ def get_cooc(request=None, corpus=None, cooc_id=None, type='node_link', size=siz
         #edges_to_remove = [ e for e in G.edges_iter() if
 
         degree = G.degree()
-        nodes_to_remove = [n for n in degree if degree[n] ==0]
+        nodes_to_remove = [n for n in degree if degree[n] <=1]
         G.remove_nodes_from(nodes_to_remove)
         uG = G.to_undirected()
         partition = best_partition(uG)
-
+        print(partition)
         print("Density of the graph:", nx.density(G))
     except:
         print("-" * 30)
@@ -315,7 +316,8 @@ def get_cooc(request=None, corpus=None, cooc_id=None, type='node_link', size=siz
             except Exception as error:
                 print("error02: ",error)
         data = json_graph.node_link_data(G)
-
+    elif type == 'bestpartition':
+        return(partition)
 
     #    data = json_graph.node_link_data(G, attrs={\
     #            'source':'source',\
@@ -325,5 +327,5 @@ def get_cooc(request=None, corpus=None, cooc_id=None, type='node_link', size=siz
     #            #'color':'color',\
     #            'id':'id',})
     #print(data)
-    return data
+    return(data)
 

gargantext_web/celery.py

@@ -50,7 +50,6 @@ def apply_workflow(corpus_id):
     #ngrams2miam(user_id=corpus.user_id, corpus_id=corpus_id)
     update_processing(corpus, 0)
 
-#@transaction.commit_manually
 @shared_task
 def empty_trash(corpus_id):
     nodes = models.Node.objects.filter(type_id=cache.NodeType['Trash'].id).all()

ngram/workflow.py

@@ -6,6 +6,8 @@ from ngram.specificity import compute_specificity
 from ngram.group import compute_groups
 from ngram.miam import compute_miam
 from gargantext_web.db import get_or_create_node
+#from gargantext_web.celery import update_processing
+
 
 def ngram_workflow(corpus, n=5000):
     '''
@@ -17,21 +19,27 @@ def ngram_workflow(corpus, n=5000):
 
     compute_cvalue(corpus,limit=part) # size
     
-    part = round(part * 0.6)
+    part = round(part * 0.4)
+    print('spec part:', part)
 
     compute_specificity(corpus,limit=part)
     
     part = round(part * 0.5)
 
 #    compute_stop(corpus)
-    compute_groups(corpus,limit_inf=part, limit_sup=n)
+    limit_inf = round(part * 1)
+    limit_sup = round(part * 5)
+    print(limit_inf,limit_sup)
+    compute_groups(corpus,limit_inf=limit_inf, limit_sup=limit_sup)
     
 #    compute_miam(corpus,limit=part) # size
     
     compute_tfidf(corpus)
+    
 
-#corpus=session.query(Node).filter(Node.id==244250).first()
+#corpus=session.query(Node).filter(Node.id==257579).first()
 #ngram_workflow(corpus)
+#update_processing(corpus, 0)
 
 #cvalue = get_or_create_node(corpus=corpus,nodetype='Cvalue')
 #print(session.query(NodeNodeNgram).filter(NodeNodeNgram.nodex_id==cvalue.id).count())