Merge branch 'unstable' of ssh://delanoe.org:1979/gargantext into samuel

analysis/distance.py

 from admin.utils import PrintException
 from gargantext_web.db import *
+from gargantext_web.db import get_or_create_node
 
 from collections import defaultdict
 from operator import itemgetter
@@ -8,18 +9,13 @@ from django.db import connection, transaction
 import math
 from math import log,sqrt
 
+import numpy as np
 import scipy
-
-from gargantext_web.db import get_or_create_node
-
-
 import pandas as pd
+
 from copy import copy
-import numpy as np
-import scipy
 import networkx as nx
 from networkx.readwrite import json_graph
-from rest_v1_0.api import JsonHttpResponse
 
 from analysis.louvain import best_partition, generate_dendogram, partition_at_level
 

init/sql/testNgrams.sql

+select nn.ngram_id from node_node_ngram as nn 
+    JOIN node_node as n on nn.node_id = n.id 
+    JOIN node_ngram as ng ON ng.id = nn.ngram_id
+    JOIN node_node_ngram as nng ON nng.ngram_id = nn.ngram_id
+    JOIN node_node as nodeList on nodeList.id = nn.node_id
+    JOIN node_nodetype as typ on typ.id = nodeList.type_id
+
+    WHERE n.parent_id = 1452569
+    AND ng.terms = 'moral support'
+    AND typ.name = 'MiamList'
+    GROUP BY nn.ngram_id
+    
+    ;
+

init/sql/testSynonym.sql

+select * from node_nodengramngram as nnn 
+    JOIN node_node as n on nnn.node_id = n.id 
+    JOIN node_ngram as ng ON ng.id = nnn.ngramy_id
+    JOIN node_node as nodeList on nodeList.id = nnn.node_id
+    JOIN node_nodetype as typ on typ.id = nodeList.type_id
+
+    WHERE n.parent_id = 1452569
+    AND ng.terms = 'moral support'
+    AND typ.name = 'GroupList'
+    
+    ;
+

ngram/mapList.py

@@ -70,7 +70,6 @@ def compute_mapList(corpus,limit=500,n=1):
                              .filter(NodeNgramNgram.node_id == node_group.id)
                              .all()
                     )
-    #print([t for t in top_ngrams])
     
     node_mapList = get_or_create_node(nodetype='MapList', corpus=corpus)
     session.query(NodeNgram).filter(NodeNgram.node_id==node_mapList.id).delete()
@@ -79,7 +78,7 @@ def compute_mapList(corpus,limit=500,n=1):
     data = zip(
         [node_mapList.id for i in range(1,limit)]
         , [n[0] for n in list(top_multigrams) + list(top_monograms)
-                if (n[0],) not in list(stop_ngrams) + list(grouped_ngrams)
+                if (n[0],) not in list(stop_ngrams) 
             ]
         , [1 for i in range(1,limit)]
     )

rest_v1_0/graph.py

@@ -35,11 +35,11 @@ class Graph(APIView):
         if field1 in accepted_field1 :
             if field2 in accepted_field2 :
                 if start is not None and end is not None :
-                    data = get_cooc(corpus=corpus,field1=field1, field2=field2
+                    data = get_cooc(corpus=corpus, field1=field1, field2=field2
                                     , start=start, end=end
                                     , hapax=hapax, distance=distance)
                 else:
-                    data = get_cooc(corpus=corpus,field1=field1, field2=field2
+                    data = get_cooc(corpus=corpus, field1=field1, field2=field2
                             , hapax=hapax, distance = distance, bridgeness=bridgeness)
                 if format_ == 'json':
                     return JsonHttpResponse(data)

tests/test_ngrams.py

+
+from admin.env import *
+from gargantext_web.db import session, cache, get_or_create_node
+from gargantext_web.db import Node, NodeHyperdata, Hyperdata, Ngram, NodeNgram, NodeNgramNgram, NodeHyperdataNgram
+from sqlalchemy import func, alias, asc, desc
+import sqlalchemy as sa
+from sqlalchemy.orm import aliased
+
+from ngram.group import compute_groups, getStemmer
+
+# corpus = Corpus(272)
+corpus_id = 540420
+corpus = session.query(Node).filter(Node.id==corpus_id).first()
+#group = get_or_create_node(corpus=corpus, nodetype="Group")
+
+stop_id = get_or_create_node(nodetype='StopList',corpus=corpus).id
+miam_id = get_or_create_node(nodetype='MiamList',corpus=corpus).id
+
+somme = sa.func.count(NodeNgram.weight)
+ngrams = (session.query(Ngram.id, Ngram.terms, somme )
+        .join(NodeNgram, NodeNgram.ngram_id == Ngram.id)
+        .join(Node, Node.id == NodeNgram.node_id)
+        .filter(Node.parent_id==corpus_id, Node.type_id==cache.NodeType['Document'].id)
+        .group_by(Ngram.id)
+        .order_by(desc(somme))
+        .limit(100000)
+        )
+
+stops = (session.query(Ngram.id, Ngram.terms)
+                .join(NodeNgram, NodeNgram.ngram_id == Ngram.id)
+                .filter(NodeNgram.node_id == stop_id)
+                .all()
+                )
+
+miams = (session.query(Ngram.id, Ngram.terms, somme)
+                .join(NodeNgram, NodeNgram.ngram_id == Ngram.id)
+                .filter(NodeNgram.node_id == miam_id)
+                .group_by(Ngram.id, Ngram.terms)
+                .order_by(desc(somme))
+                .all()
+                )
+
+stemmer = getStemmer(corpus)
+
+ws = ['honeybee', 'honeybees']
+
+print(stemmer(ws[0]) == stemmer(ws[1]))
+
+#
+#for n in miams:
+#    if n[1] == 'bees':
+#        print("!" * 30)
+#        print(n)
+#        print("-" * 30)
+#    else:
+#        print(n)
+#