Merge branch 'alex'

Intégration des mes modifs locales

gargantext_web/settings.py

@@ -63,7 +63,6 @@ INSTALLED_APPS = (
     'django.contrib.staticfiles',
     'django_extensions',
     #'south',
-    #'documents',
     'cte_tree',
     'node',
     'ngram',

gargantext_web/views.py

@@ -228,7 +228,8 @@ def project(request, project_id):
                     )
 
             try:
-                corpus.parse_resources.apply_async((), countdown=1)
+                #corpus.parse_resources.apply_async((), countdown=1)
+                corpus.parse_resources()
             except Exception as error:
                 print(error)
 

init/graph.sh

 #Usage : ./manage.ph > graph.pdf
 
-./manage.py graph_models documents | dot -Tpdf 
+./manage.py graph_models node | dot -Tpdf 

node/admin.py

@@ -2,12 +2,19 @@ from django.contrib import admin
 from django.forms import ModelForm, ModelChoiceField
 from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
 
-from node.models import NodeType, Language, Node, Project, Corpus, Document, ResourceType, Resource, Node_Ngram, Node_Resource
+from node.models import NodeType, Language, Node, \
+        Project, Corpus, Document, \
+        ResourceType, Resource, \
+        Ngram, Node_Ngram, Node_Resource
 
 class ResourceInLine(admin.TabularInline):
     model = Resource
     extra = 0
 
+class NgramAdmin(admin.ModelAdmin):
+    list_display = ('N', 'Terms')
+    search_fields = ('Terms',)
+
 class NodeAdmin(admin.ModelAdmin):
     exclude = ('user', 'path', 'depth', 'numchild', 'ngrams')
     list_display = ('name', 'date')
@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin)
 admin.site.register(Corpus, CorpusAdmin)
 admin.site.register(Document, DocumentAdmin)
 
+
 admin.site.register(Node_Resource)
+admin.site.register(Ngram)
 admin.site.register(Node_Ngram)
 

node/models.py

@@ -46,6 +46,9 @@ class Ngram(models.Model):
     n           = models.IntegerField()
     terms       = models.CharField(max_length=255)
     nodes       = models.ManyToManyField(through='Node_Ngram', to='Node')
+    def __str__(self):
+        return self.terms
+
 
 class Resource(models.Model):
     user        = models.ForeignKey(User)
@@ -207,8 +210,6 @@ class Node_Ngram(models.Model):
     def __str__(self):
         return "%s: %s" % (self.node.name, self.ngram.terms)
 
-
-
 class Project(Node):
     class Meta:
         proxy=True

tests/cooc_whitelist.sql

+SELECT
+    COUNT(*) AS cooccurrences,
+    ngX.terms,
+    ngY.terms
+FROM
+    node_node AS n  -- the nodes who are direct children of the corpus
+    
+INNER JOIN
+    node_node_ngram AS nngX ON nngX.node_id = n.id  --  list of ngrams contained in the node
+INNER JOIN
+    node_node_ngram AS whitelistX ON whitelistX.ngram_id = nngX.ngram_id -- list of ngrams contained in the whitelist and in the node
+INNER JOIN
+    node_ngram AS ngX ON ngX.id = whitelistX.ngram_id -- ngrams which are in both
+    
+INNER JOIN
+    node_node_ngram AS nngY ON nngY.node_id = n.id
+INNER JOIN
+    node_node_ngram AS whitelistY ON whitelistY.ngram_id = nngY.ngram_id
+INNER JOIN
+    node_ngram AS ngY ON ngY.id = whitelistY.ngram_id
+    
+WHERE
+    n.parent_id = %s
+AND
+    whitelistX.node_id = %s
+AND
+    whitelistY.node_id = %s
+AND
+    nngX.ngram_id < nngY.ngram_id   --  so we only get distinct pairs of ngrams
+    
+GROUP BY
+    ngX.id,
+    ngX.terms,
+    ngY.id,
+    ngY.terms
+ORDER BY
+    cooccurrences DESC
+LIMIT
+    200
\ No newline at end of file