Merge branch 'dbmigration'

Towards node db

README.txt

+Here informations for installation:
+
+I - Local Installation
+
+1) Create virtualenv with python version 3.4
+
+2) pip install -r requirements.txt
+
+3) Manually install nltk (inside the sources)
+	nltk==3.0a4
+
+4) adapt database configuration in gargantext_web settings 
+(Development done on postgresql)
+
+5) source env/bin/activate
+
+6) ./manage.py runserver
+
+That's it

documents/admin.py

@@ -71,13 +71,13 @@ class CorpusAdmin(admin.ModelAdmin):
             #importateur.importer(obj)
         except Exception as e:
             print("Error importateur", e)
-        
+
 class DocumentAdmin(admin.ModelAdmin):
-    exclude = ('user',)
-    list_display = ('date', 'source', 'title')
-    list_per_page = 20
-    list_filter = ('project', 'corpus')
-    search_fields = ('title',)
+    exclude         = ('user',)
+    list_display    = ('date', 'source', 'title')
+    list_per_page   = 20
+    list_filter     = ('project', 'corpus')
+    search_fields   = ('title',)
 
     def has_change_permission(self, request, obj=None):
         has_class_permission = super(DocumentAdmin, self).has_change_permission(request, obj)

gargantext_web/settings.py

@@ -60,8 +60,9 @@ INSTALLED_APPS = (
     'django.contrib.staticfiles',
     'django_extensions',
     #'south',
-    'documents',
+    #'documents',
     'node',
+    'ngram',
     'django_hstore',
 )
 
@@ -84,7 +85,7 @@ WSGI_APPLICATION = 'wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'gargantext',
+        'NAME': 'ggdev',
         'USER': 'alexandre',
         'PASSWORD': 'C8kdcUrAQy66U',
         'HOST': '127.0.0.1',

gargantext_web/views.py

@@ -6,6 +6,7 @@ from django.template.loader import get_template
 from django.template import Context
 
 from documents.models import Project, Corpus, Document
+from node.models import Node, NodeType
 
 from django.contrib.auth.models import User
 
@@ -73,10 +74,12 @@ def projects(request):
         return redirect('/login/?next=%s' % request.path)
     
     t = get_template('projects.html')
+    
     user = request.user
-
     date = datetime.datetime.now()
-    projects = Project.objects.all().filter(user=request.user.pk).order_by("-date")
+    
+    project = NodeType.objects.get(name='Project')
+    projects = Node.objects.filter(user=user, type_id = project.id).order_by("-date")
     number = len(projects)
 
     html = t.render(Context({\
@@ -88,12 +91,12 @@ def projects(request):
     
     return HttpResponse(html)
 
-def project(request, p_id):
+def project(request, project_id):
     if not request.user.is_authenticated():
         return redirect('/login/?next=%s' % request.path)
-    
+
     try:
-        offset = str(p_id)
+        offset = str(project_id)
     except ValueError:
         raise Http404()
 
@@ -101,10 +104,11 @@ def project(request, p_id):
     user = request.user
 
     date = datetime.datetime.now()
-    project = Project.objects.get(pk=p_id)
-    corpora = Corpus.objects.all().filter(project_id=p_id,user=request.user.pk)
-    number = len(corpora)
     
+    project = Node.objects.get(id=project_id)
+    corpora = project.get_children()
+    number = len(corpora)
+
     html = t.render(Context({\
             'user': user,\
             'date': date,\
@@ -112,7 +116,7 @@ def project(request, p_id):
             'corpora' : corpora,\
             'number': number,\
             }))
-    
+
     return HttpResponse(html)
 
 def corpus(request, p_id, c_id):

ngram/admin.py

 from django.contrib import admin
 
-# Register your models here.
+from ngram.models import Ngram, NodeNgram, NodeNgramNgram
+
+admin.site.register(Ngram)
+admin.site.register(NodeNgram)
+admin.site.register(NodeNgramNgram)
+

ngram/models.py

 from django.db import models
+from django.utils import timezone
+from django.contrib.auth.models import User
+
+
+from node.models import Node
+
+
+class Ngram(models.Model):
+    terms    = models.TextField(unique=True)
+    n        = models.IntegerField()
+    def __str__(self):
+        return "[%d] %s" % (self.pk, self.terms)
+
+class NodeNgram(models.Model):
+    node        = models.ForeignKey(Node)
+    ngram       = models.ForeignKey(Ngram, related_name="nodengram")
+    def __str__(self):
+        return "%s: %s" % (self.node.name, self.ngram.terms)
+
+class NodeNgramNgram(models.Model):
+    node        = models.ForeignKey(Node)
+    
+    ngramX      = models.ForeignKey(Ngram, related_name="nodengramngramx")
+    ngramY      = models.ForeignKey(Ngram, related_name="nodengramngramy")
+
+    score       = models.FloatField(default=0)
+
+    def __str__(self):
+        return "%s: %s / %s" % (self.node.name, self.ngramX.terms, self.ngramY.terms)
+
 
-# Create your models here.

node/admin.py

 from django.contrib import admin
+from django.forms import ModelForm, ModelChoiceField
 
-# Register your models here.
-
-from node.models import NodeType, Node, Ngram, NodeNgramNgram, Project
+from node.models import NodeType, Node, Project, Corpus, Document
 
 
 class NodeAdmin(admin.ModelAdmin):
@@ -39,17 +38,19 @@ class NodeAdmin(admin.ModelAdmin):
             nodeTypeParent  = NodeType.objects.get(name=self._parent_nodetype_name)
             
             try:
-                nodeParent      = Node.objects.get(type = nodeTypeParent, user = request.user)
+                if nodeType.name == 'Project':
+                    nodeParent  = Node.objects.get(type = nodeTypeParent, user = request.user)
+                else:
+                    nodeParent  = Node.objects.get(id = request.POST['parent'])
             except:
-                nodeParent = Node.add_root(type = nodeTypeParent, user = request.user)
-            
-            obj.user = request.user
+                nodeParent  = Node.add_root(type = nodeTypeParent, user = request.user, name=request.user.username)
+            obj.user        = request.user
             
             node            = nodeParent.add_child(type = nodeType,\
-                                               user = request.user,\
-                                               name=obj.name,\
-                                               file=obj.file,\
-                                               metadata=obj.metadata)
+                                                user    = request.user,\
+                                                name    = obj.name,\
+                                                file    = obj.file,\
+                                                metadata= obj.metadata)
             
             #nodeParent.save()
             #node.save()
@@ -58,27 +59,44 @@ class NodeAdmin(admin.ModelAdmin):
         else:
             obj.save()
 
+######################################################################
+
 class ProjectAdmin(NodeAdmin):
-    _parent_nodetype_name = 'Root'
-    _nodetype_name = 'Project'
+    _parent_nodetype_name   = 'Root'
+    _nodetype_name          = 'Project'
+
+######################################################################
+
+class CorpusForm(ModelForm):
+    #parent = ModelChoiceField(Node.objects.filter(user_id=request.user.id, type_id=2))
+    def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop('request',None)
+        super(CorpusForm, self).__init__(*args, **kwargs)
+        #self.request = kwargs.pop('request', None)
+        #self.request = kwargs.pop("request")
+        #print(self.request)
+    parent = ModelChoiceField(Node.objects.filter(user_id=1, type_id=2))
 
 class CorpusAdmin(NodeAdmin):
     _parent_nodetype_name = 'Project'
     _nodetype_name = 'Corpus'
+    form = CorpusForm
+
+######################################################################
+
+class DocumentForm(ModelForm):
+    parent = ModelChoiceField(Node.objects.filter(user_id=1, type_id=3))
 
 class DocumentAdmin(NodeAdmin):
     _parent_nodetype_name = 'Corpus'
     _nodetype_name = 'Document'
+    form = DocumentForm
 
 
 admin.site.register(NodeType)
 
 admin.site.register(Project, ProjectAdmin)
-#admin.site.register(Corpus, CorpusAdmin)
-#admin.site.register(Document, DocumentAdmin)
-
-admin.site.register(Node)
-admin.site.register(Ngram)
-admin.site.register(NodeNgramNgram)
+admin.site.register(Corpus, CorpusAdmin)
+admin.site.register(Document, DocumentAdmin)
 
 

node/models.py

-
 from django.db import models
 from django.utils import timezone
 from django.contrib.auth.models import User
@@ -17,7 +16,6 @@ class NodeType(models.Model):
     def __str__(self):
         return self.name
 
-
 class Node(MP_Node):
     #parent = models.ForeignKey('self', related_name='children_set', null=True, db_index=True)
     user       = models.ForeignKey(User)
@@ -32,6 +30,10 @@ class Node(MP_Node):
     def __str__(self):
         return self.name
 
+    def liste(self, user):
+        for noeud in Node.objects.filter(user=user):
+            print(noeud.depth * "    " + "[%d] %d" % (noeud.pk, noeud.name))
+
 class Project(Node):
     class Meta:
         proxy=True
@@ -39,30 +41,10 @@ class Project(Node):
 class Corpus(Node):
     class Meta:
         proxy=True
+        verbose_name_plural = 'Corpora'
 
 class Document(Node):
     class Meta:
         proxy=True
 
 
-
-
-class Ngram(models.Model):
-    terms    = models.TextField(unique=True)
-    n        = models.IntegerField()
-    # post-tag = models.ManyToMany(blank=True)
-    # ajouter une table stem ?
-    def __str__(self):
-        return self.terms
-
-class NodeNgramNgram(models.Model):
-    ngramX      = models.ForeignKey(Ngram, related_name="X")
-    ngramY      = models.ForeignKey(Ngram, related_name="Y")
-
-    node        = models.ForeignKey(Node)
-    score       = models.FloatField(default=0)
-
-    def __str__(self):
-        return self.node
-
-

requirements.txt

@@ -5,31 +5,42 @@ MarkupSafe==0.23
 Pillow==2.5.3
 Pygments==1.6
 South==1.0
+amqp==1.4.6
+anyjson==0.3.3
+billiard==3.3.0.18
+celery==3.1.15
 certifi==14.05.14
+cffi==0.8.6
+cryptography==0.6
 decorator==3.4.0
 django-extensions==1.4.0
 django-grappelli==2.5.3
 django-hstore==1.3.1
+django-mptt==0.6.1
 django-nested-inlines==0.1
+django-treebeard==2.0
 graphviz==0.4
 ipython==2.2.0
+kombu==3.0.23
 lxml==3.3.6
 matplotlib==1.4.0
 networkx==1.9
-nltk==3.0a4
 nose==1.3.4
 numpy==1.8.2
 pandas==0.14.1
 pexpect-u==2.5.1
 psycopg2==2.5.4
+pycparser==2.10
 pydot2==1.0.33
 pyparsing==2.0.2
 python-dateutil==2.2
 pytz==2014.7
 pyzmq==14.3.1
 readline==6.2.4.1
+redis==2.10.3
 scikit-learn==0.15.1
 scipy==0.14.0
 six==1.7.3
 sympy==0.7.5
 tornado==4.0.1
+uWSGI==2.0.7

templates/project.html

@@ -16,8 +16,7 @@
 <div class="container theme-showcase" role="main">
         <div class="jumbotron">
             {% if project %}
-            <h1>{{ project.title }}</h1>
-            <h2> {{ project.subtitle }}</h2>
+            <h1>{{ project.name }}</h1>
             <h3> {{number}} corpora </h3>
             <p>
             <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/add/">Add a corpus</a></p>
@@ -32,11 +31,11 @@
 								{% if corpora  %}
 										{% for corpus in corpora %}
 										<div class="col-md-4">
-                        <h3><a href="/project/{{ project.id }}/corpus/{{corpus.id}}">{{ corpus.title }}</a></h3>
+                        <h3><a href="/project/{{ project.id }}/corpus/{{corpus.id}}">{{ corpus.name }}</a></h3>
 												<h4>{{ corpus.subtitle }}</h4>
-                        <p>{{ corpus.language }}, {{ corpus.database}}</p>
+                        <p>{{ corpus.language }} {{ corpus.database}}</p>
 												<h5>Activity:</h5>
-												<div class="chart" data-percent="50">73%</div>
+												<div class="chart" data-percent="73">73%</div>
 
 										</div>
 										{% endfor %}

templates/projects.html

@@ -25,7 +25,7 @@
 										{% for project in projects %}
 										<!--<div class="col-md-offset-7 col-md-4 content" style="background-color:grey">!-->
 										<div class="col-md-3 content">
-												<h3><a href="/project/{{ project.id }}">{{ project.title }}</a></h3>
+												<h3><a href="/project/{{ project.id }}">{{ project.name }}</a></h3>
 												<h4>{{ project.subtitle }}<h4>
 												<h5>Completed:</h5>
 												<div class="chart" barColor="#fffff" data-percent="75">75%</div>