Integration Alexs work

gargantext_web/urls.py

@@ -3,7 +3,7 @@ from django.conf.urls import patterns, include, url
 from django.contrib import admin
 
 from gargantext_web.views import home, projects, project, corpus
-from gargantext_web.views import add_corpus, delete_project, delete_corpus
+from gargantext_web.views import delete_project, delete_corpus
 from gargantext_web.views import exploration
 from gargantext_web.views import explorer_graph, explorer_matrix
 
@@ -22,7 +22,6 @@ urlpatterns = patterns('',
     url(r'^project/(\d+)/delete/$', delete_project),
     
     url(r'^project/(\d+)/$', project),
-    url(r'^project/(\d+)/add/$', add_corpus),
     
     url(r'^project/(\d+)/corpus/(\d+)/$', corpus),
     url(r'^project/(\d+)/corpus/(\d+)/delete/$', delete_corpus),

gargantext_web/views.py

@@ -8,7 +8,7 @@ from django.template import Context
 #from documents.models import Project, Corpus, Document
 
 from node.models import Language, ResourceType, Resource
-from node.models import Node, NodeType, Project, Corpus
+from node.models import Node, NodeType, Node_Resource, Project, Corpus
 from node.admin import CorpusForm, ProjectForm, ResourceForm
 
 from django.contrib.auth.models import User
@@ -25,7 +25,6 @@ from collections import defaultdict
 from parsing.FileParsers import *
 
 
-
 # SOME FUNCTIONS
 
 def query_to_dicts(query_string, *query_args):
@@ -128,6 +127,20 @@ def project(request, project_id):
     corpora = project.children.all()
     number  = project.children.count()
     
+    total = 0
+    donut = list()
+    donut_part = dict()
+    for corpus in corpora:
+        count =  corpus.children.count()
+        total += count
+        for node_resource in Node_Resource.objects.filter(node=corpus):
+
+            print(node_resource.resource.type,
+                    count,
+                    total,
+                    )
+
+
     board = list()
     for corpus in corpora:
         dashboard = dict()
@@ -139,51 +152,63 @@ def project(request, project_id):
 
     if request.method == 'POST':
         #form = CorpusForm(request.POST, request.FILES)
+        #print(str(request.POST))
         name        = str(request.POST['name'])
 
         try:
-            bdd_type = ResourceType.objects.get(id=str(request.POST['bdd_type']))
-        except:
-            bdd_type = None
+            resource_type = ResourceType.objects.get(id=str(request.POST['type']))
+        except Exception as error:
+            print(error)
+            resource_type = None
         
         try:
             file = request.FILES['file']
-        except:
+        except Exception as error:
+            print(error)
             file = None
 
-        if  name != "" and bdd_type != None and file != None :
+        #if name != "" and resource_type is not None and file is not None:
         try:
-                node_type   = NodeType.objects.get(name='Corpus')
             parent      = Node.objects.get(id=project_id)
-                corpus = Node(parent=parent, name=name, type=node_type, user=request.user)
+            node_type   = NodeType.objects.get(name='Corpus')
+
+            if resource_type.name == "europresse_french":
+                language    = Language.objects.get(iso2='fr')
+            elif resource_type.name == "europresse_english":
+                language    = Language.objects.get(iso2='en')
+            
+            try:
+                corpus = Node(
+                        user=request.user,
+                        parent=parent,
+                        type=node_type,
+                        name=name,
+                        )
+            except:
+                corpus = Node(
+                        user=request.user,
+                        parent=parent,
+                        type=node_type,
+                        language=language,
+                        name=name,
+                        )
+
             corpus.save()
-                #corpus.add_ressource(file=file)
+            
+            corpus.add_resource(
+                    user=request.user, 
+                    type=resource_type,
+                    file=file
+                    )
+
+            try:
+                corpus.parse_resources()
             except Exception as error:
                 print(error)
-            #resource = Resource(user=request.user, guid=str(date), bdd_type=bdd_type, file=file)
-            #language    = Language.objects.get(iso2='fr')
-
-            node = Node(parent=parent, type=node_type, name=name, user=request.user)#, language=language)
-            node.save()
-            #node.resource.add(resource)
-
-#            try:
-#                for resource in node.resource.all():
-#                    print(resource.bdd_type.name)
-#                    if resource.bdd_type.name == "PubMed":
-#                        fileparser = PubmedFileParser(file='/var/www/gargantext/media/' + str(resource.file))
-#                        fileparser.parse(node)
-#                    elif resource.bdd_type.name == "Web Of Science (WOS), ISI format":
-#                        fileparser = IsiParser(file='/var/www/gargantext/media/' + str(resource.file))
-#                        fileparser.parse(node)
-#                    elif node.bdd_type.name == "Europresse":
-#                        pass
-#
-#            except Exception as error:
-#                print(error)
 
             return HttpResponseRedirect('/project/' + str(project_id))
-        else:
+        except Exception as error:
+            print('ee', error)
             form = CorpusForm(request=request)
             formResource = ResourceForm()
 
@@ -315,38 +340,6 @@ def corpus(request, project_id, corpus_id):
     
     return HttpResponse(html)
 
-def add_corpus(request):
-    form = CorpusForm(request=request)
-    if request.method == 'POST':
-        #form = CorpusForm(request.POST, request.FILES)
-        name        = str(request.POST['name'])
-        
-        try:
-            #language    = Language.objects.get(name=str(request.POST['language']))
-            language    = Language.objects.get(iso2='fr')
-        except Exception as e:
-            print('line 323', e)
-            language = None
-        
-        if name != "" :
-            project_id  = 1047
-            node_type   = NodeType.objects.get(name='Corpus')
-            parent      = Node.objects.get(id=project_id)
-            Corpus(parent=parent, type=node_type, name=name, user=request.user, language=language).save()
-#            try:
-#                for resource in node.resource.all():
-#                    fileparser = PubmedFileParser.PubmedFileParser(file='/var/www/gargantext/media/' + str(resource.file))
-#                    fileparser.parse(node)
-#
-#            except Exception as error:
-#                print(error)
-
-            return HttpResponseRedirect('/project/' + str(project_id))
-
-    else:
-        form = CorpusForm(request=request)
-
-    return render(request, 'add_corpus.html', {'form': form})
 
 def delete_project(request, node_id):
     Node.objects.filter(id=node_id).all().delete()

node/admin.py

@@ -56,7 +56,7 @@ class NodeAdmin(admin.ModelAdmin):
                                                 user    = request.user,\
                                                 name    = obj.name,\
                                                 metadata= obj.metadata,\
-                                                resource = obj.resource)
+                                                )
             
             #nodeParent.save()
             #node.save()

node/models.py

@@ -12,13 +12,12 @@ from parsing.FileParsers import *
 from time import time
 from collections import defaultdict
 
-
+from gargantext_web.settings import MEDIA_ROOT
 
 # Some usefull functions
 # TODO: start the function name with an underscore (private)
-def upload_to(instance, filename):
-    return 'corpora/%s/%s' % (instance.user.username, filename)
-    #return 'corpora/%s/%f/%s' % (instance.user.username, time(), filename)
+def _upload_to(instance, filename):
+    return MEDIA_ROOT + '/corpora/%s/%s' % (instance.user.username, filename)
 
 # All classes here
 
@@ -42,9 +41,10 @@ class Ngram(models.Model):
     terms       = models.CharField(max_length=255)
 
 class Resource(models.Model):
+    user        = models.ForeignKey(User)
     guid        = models.CharField(max_length=255)
     type        = models.ForeignKey(ResourceType, blank=True, null=True)
-    file        = models.FileField(upload_to=upload_to, blank=True)
+    file        = models.FileField(upload_to=_upload_to, blank=True)
     digest      = models.CharField(max_length=32) # MD5 digest
 
 class NodeType(models.Model):
@@ -52,7 +52,6 @@ class NodeType(models.Model):
     def __str__(self):
         return self.name
 
-        
 class NodeQuerySet(models.query.QuerySet):
     """Methods available from Node querysets."""
     def extract_ngrams(self, keys, ngramsextractorscache=None, ngramscaches=None):
@@ -85,12 +84,6 @@ class Node(CTENode):
     date        = models.DateField(default=timezone.now, blank=True)
     metadata    = hstore.DictionaryField(blank=True)
 
-    # TODO: remove the three following fields
-    #fichier      = models.FileField(upload_to=upload_to, blank=True)
-    #resource    = models.ForeignKey(Resource, blank=True, null=True)
-    #ngrams      = models.ManyToManyField(NGrams)
-    
-    
     def __str__(self):
         return self.name
     
@@ -118,6 +111,8 @@ class Node(CTENode):
                 'isi'       : IsiFileParser,
                 'ris'       : RisFileParser,
                 'europress' : EuropressFileParser,
+                'europress_french'  : EuropressFileParser,
+                'europress_english' : EuropressFileParser,
             })[resource.type.name]()
             metadata_list += parser.parse(str(resource.file))
         # insert the new resources in the database!
@@ -171,7 +166,6 @@ class Node(CTENode):
             for ngram_text, weight in associations.items()
         ])
 
-
 class Node_Resource(models.Model):
     node     = models.ForeignKey(Node, related_name='node_resource')
     resource = models.ForeignKey(Resource)

parsing/FileParsers/FileParser.py

@@ -5,7 +5,6 @@ import zipfile
 from parsing.Caches import LanguagesCache
     
 
-
 class FileParser:
     """Base class for performing files parsing depending on their type.
     """

templates/project.html

@@ -78,7 +78,6 @@
 												
 												</h3>
 												<h4>{{ corpus.count }} Documents </h4>
-                        <p>{{ corpus.language }} {{ corpus.database}}</p>
 												<h5>Activity:</h5>
 												<div class="chart" data-percent="73">73%</div>