Skip to content

gargantext
gargantext
Commit 6239e326 authored by Administrator's avatar Administrator
Browse files
Options

Merge branch 'samuel' into unstable

parents 87ff5323 6d6ccfa1
Hide whitespace changes
Inline Side-by-side
Showing with 222 additions and 101 deletions
gargantext_web/urls.py
View file @ 6239e326
...@@ -63,7 +63,7 @@ urlpatterns = patterns('', ...@@ -63,7 +63,7 @@ urlpatterns = patterns('',
url(r'^ngrams$', views.ngrams), url(r'^ngrams$', views.ngrams),
url(r'^nodeinfo/(\d+)$', views.nodeinfo), url(r'^nodeinfo/(\d+)$', views.nodeinfo),
url(r'^tests/mvc$', views.tests_mvc), url(r'^tests/mvc$', views.tests_mvc),
url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments), url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments)
) )
......
gargantext_web/views.py
View file @ 6239e326
...@@ -9,7 +9,7 @@ from node.models import Language, ResourceType, Resource, \ ...@@ -9,7 +9,7 @@ from node.models import Language, ResourceType, Resource, \
Node, NodeType, Node_Resource, Project, Corpus, \ Node, NodeType, Node_Resource, Project, Corpus, \
Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram
from node.admin import CorpusForm, ProjectForm, ResourceForm from node.admin import CorpusForm, ProjectForm, ResourceForm, CustomForm
from django.contrib.auth.models import User from django.contrib.auth.models import User
...@@ -169,6 +169,7 @@ def projects(request): ...@@ -169,6 +169,7 @@ def projects(request):
'projects': projects 'projects': projects
}) })
def project(request, project_id): def project(request, project_id):
''' '''
This view represents all corpora in a panoramic way. This view represents all corpora in a panoramic way.
...@@ -234,81 +235,172 @@ def project(request, project_id): ...@@ -234,81 +235,172 @@ def project(request, project_id):
if request.method == 'POST': if request.method == 'POST':
#form = CorpusForm(request.POST, request.FILES) print("original file:")
#print(str(request.POST)) print(request.FILES)
name = str(request.POST['name'])
try: form = CustomForm(request.POST, request.FILES)
resource_type = ResourceType.objects.get(id=str(request.POST['type'])) if form.is_valid():
except Exception as error:
print(error)
resource_type = None name = form.cleaned_data['name']
thefile = form.cleaned_data['file']
try: resource_type = ResourceType.objects.get(id=str( form.cleaned_data['type'] ))
file = request.FILES['file']
except Exception as error: print("-------------")
print(error) print(name,"|",resource_type,"|",thefile)
file = None print("-------------")
#if name != "" and resource_type is not None and file is not None:
try:
parent = Node.objects.get(id=project_id)
node_type = NodeType.objects.get(name='Corpus')
if resource_type.name == "europress_french":
language = Language.objects.get(iso2='fr')
elif resource_type.name == "europress_english":
language = Language.objects.get(iso2='en')
print("new file:")
print(thefile)
try: try:
corpus = Node( parent = Node.objects.get(id=project_id)
user=request.user, node_type = NodeType.objects.get(name='Corpus')
parent=parent,
type=node_type, if resource_type.name == "europress_french":
language=language, language = Language.objects.get(iso2='fr')
name=name, elif resource_type.name == "europress_english":
) language = Language.objects.get(iso2='en')
except:
corpus = Node( try:
corpus = Node(
user=request.user,
parent=parent,
type=node_type,
language=language,
name=name,
)
except:
corpus = Node(
user=request.user,
parent=parent,
type=node_type,
name=name,
)
corpus.save()
print(request.user, resource_type , thefile )
corpus.add_resource(
user=request.user, user=request.user,
parent=parent, type=resource_type,
type=node_type, file=thefile
name=name,
) )
corpus.save() try:
#corpus.parse_and_extract_ngrams()
#corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
if DEBUG is True:
corpus.workflow()
else:
corpus.workflow.apply_async((), countdown=3)
print(request.user, resource_type , file ) except Exception as error:
print(corpus.language) print(error)
corpus.add_resource(
user=request.user,
type=resource_type,
file=file
)
try: return HttpResponseRedirect('/project/' + str(project_id))
#corpus.parse_and_extract_ngrams()
#corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
if DEBUG is True:
corpus.workflow()
else:
corpus.workflow.apply_async((), countdown=3)
except Exception as error: except Exception as error:
print(error) print('ee', error)
form = CorpusForm(request=request)
return HttpResponseRedirect('/project/' + str(project_id)) formResource = ResourceForm()
except Exception as error:
print('ee', error)
form = CorpusForm(request=request) else:
formResource = ResourceForm() print("bad form, bad form")
return render(request, 'project.html', {
'form' : form,
'user' : user,
'date' : date,
'project' : project,
'donut' : donut,
'list_corpora' : list_corpora,
'whitelists' : whitelists,
'blacklists' : blacklists,
'cooclists' : cooclists,
'number' : number,
})
else: else:
form = CorpusForm(request=request) form = CustomForm()
formResource = ResourceForm()
# if request.method == 'POST':
# #form = CorpusForm(request.POST, request.FILES)
# #print(str(request.POST))
# name = str(request.POST['name'])
# try:
# resource_type = ResourceType.objects.get(id=str(request.POST['type']))
# except Exception as error:
# print(error)
# resource_type = None
# try:
# file = request.FILES['file']
# except Exception as error:
# print(error)
# file = None
# #if name != "" and resource_type is not None and file is not None:
# try:
# parent = Node.objects.get(id=project_id)
# node_type = NodeType.objects.get(name='Corpus')
# if resource_type.name == "europress_french":
# language = Language.objects.get(iso2='fr')
# elif resource_type.name == "europress_english":
# language = Language.objects.get(iso2='en')
# try:
# corpus = Node(
# user=request.user,
# parent=parent,
# type=node_type,
# language=language,
# name=name,
# )
# except:
# corpus = Node(
# user=request.user,
# parent=parent,
# type=node_type,
# name=name,
# )
# corpus.save()
# print(request.user, resource_type , file )
# print(corpus.language)
# corpus.add_resource(
# user=request.user,
# type=resource_type,
# file=file
# )
# try:
# #corpus.parse_and_extract_ngrams()
# #corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
# if DEBUG is True:
# corpus.workflow()
# else:
# corpus.workflow.apply_async((), countdown=3)
# except Exception as error:
# print(error)
# return HttpResponseRedirect('/project/' + str(project_id))
# except Exception as error:
# print('ee', error)
# form = CorpusForm(request=request)
# formResource = ResourceForm()
# else:
# form = CorpusForm(request=request)
# formResource = ResourceForm()
return render(request, 'project.html', { return render(request, 'project.html', {
'form' : form, 'form' : form,
'formResource' : formResource,
'user' : user, 'user' : user,
'date' : date, 'date' : date,
'project' : project, 'project' : project,
...@@ -441,10 +533,11 @@ def subcorpus(request, project_id, corpus_id, start , end ): ...@@ -441,10 +533,11 @@ def subcorpus(request, project_id, corpus_id, start , end ):
project = Node.objects.get(id=project_id) project = Node.objects.get(id=project_id)
corpus = Node.objects.get(id=corpus_id) corpus = Node.objects.get(id=corpus_id)
type_document = NodeType.objects.get(name="Document")
# retrieving all the documents # retrieving all the documents
documents = corpus.children.all() # documents = corpus.children.all()
number = corpus.children.count() documents = corpus.__class__.objects.filter(parent_id=corpus_id , type = type_document )
number = len(documents)
filtered_docs = [] filtered_docs = []
# filtering documents by range-date # filtering documents by range-date
...@@ -509,10 +602,11 @@ def subcorpusJSON(request, project_id, corpus_id, start , end ): ...@@ -509,10 +602,11 @@ def subcorpusJSON(request, project_id, corpus_id, start , end ):
project = Node.objects.get(id=project_id) project = Node.objects.get(id=project_id)
corpus = Node.objects.get(id=corpus_id) corpus = Node.objects.get(id=corpus_id)
type_document = NodeType.objects.get(name="Document")
# retrieving all the documents # retrieving all the documents
documents = corpus.children.all() # documents = corpus.children.all()
number = corpus.children.count() documents = corpus.__class__.objects.filter(parent_id=corpus_id , type = type_document )
number = len(documents)
filtered_docs = [] filtered_docs = []
# filtering documents by range-date # filtering documents by range-date
......
node/admin.py
View file @ 6239e326
...@@ -92,6 +92,49 @@ class ResourceForm(ModelForm): ...@@ -92,6 +92,49 @@ class ResourceForm(ModelForm):
model = Resource model = Resource
exclude = ['user', 'guid', 'digest'] exclude = ['user', 'guid', 'digest']
# for formexample.html
from django import forms
from django.utils.translation import ugettext_lazy as _
class CustomForm(forms.Form):
name = forms.CharField( label='Name', max_length=199 , required=True)
parsing_options = ResourceType.objects.all().values_list('id', 'name')
type = forms.IntegerField( widget=forms.Select( choices= parsing_options) , required=True )
file = forms.FileField()
# Description: clean_file()
"""
* file_.content_type - Example: ['application/pdf', 'image/jpeg']
* len(file_) - file size.
2.5MB - 2621440
5MB - 5242880
10MB - 10485760
20MB - 20971520
50MB - 5242880
100MB 104857600
250MB - 214958080
500MB - 429916160
"""
def clean_file(self):
file_ = self.cleaned_data.get('file')
#Filename length
if len(file_.name)>30:
from datetime import datetime
file_.name = str(datetime.now().microsecond)
raise forms.ValidationError(_('Come on dude, name too long. Now is:'+file_.name))
#File size
if len(file_)>104857600:
raise forms.ValidationError(_('File to heavy! (<100MB svp mec).'))
## File type:
# if file_.content_type == "application/zip":
# raise forms.ValidationError(_('We need a zip pls.'))
return file_
class CorpusForm(ModelForm): class CorpusForm(ModelForm):
#parent = ModelChoiceField(EmptyQuerySet) #parent = ModelChoiceField(EmptyQuerySet)
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
...@@ -99,12 +142,6 @@ class CorpusForm(ModelForm): ...@@ -99,12 +142,6 @@ class CorpusForm(ModelForm):
self.request = kwargs.pop('request', None) self.request = kwargs.pop('request', None)
super(CorpusForm, self).__init__(*args, **kwargs) super(CorpusForm, self).__init__(*args, **kwargs)
parent_type = NodeType.objects.get(name="Project") parent_type = NodeType.objects.get(name="Project")
#parent_type = NodeType.objects.get(name=self._parent_nodetype_name)
# self.fields['parent'].queryset = Node.objects.filter(
# user_id=self.request.user.id,
# type_id=parent_type.id
# )
#self.fields['language'].queryset = Language.objects.filter(implemented=1)
except Exception as error: except Exception as error:
print("Error with", error) print("Error with", error)
......
node/models.py
View file @ 6239e326
...@@ -129,33 +129,10 @@ class Node(CTENode): ...@@ -129,33 +129,10 @@ class Node(CTENode):
return Resource.objects.select_related('node_resource').filter(node_resource__node = self) return Resource.objects.select_related('node_resource').filter(node_resource__node = self)
def add_resource(self, **kwargs): def add_resource(self, **kwargs):
# print("printing arguments for add_resource():")
# print(kwargs)
# from django.core.files.storage import default_storage
# from django.core.files.base import ContentFile
# import os
# thefile = kwargs["file"]
# path = default_storage.save('tmp/somename.zip', ContentFile(thefile.read()))
# tmp_file = os.path.join(MEDIA_ROOT, path)
# print(tmp_file)
# kwargs["file"] = tmp_file
# print("final kwargs:")
# print(kwargs)
# only for tests # only for tests
resource = Resource(guid=str(time()), digest=str(time()), **kwargs ) resource = Resource(guid=str(time()), digest=str(time()), **kwargs )
#resource = Resource(**kwargs) #resource = Resource(**kwargs)
resource.save() resource.save()
print("printing rresource.file:")
print(resource.file)
# print("printing the resource 01____:")
# print(resource.file)
# print("printing the resource 02____: asdasdasd")
# User # User
if 'user' not in kwargs and 'user_id' not in kwargs: if 'user' not in kwargs and 'user_id' not in kwargs:
...@@ -193,8 +170,6 @@ class Node(CTENode): ...@@ -193,8 +170,6 @@ class Node(CTENode):
'europress_french' : EuropressFileParser, 'europress_french' : EuropressFileParser,
'europress_english' : EuropressFileParser, 'europress_english' : EuropressFileParser,
})[resource.type.name]() })[resource.type.name]()
print("parse_resources:")
print(resource.file)
metadata_list += parser.parse(str(resource.file)) metadata_list += parser.parse(str(resource.file))
# retrieve info from the database # retrieve info from the database
type_id = NodeType.objects.get(name='Document').id type_id = NodeType.objects.get(name='Document').id
......
templates/tests/formexample.html 0 → 100644
View file @ 6239e326
<html>
<body>
<form action="/formexample/" enctype="multipart/form-data" method="post">
{% csrf_token %}
{{ form.as_p }}
<input type="submit" value="OK">
</form>
{% if msg %}
<p>Response: {{msg}}</p>
{% endif %}
</body>
</html>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment