Skip to content

gargantext
gargantext
Commit 6b0538e6 authored by Administrator's avatar Administrator
Browse files
Options

[CLEAN] CLASS removing old documents class.

parent 4eab6f6f
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 443 deletions
documents/admin.py deleted 100644 → 0
View file @ 4eab6f6f
from django.contrib import admin
# Register your models here.
from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
from documents.models import Source, Language,\
Project, Corpus, Document,\
Ngram, NgramDocument, List, ListNgram,\
Coocurrence, Clique, Phylo
from sources import importateur
class DocumentInLine(admin.StackedInline):
model = Document
extra = 0
class CorpusInLine(admin.StackedInline):
model = Corpus
extra = 0
inlines = [DocumentInLine,]
class ProjectAdmin(admin.ModelAdmin):
exclude = ('user',)
list_display = ('title', 'date', 'user')
inlines = [CorpusInLine,]
def has_change_permission(self, request, obj=None):
has_class_permission = super(ProjectAdmin, self).has_change_permission(request, obj)
if not has_class_permission:
return False
if obj is not None and not request.user.is_superuser and request.user.id != obj.user.id:
return False
return True
def get_queryset(self, request):
if request.user.is_superuser:
return Project.objects.all()
return Project.objects.filter(user=request.user)
def save_model(self, request, obj, form, change):
if not change:
obj.user = request.user
obj.save()
class CorpusAdmin(admin.ModelAdmin):
exclude = ('user',)
list_display = ('title', 'date', 'database')
#inlines = [DocumentInLine,]
def has_change_permission(self, request, obj=None):
has_class_permission = super(CorpusAdmin, self).has_change_permission(request, obj)
if not has_class_permission:
return False
if obj is not None and not request.user.is_superuser and request.user.id != obj.user.id:
return False
return True
def get_queryset(self, request):
if request.user.is_superuser:
return Corpus.objects.all()
return Corpus.objects.filter(user=request.user)
def save_model(self, request, obj, form, change):
if not change:
obj.user = request.user
obj.save()
try:
importateur.importer(obj.database, obj.language, obj.zip_file, project=obj.project, corpus=obj, user=obj.user)
#importateur.importer(obj)
except Exception as e:
print("Error importateur", e)
class DocumentAdmin(admin.ModelAdmin):
exclude = ('user',)
list_display = ('date', 'source', 'title')
list_per_page = 20
list_filter = ('project', 'corpus')
search_fields = ('title',)
def has_change_permission(self, request, obj=None):
has_class_permission = super(DocumentAdmin, self).has_change_permission(request, obj)
if not has_class_permission:
return False
if obj is not None and not request.user.is_superuser and request.user.id != obj.user.id:
return False
return True
def get_queryset(self, request):
if request.user.is_superuser:
return Document.objects.all()
return Document.objects.filter(user=request.user)
def save_model(self, request, obj, form, change):
if not change:
obj.user = request.user
obj.save()
class NgramAdmin(admin.ModelAdmin):
list_display = ('terms', 'stem', 'n')
list_per_page = 20
class NgramDocAdmin(admin.ModelAdmin):
list_display = ('terms', 'document', 'occurrences')
list_per_page = 20
class ListNgramAdmin(admin.ModelAdmin):
list_display = ('mainForm', 'liste', 'cvalue')
list_per_page = 20
admin.site.register(Source)
admin.site.register(Language)
admin.site.register(Project, ProjectAdmin)
admin.site.register(Corpus, CorpusAdmin)
admin.site.register(Document, DocumentAdmin)
admin.site.register(Ngram,NgramAdmin)
admin.site.register(NgramDocument, NgramDocAdmin)
admin.site.register(List)
admin.site.register(ListNgram, ListNgramAdmin)
admin.site.register(Coocurrence)
admin.site.register(Clique)
admin.site.register(Phylo)
documents/models.py deleted 100644 → 0
View file @ 4eab6f6f
from django.contrib.auth.models import User
from django.db import models
from django.utils import timezone
from django_hstore import hstore
#from treebeard.mp_tree import MP_Node
from time import time
######################################################################
# DATABASES / LANGUAGES
# PROJECT / CORPUS / DOCUMENT
######################################################################
def upload_to(instance, filename):
return 'corpora/%s/%s' % (instance.user.username, filename)
#return 'corpora/%s/%f/%s' % (instance.user.username, time(), filename)
class Source(models.Model):
"""
Web Of Science, Europresse, Pubmed...
"""
def __str__(self):
return self.database
database = models.CharField(max_length=50, unique=True)
class Language(models.Model):
"""
French or english or...
"""
def __str__(self):
return self.language
language = models.CharField(max_length=30, unique=True)
class Project(models.Model):
date = models.DateField(default=timezone.now(), verbose_name="Date of creation")
user = models.ForeignKey(User, related_name='entries')
title = models.CharField(max_length=300, blank=True)
subtitle = models.CharField(max_length=300, blank=True)
others = hstore.DictionaryField(blank=True)
objects = hstore.HStoreManager()
#class Meta:
#get_latest_by = 'date'
#ordering = ('-date',)
#verbose_name_plural = 'entries'
def __str__(self):
return self.title
class Corpus(models.Model):
project = models.ForeignKey(Project)
database = models.ForeignKey(Source)
language = models.ForeignKey(Language)
date = models.DateField(default=timezone.now(), verbose_name="Date of creation")
user = models.ForeignKey(User)
title = models.CharField(max_length=300, blank=True)
subtitle = models.CharField(max_length=300, blank=True)
zip_file = models.FileField(upload_to=upload_to, blank=True)
others = hstore.DictionaryField(blank=True, null=True)
objects = hstore.HStoreManager()
class Meta:
get_latest_by = 'date'
ordering = ('-date',)
verbose_name_plural = 'corpora'
def __str__(self):
return self.title
class Document(models.Model):
project = models.ForeignKey(Project)
corpus = models.ManyToManyField(Corpus)
user = models.ForeignKey(User)
date = models.DateField(blank=True, null=True)
uniqu_id = models.CharField(max_length=150, unique=True)
title = models.TextField(blank=True)
authors = models.TextField(blank=True)
source = models.TextField(blank=True)
country = models.CharField(max_length=100, blank=True)
url = models.URLField(blank=True)
abstract = models.TextField(blank=True)
text = models.TextField(blank=True)
others = hstore.DictionaryField(blank=True)
objects = hstore.HStoreManager()
class Meta:
get_latest_by = 'date'
ordering = ('-date',)
#verbose_name_plural = 'entries'
def __str__(self):
return self.title
# class relation corpus / document (pour enlever des documents d'un corpus)
######################################################################
# NGRAM / NgramDocument
# LIST / ListNgram
######################################################################
class Ngram(models.Model):
terms = models.TextField(unique=True)
stem = models.TextField(blank=True)
n = models.IntegerField()
# post-tag = models.ManyToMany(blank=True)
# ajouter une table stem ?
def __str__(self):
return self.terms
class NgramTemporary(models.Model):
terms = models.TextField(unique=True)
stem = models.TextField(blank=True)
n = models.IntegerField()
# post-tag = models.ManyToMany(blank=True)
# ajouter une table stem ?
def __str__(self):
return self.terms
class NgramDocument(models.Model):
terms = models.ForeignKey(Ngram)
document = models.ForeignKey(Document)
occurrences = models.IntegerField(default=1, blank=True, null=True)
def __str__(self):
return self.terms.terms
class NgramDocumentTemporary(models.Model):
terms = models.TextField(blank=True)
document = models.IntegerField(blank=True)
occurrences = models.IntegerField(blank=True, null=True)
def __str__(self):
return self.terms
class List(models.Model):
title = models.CharField(max_length=100, unique=True)
user = models.ForeignKey(User)
date = models.DateField(default=timezone.now(), verbose_name="Date of creation")
TYPES = ((1,"Black List"), (2,"White List"), (3,"Grey List"))
type_list = models.IntegerField(choices=TYPES)
corpus = models.ManyToManyField(Corpus, related_name="relatedCorpus", blank=True)
def __str__(self):
return self.title
class ListNgram(models.Model):
liste = models.ForeignKey(List)
mainForm = models.ForeignKey(Ngram, related_name="mainForm")
othersForms = models.ManyToManyField(Ngram, related_name="otherForms", blank=True)
cvalue = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
def __str__(self):
return self.mainForm.terms
#class Postag(models.Model):
# abrev = models.CharField(max_length=30, unique=True)
# description = models.TextField(blank=True)
# def __str__(self):
# return(self.abrev)
######################################################################
# Coocurrences
# Graph
######################################################################
class Coocurrence(models.Model):
title = models.CharField(max_length=100)
corpus = models.ForeignKey(Corpus)
whitelists = models.ManyToManyField(List)
ngram1 = models.ForeignKey(Ngram, related_name="x")
ngram2 = models.ForeignKey(Ngram, related_name="y")
occurrence = models.IntegerField()
distance = models.DecimalField(max_digits=19, decimal_places=10,blank=True)
def __str__(self):
return self.title
class Clique(models.Model):
title = models.CharField(max_length=100)
ngram = models.ForeignKey(Ngram)
def __str__(self):
return self.title
class Phylo(models.Model):
date = models.DateField(blank=True, null=True)
clique = models.ForeignKey(Clique)
clique = models.ForeignKey(Clique)
score = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
def __str__(self):
return self.title
documents/test_documents.py deleted 100644 → 0
View file @ 4eab6f6f
from django.db import transaction
from django.contrib.auth.models import User
from documents.models import Project, Corpus, Document, Source, Language
from sources import importateur
from analysis import extract
try:
user = User.objects.get(username="alexandre")
except Exception as e:
print(e, "Error user")
# LANGUAGES
try:
french = Language.objects.get(language = "Français")
except:
french = Language()
french.language = "Français"
french.save()
try:
english = Language.objects.get(language = "English")
except:
english = Language()
english.language = "English"
english.save()
### DATABASES
try:
presse = Source.objects.get(database="Europresse")
except:
presse = Source()
presse.database = "Europresse"
presse.save()
try:
science = Source.objects.get(database="Web of Science (ISI format)")
except:
science = Source()
science.database = "Web of Science (ISI format)"
science.save()
# PROJECT
try:
p = Project.objects.get(title = "[TEST] Les abeilles")
except:
p = Project()
p.title = "[TEST] Les abeilles"
p.user = user
p.save()
# CORPORA
try:
c = Corpus.objects.get(title = "Presse francophone")
except:
c = Corpus()
c.project = p
c.database = presse
c.language = french
c.title = "Presse francophone"
c.user = user
c.zip_file = "corpora/" + user.username + "/" + "html_french.zip"
c.save()
try:
i = Corpus.objects.get(title = "Science ISI WOS")
i.zip_file = "corpora/" + user.username + "/" + "pesticidesOnly.zip"
i.save()
except:
i = Corpus()
i.project = p
i.database = science
i.language = french
i.title = "Science ISI WOS"
i.user = user
i.zip_file = "corpora/" + user.username + "/" + "isi.zip"
i.save()
def test_import(corpus):
try:
# TODO factorization here
importateur.importer(corpus.database, corpus.language, corpus.zip_file, project=corpus.project, corpus=corpus, user=corpus.user)
except Exception as e:
print(e)
test_import(i)
def test_words(corpus, field='abstract'):
try:
extract.words_field(corpus, field=field)
except Exception as e: print(e)
test_words(i)
documents/tests.py deleted 100644 → 0
View file @ 4eab6f6f
from django.test import TestCase
# Create your tests here.
documents/views.py deleted 100644 → 0
View file @ 4eab6f6f
from django.shortcuts import render
# Create your views here.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment