Skip to content

gargantext
gargantext
Commit 8b81babb authored by Administrator's avatar Administrator
Browse files
Options

Merge branch 'alex'

parents 832351f8 afd763af
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1732 additions and 162 deletions
.ipynb_checkpoints/Untitled0-checkpoint.ipynb deleted 100644 → 0
View file @ 832351f8
{
"metadata": {
"name": "",
"signature": "sha256:a5146fbde2b6bf2e3ed4e2bdddfb62662f99272f26e82bf86110680ff3595332"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"from node.models import Node, NodeType, Language\n",
"import parsing\n",
"from parsing.FileParsers import *"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"node = Node.objects.get(name=\"PubMed corpus\")"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"fileparser = PubmedFileParser.PubmedFileParser(file='/var/www/gargantext/media/' + node.fichier.name)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"fileparser.parse(node)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Cannot assign \"24\": \"Node.user\" must be a \"User\" instance.",
"output_type": "pyerr",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-4-8c1443001599>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mfileparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[1;32m/srv/gargantext/parsing/FileParsers/PubmedFileParser.py\u001b[0m in \u001b[0;36mparse\u001b[1;34m(self, parentNode, tag)\u001b[0m\n\u001b[0;32m 45\u001b[0m \u001b[0mlanguage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_languages_iso3\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"language_iso3\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 46\u001b[0m \u001b[0mmetadata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m \u001b[0mguid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"doi\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 48\u001b[0m )\n\u001b[0;32m 49\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdocument\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/srv/gargantext/parsing/FileParsers/FileParser.py\u001b[0m in \u001b[0;36mcreate_document\u001b[1;34m(self, parentNode, title, contents, language, metadata, guid)\u001b[0m\n\u001b[0;32m 100\u001b[0m \u001b[0mmetadata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 101\u001b[0m \u001b[1;31m#resource = resource,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m \u001b[0mparent\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparentNode\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 103\u001b[0m )\n\u001b[0;32m 104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/models/base.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 403\u001b[0m \u001b[1;31m# \"user_id\") so that the object gets properly cached (and type\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 404\u001b[0m \u001b[1;31m# checked) by the RelatedObjectDescriptor.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 405\u001b[1;33m \u001b[0msetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfield\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrel_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 406\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 407\u001b[0m \u001b[0msetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfield\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mattname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/models/fields/related.py\u001b[0m in \u001b[0;36m__set__\u001b[1;34m(self, instance, value)\u001b[0m\n\u001b[0;32m 337\u001b[0m raise ValueError('Cannot assign \"%r\": \"%s.%s\" must be a \"%s\" instance.' %\n\u001b[0;32m 338\u001b[0m (value, instance._meta.object_name,\n\u001b[1;32m--> 339\u001b[1;33m self.field.name, self.field.rel.to._meta.object_name))\n\u001b[0m\u001b[0;32m 340\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 341\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minstance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdb\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mValueError\u001b[0m: Cannot assign \"24\": \"Node.user\" must be a \"User\" instance."
]
}
],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"node.children.all()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
\ No newline at end of file
gargantext_web/views.py
View file @ 8b81babb
......@@ -5,7 +5,7 @@ from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.template.loader import get_template
from django.template import Context
from documents.models import Project, Corpus, Document
#from documents.models import Project, Corpus, Document
from node.models import Node, NodeType
from django.contrib.auth.models import User
......@@ -15,6 +15,9 @@ from itertools import *
from django.db import connection
from django import forms
from collections import defaultdict
# SOME FUNCTIONS
def query_to_dicts(query_string, *query_args):
......@@ -151,28 +154,25 @@ def corpus(request, project_id, corpus_id):
number = corpus.children.count()
try:
sources = query_to_dicts('''select count(*), source
from node_node as t1
INNER JOIN documents_document_corpus as t2
ON ( t1.id = t2.document_id )
WHERE ( t1.user_id = %d AND t2.corpus_id = %d )
GROUP BY source
order by 1 DESC limit %d;''' % (request.user.pk, int(corpus_id), int(15)))
sources = defaultdict(int)
for document in documents.all():
sources[document.metadata['journal']] += 1
sources_donut = []
for source in sources.keys():
source_count = dict()
source_count['count'] = source['count']
try:
source_count['part'] = round(source_count['count'] * 100 / number)
except:
source_count['part'] = None
source_count['source'] = source['source']
sources_donut.append(source_count)
except:
pass
sources_donut = []
for s in sources:
ss = dict()
ss['count'] = s['count']
try:
ss['part'] = round(ss['count'] * 100 / number)
except:
ss['part'] = None
ss['source'] = s['source']
sources_donut.append(ss)
try:
first = documents.first().date
last = documents.last().date
......@@ -190,13 +190,14 @@ def corpus(request, project_id, corpus_id):
date_form = 'days'
try:
dates = query_to_dicts('''select to_char(t1.date, '%s'), count(*)
from documents_document as t1
INNER JOIN documents_document_corpus as t2
ON ( t1.id = t2.document_id )
WHERE ( t1.user_id = %d AND t2.corpus_id = %d )
group by to_char(t1.date, '%s')
order by 1 DESC;''' % (date_format, request.user.pk, int(corpus_id), date_format))
dates = dict()
# query_to_dicts('''select to_char(t1.date, '%s'), count(*)
# from documents_document as t1
# INNER JOIN documents_document_corpus as t2
# ON ( t1.id = t2.document_id )
# WHERE ( t1.user_id = %d AND t2.corpus_id = %d )
# group by to_char(t1.date, '%s')
# order by 1 DESC;''' % (date_format, request.user.pk, int(corpus_id), date_format))
except:
pass
......
init/dependances.deb
View file @ 8b81babb
# Paquets Debian a installer
sudo apt-get install python-virtualenv
sudo apt-cache search libpng
sudo apt-get install libpng12-dev
sudo apt-get install libpng-dev
apt-cache search freetype
apt-cache search freetype | grep dev
sudo apt-get install libfreetype6-dev
sudo apt-cache search python-dev
sudo apt-get install python-dev
sudo apt-get install libpq-dev
postgresql-contrib
libpq-dev
# rajouter david
#
#
#Paquets Debian a installer
# easy_install -U distribute (matplotlib)
#lxml
libffi-dev
......
init/init.py
View file @ 8b81babb
from node.models import NodeType
from django.contrib.auth.models import User
user = User.objects.get(username="alexandre")
#NodeType.objects.all().delete()
NodeType(name="Root").save()
NodeType(name="Project").save()
type_root = NodeType(name="Root")
type_root.save()
type_project = NodeType(name="Project")
type_project.save()
NodeType(name="Corpus").save()
NodeType(name="Document").save()
from node.models import Project
Project(name="Projet sur les abeilles", user=user, type=type_project).save()
from node.models import DatabaseType
for bdd in ['Europresse', 'PubMed', 'Web Of Science (WOS), ISI format']:
DatabaseType(name=bdd).save()
......
init/init.sh
View file @ 8b81babb
......@@ -3,7 +3,7 @@
psql -d gargandb -f init.sql
sleep 2
./manage.py syncdb
../manage.py syncdb
sleep 2
./manage.py shell < init.py
../manage.py shell < init.py
node/admin.py
View file @ 8b81babb
from django.contrib import admin
from django.forms import ModelForm, ModelChoiceField
from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
from node.models import NodeType, Language, Node, Project, Corpus, Document
from node.models import NodeType, Language, Node, Project, Corpus, Document, DatabaseType, Resource
class ResourceInLine(admin.TabularInline):
model = Resource
extra = 0
class NodeAdmin(admin.ModelAdmin):
exclude = ('user', 'path', 'depth', 'numchild')
......@@ -9,7 +14,7 @@ class NodeAdmin(admin.ModelAdmin):
search_fields = ('name',)
# list_filter = ('type',)
# date_hierarchy
# inlines = [CorpusInLine,]
#inlines = [ResourceInLine,]
#_nodetype_name = 'Project'
#_parent_nodetype_name = 'Root'
......@@ -50,7 +55,8 @@ class NodeAdmin(admin.ModelAdmin):
parent = nodeParent,\
user = request.user,\
name = obj.name,\
metadata= obj.metadata)
metadata= obj.metadata,\
resource = obj.resource)
#nodeParent.save()
#node.save()
......@@ -108,10 +114,11 @@ class LanguageAdmin(admin.ModelAdmin):
class Meta:
ordering = ['fullname',]
admin.site.register(NodeType)
admin.site.register(Resource)
admin.site.register(DatabaseType)
admin.site.register(Language, LanguageAdmin)
admin.site.register(NodeType)
admin.site.register(Project, ProjectAdmin)
admin.site.register(Corpus, CorpusAdmin)
admin.site.register(Document, DocumentAdmin)
......
node/models.py
View file @ 8b81babb
......@@ -39,9 +39,10 @@ class Ngram(models.Model):
terms = models.CharField(max_length=255)
class Resource(models.Model):
user = models.ForeignKey(User)
guid = models.CharField(max_length=255)
bdd_type = models.ForeignKey(DatabaseType, blank=True, null=True)
#file = models.FileField(upload_to=upload_to, blank=True)
file = models.FileField(upload_to=upload_to, blank=True)
class NodeType(models.Model):
name = models.CharField(max_length=200)
......@@ -60,9 +61,8 @@ class Node(CTENode):
date = models.DateField(default=timezone.now, blank=True)
metadata = hstore.DictionaryField(blank=True)
fichier = models.FileField(upload_to=upload_to, blank=True)
#resource = models.ForeignKey(Resource, blank=True, null=True)
#ngrams = models.ManyToManyField(NGrams)
resource = models.ManyToManyField(Resource, blank=True)
ngrams = models.ManyToManyField(Ngram, blank=True)
def __str__(self):
......
templates/corpus.html
View file @ 8b81babb
......@@ -66,7 +66,7 @@
<div class="col-md-4">
<div class="jumbotron">
<h3>Catography</h3>
<h3>Cartography</h3>
<ol>
<li>Feature</li>
<li>Feature</li>
......
test_parser.ipynb 0 → 100644
View file @ 8b81babb
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment