Skip to content

gargantext
gargantext
Commit 9d3682eb authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'master' of ssh://delanoe.org:1979/gargantext into mat-master

parents e80fdd78 d18065e7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 895 additions and 547 deletions
analysis/functions.py
View file @ 9d3682eb
...@@ -249,9 +249,10 @@ def tfidf(corpus, document, ngram): ...@@ -249,9 +249,10 @@ def tfidf(corpus, document, ngram):
xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count() xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
yy = Node_Ngram.objects.filter(ngram=ngram).count() yy = Node_Ngram.objects.filter(ngram=ngram).count()
idf= log(xx/yy) inverse_d_frequency= log(xx/yy)
result = tf * idf # result = tf * idf
result = term_frequency * inverse_d_frequency
except Exception as error: except Exception as error:
print(error) print(error)
result = 0 result = 0
......
analysis/tfidf.py deleted 100644 → 0
View file @ e80fdd78
from node.models import Node, NodeType, Node_Resource,\
Project, Corpus, Document,\
Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram,\
User, Language, ResourceType, Resource
from math import log
# - tfidf / corpus , type de corpus, tous corpus
# - tfidf / échelle de temps
# - tfidf / sources, auteurs etc.
# => liste de listes
def tfidf(corpus, document, ngram):
try:
x = Node_Ngram.objects.get(node=document, ngram=ngram).weight
y = Node_Ngram.objects.filter(node=document).count()
tf = x/y
xx = Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")).count()
yy = Node_Ngram.objects.filter(ngram=ngram).count()
idf= log(xx/yy)
result = tf * idf
except Exception as error:
print(error)
result = 0
return result
def do_tfidf(corpus, reset=True):
with transaction.atomic():
if reset==True:
NodeNodeNgram.objects.filter(nodex=corpus).delete()
if isinstance(corpus, Node) and corpus.type.name == "Corpus":
for document in Node.objects.filter(parent=corpus, type=NodeType.objects.get(name="Document")):
for node_ngram in Node_Ngram.objects.filter(node=document):
try:
nnn = NodeNodeNgram.objects.get(nodex=corpus, nodey=document, ngram=node_ngram.ngram)
except:
score = tfidf(corpus, document, node_ngram.ngram)
nnn = NodeNodeNgram(nodex=corpus, nodey=node_ngram.node, ngram=node_ngram.ngram, score=score)
nnn.save()
else:
print("Only implemented for corpus yet, whereas you put:", type(corpus))
gargantext_web/urls.py
View file @ 9d3682eb
...@@ -17,36 +17,43 @@ urlpatterns = patterns('', ...@@ -17,36 +17,43 @@ urlpatterns = patterns('',
url(r'^login/', include(admin.site.urls)), url(r'^login/', include(admin.site.urls)),
url(r'^grappelli/', include('grappelli.urls')), url(r'^grappelli/', include('grappelli.urls')),
# User views # User Home view
url(r'^$', views.home), url(r'^$', views.home),
# Project Management
url(r'^projects/$', views.projects), url(r'^projects/$', views.projects),
url(r'^project/(\d+)/delete/$', views.delete_project), url(r'^project/(\d+)/delete/$', views.delete_project),
url(r'^project/(\d+)/$', views.project), url(r'^project/(\d+)/$', views.project),
# Corpus management
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus), url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
url(r'^project/(\d+)/corpus/(\d+)/delete/$', views.delete_corpus), url(r'^project/(\d+)/corpus/(\d+)/delete/$', views.delete_corpus),
url(r'^project/(\d+)/corpus/(\d+)/corpus.csv$', views.corpus_csv),
url(r'^project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpus),
# Visualizations # Visualizations
url(r'^corpus/(\d+)/explorer$', views.explorer_graph), url(r'^project/(\d+)/corpus/(\d+)/chart$', views.chart),
url(r'^corpus/(\d+)/matrix$', views.explorer_matrix), url(r'^corpus/(\d+)/explorer$', views.graph),
url(r'^corpus/(\d+)/matrix$', views.matrix),
# Getting data # Data management
url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv), url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
url(r'^corpus/(\d+)/node_link.json$', views.node_link), url(r'^corpus/(\d+)/node_link.json$', views.node_link),
url(r'^corpus/(\d+)/adjacency.json$', views.adjacency), url(r'^corpus/(\d+)/adjacency.json$', views.adjacency),
url(r'^api/tfidf/(\d+)/(\w+)$', views.tfidf), url(r'^api/tfidf/(\d+)/(\w+)$', views.tfidf),
# Data management
url(r'^api$', gargantext_web.api.Root), url(r'^api$', gargantext_web.api.Root),
url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()), url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()), url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()), url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()), url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
url(r'^api/project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpusJSON),
url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams), url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
url(r'^graph-it$', views.graph_it),
url(r'^ngrams$', views.ngrams), url(r'^ngrams$', views.ngrams),
url(r'^nodeinfo/(\d+)$', views.nodeinfo), url(r'^nodeinfo/(\d+)$', views.nodeinfo),
url(r'^tests/mvc$', views.tests_mvc), url(r'^tests/mvc$', views.tests_mvc),
......
gargantext_web/views.py
View file @ 9d3682eb
...@@ -285,8 +285,8 @@ def corpus(request, project_id, corpus_id): ...@@ -285,8 +285,8 @@ def corpus(request, project_id, corpus_id):
project = Node.objects.get(id=project_id) project = Node.objects.get(id=project_id)
corpus = Node.objects.get(id=corpus_id) corpus = Node.objects.get(id=corpus_id)
#documents = corpus.children.all() type_doc = NodeType.objects.get(name="Document")
#number = corpus.children.count() number = Node.objects.filter(parent=corpus, type=type_doc).count()
# try: # try:
# sources = defaultdict(int) # sources = defaultdict(int)
...@@ -357,12 +357,171 @@ def corpus(request, project_id, corpus_id): ...@@ -357,12 +357,171 @@ def corpus(request, project_id, corpus_id):
'project': project,\ 'project': project,\
'corpus' : corpus,\ 'corpus' : corpus,\
'documents': documents,\ 'documents': documents,\
# 'number' : number,\ 'number' : number,\
'dates' : chart,\ 'dates' : chart,\
})) }))
return HttpResponse(html) return HttpResponse(html)
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
def subcorpus(request, project_id, corpus_id, start , end ):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = str(project_id)
offset = str(corpus_id)
offset = str(start)
offset = str(end)
except ValueError:
raise Http404()
# parameters received via web. Format = (yearmonthday = 20150106 = 06 jan 2015)
import datetime
dateini = datetime.datetime.strptime(str(start), '%Y%m%d').date()
datefin = datetime.datetime.strptime(str(end), '%Y%m%d').date()
t = get_template('subcorpus.html')
user = request.user
date = datetime.datetime.now()
project = Node.objects.get(id=project_id)
corpus = Node.objects.get(id=corpus_id)
# retrieving all the documents
documents = corpus.children.all()
number = corpus.children.count()
filtered_docs = []
# filtering documents by range-date
for doc in documents:
if "publication_date" in doc.metadata:
realdate = doc.metadata["publication_date"].split(" ")[0] # in database is = (year-month-day = 2015-01-06 00:00:00 = 06 jan 2015 00 hrs)
realdate = datetime.datetime.strptime(str(realdate), '%Y-%m-%d').date() # finalform = (yearmonthday = 20150106 = 06 jan 2015)
if dateini <= realdate <= datefin:
doc.date = realdate
filtered_docs.append(doc)
# ordering from most recent to the older.
ordered = sorted(filtered_docs, key=lambda x: x.date, reverse=True)
# pages of 10 elements. Like a sir.
paginator = Paginator(ordered, 10)
page = request.GET.get('page')
try:
results = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
results = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
results = paginator.page(paginator.num_pages)
html = t.render(Context({\
'user': user,\
'date': date,\
'project': project,\
'corpus' : corpus,\
'documents': results,\
# 'number' : len(filtered_docs),\
# 'dates' : chart,\
}))
return HttpResponse(html)
import json
def subcorpusJSON(request, project_id, corpus_id, start , end ):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = str(project_id)
offset = str(corpus_id)
offset = str(start)
offset = str(end)
except ValueError:
raise Http404()
# parameters received via web. Format = (yearmonthday = 20150106 = 06 jan 2015)
import datetime
dateini = datetime.datetime.strptime(str(start), '%Y%m%d').date()
datefin = datetime.datetime.strptime(str(end), '%Y%m%d').date()
t = get_template('subcorpus.html')
user = request.user
date = datetime.datetime.now()
project = Node.objects.get(id=project_id)
corpus = Node.objects.get(id=corpus_id)
# retrieving all the documents
documents = corpus.children.all()
number = corpus.children.count()
filtered_docs = []
# filtering documents by range-date
for doc in documents:
if "publication_date" in doc.metadata:
realdate = doc.metadata["publication_date"].split(" ")[0] # in database is = (year-month-day = 2015-01-06 00:00:00 = 06 jan 2015 00 hrs)
realdate = datetime.datetime.strptime(str(realdate), '%Y-%m-%d').date() # finalform = (yearmonthday = 20150106 = 06 jan 2015)
if dateini <= realdate <= datefin:
doc.date = realdate
filtered_docs.append(doc)
# ordering from most recent to the older.
ordered = sorted(filtered_docs, key=lambda x: x.date, reverse=True)
# pages of 10 elements. Like a sir.
paginator = Paginator(ordered, 10)
page = request.GET.get('page')
try:
results = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
results = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
results = paginator.page(paginator.num_pages)
from rest_framework.pagination import PaginationSerializer
serializer = PaginationSerializer(instance=results)
print(serializer.data)
html = t.render(Context({\
'user': user,\
'date': date,\
'corpus': corpus,\
}))
# return HttpResponse(html)
return HttpResponse( serializer.data , content_type='application/json')
# for pagexample.html
from django.core.paginator import Paginator, InvalidPage, EmptyPage
def get_pagination_page(page=1):
items = range(0, 100)
paginator = Paginator(items, 10)
try:
page = int(page)
except ValueError:
page = 1
try:
items = paginator.page(page)
except (EmptyPage, InvalidPage):
items = paginator.page(paginator.num_pages)
return items
def delete_project(request, node_id): def delete_project(request, node_id):
Node.objects.filter(id=node_id).all().delete() Node.objects.filter(id=node_id).all().delete()
return HttpResponseRedirect('/projects/') return HttpResponseRedirect('/projects/')
...@@ -371,8 +530,22 @@ def delete_corpus(request, project_id, corpus_id): ...@@ -371,8 +530,22 @@ def delete_corpus(request, project_id, corpus_id):
Node.objects.filter(id=corpus_id).all().delete() Node.objects.filter(id=corpus_id).all().delete()
return HttpResponseRedirect('/project/' + project_id) return HttpResponseRedirect('/project/' + project_id)
def explorer_graph(request, corpus_id):
t = get_template('explorer.html') def chart(request, project_id, corpus_id):
''' Charts to compare, filter, count'''
t = get_template('chart.html')
user = request.user
date = datetime.datetime.now()
project = Node.objects.get(id=project_id)
html = t.render(Context({
'user': user,
'date': date,
'project' : project,
}))
return HttpResponse(html)
def matrix(request, corpus_id):
t = get_template('matrix.html')
user = request.user user = request.user
date = datetime.datetime.now() date = datetime.datetime.now()
corpus = Node.objects.get(id=corpus_id) corpus = Node.objects.get(id=corpus_id)
...@@ -385,8 +558,8 @@ def explorer_graph(request, corpus_id): ...@@ -385,8 +558,8 @@ def explorer_graph(request, corpus_id):
return HttpResponse(html) return HttpResponse(html)
def explorer_matrix(request, corpus_id): def graph(request, corpus_id):
t = get_template('matrix.html') t = get_template('explorer.html')
user = request.user user = request.user
date = datetime.datetime.now() date = datetime.datetime.now()
corpus = Node.objects.get(id=corpus_id) corpus = Node.objects.get(id=corpus_id)
...@@ -399,6 +572,10 @@ def explorer_matrix(request, corpus_id): ...@@ -399,6 +572,10 @@ def explorer_matrix(request, corpus_id):
return HttpResponse(html) return HttpResponse(html)
def exploration(request): def exploration(request):
t = get_template('exploration.html') t = get_template('exploration.html')
user = request.user user = request.user
...@@ -426,6 +603,36 @@ def explorer_chart(request): ...@@ -426,6 +603,36 @@ def explorer_chart(request):
import csv import csv
from django.db import connection from django.db import connection
def corpus_csv(request, project_id, corpus_id):
'''
Create the HttpResponse object with the appropriate CSV header.
'''
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="corpus.csv"'
writer = csv.writer(response)
corpus = Node.objects.get(id=corpus_id)
type_document = NodeType.objects.get(name="Document")
documents = Node.objects.filter(parent=corpus, type=type_document)
keys = list(documents[0].metadata.keys())
writer.writerow(keys)
for doc in documents:
data = list()
for key in keys:
try:
data.append(doc.metadata[key])
except:
data.append("")
writer.writerow(data)
return response
def send_csv(request, corpus_id): def send_csv(request, corpus_id):
''' '''
Create the HttpResponse object with the appropriate CSV header. Create the HttpResponse object with the appropriate CSV header.
......
parsing/FileParsers/EuropressFileParser.py
View file @ 9d3682eb
This diff is collapsed.
parsing/test.py deleted 100644 → 0
View file @ e80fdd78
from NgramsExtractors import *
from Taggers import *
#texts = [
# "This is quite a simple test.",
# "Forman Brown (1901–1996) was one of the world's leaders in puppet theatre in his day, as well as an important early gay novelist. He was a member of the Yale Puppeteers and the driving force behind Turnabout Theatre. He was born in Otsego, Michigan, in 1901 and died in 1996, two days after his 95th birthday. Brown briefly taught at North Carolina State College, followed by an extensive tour of Europe.",
# "James Patrick (born c. 1940) is the pseudonym of a Scottish sociologist, which he used to publish a book A Glasgow Gang Observed. It attracted some attention in Scotland when it was published in 1973. It was based on research he had done in 1966, when he was aged 26. At that time he was working as a teacher in an Approved School, a Scottish reformatory. One gang member in the school, \"Tim Malloy\" (born 1950, also a pseudonym and a generic term for a Glasgow Catholic), agreed to infiltrate him into his gang in Maryhill in Glasgow. Patrick spent four months as a gang member, observing their behaviour.",
#]
#tagger = NltkTagger()
#extractor = EnglishNgramsExtractor()
#
texts = [
"La saison 1921-1922 du Foot-Ball Club Juventus est la vingtième de l'histoire du club, créé vingt-cinq ans plus tôt en 1897. La société turinoise qui fête cette année son 25e anniversaire prend part à l'édition du championnat dissident d'Italie de la CCI (appelé alors la Première division), la dernière édition d'une compétition annuelle de football avant l'ère fasciste de Mussolini.",
"Le terme oblong désigne une forme qui est plus longue que large et dont les angles sont arrondis. En langage bibliographique, oblong signifie un format dont la largeur excède la hauteur. Ce qui correspond au format paysage en termes informatiques et \"à l'italienne\", pour l'imprimerie.",
"Les sanglots longs des violons de l'automne bercent mon coeur d'une langueur monotone.",
]
tagger = TreeTagger()
extractor = FrenchNgramsExtractor()
for text in texts:
print(tagger.tag_text(text))
print()
ngrams = extractor.extract_ngrams(text)
for ngram in ngrams:
print("\t" + str(ngram))
print("\n")
templates/chart.html
View file @ 9d3682eb
This diff is collapsed.
templates/corpus.html
View file @ 9d3682eb
This diff is collapsed.
templates/node-info.html
View file @ 9d3682eb
...@@ -6,6 +6,33 @@ ...@@ -6,6 +6,33 @@
<link rel="stylesheet" href="{% static "css/bootstrap-theme.min.css" %}"> <link rel="stylesheet" href="{% static "css/bootstrap-theme.min.css" %}">
<link rel="stylesheet" href="{% static "js/libs/jquery/jquery-ui.css" %}" media="screen"> <link rel="stylesheet" href="{% static "js/libs/jquery/jquery-ui.css" %}" media="screen">
<style>
.nodeinfo-container {
margin: 25px;
padding: 25px;
}
.nodeinfo-elem {
border-style: solid;
border-width: 1px;
margin: 5px;
padding: 5px;
}
.nodeinfo-head {
font-size: large;
margin: 5px;
padding: 5px;
}
.nodeinfo-content {
margin: 5px;
padding: 5px;
}
</style>
{% endblock %} {% endblock %}
...@@ -16,6 +43,7 @@ ...@@ -16,6 +43,7 @@
<div id="metadata"></div> <div id="metadata"></div>
{% endif %} {% endif %}
<script src="{% static "js/jquery/jquery.min.js" %}" type="text/javascript"></script> <script src="{% static "js/jquery/jquery.min.js" %}" type="text/javascript"></script>
<script src="{% static "js/libs/jquery/jquery-ui.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/jquery/jquery-ui.js" %}" type="text/javascript"></script>
...@@ -44,11 +72,25 @@ ...@@ -44,11 +72,25 @@
jsondata = data.metadata jsondata = data.metadata
console.log(jsondata) console.log(jsondata)
var output = "" var output = ""
if(jsondata.title) output += "Title:<br>"+jsondata.title+"<br>"; // if(jsondata.title) output += "Title:<br>"+jsondata.title+"<br>";
if(jsondata.publication_date) output += jsondata.publication_date.split(" ")[0]+"<br><br>"; // if(jsondata.publication_date) output += jsondata.publication_date.split(" ")[0]+"<br><br>";
if(jsondata.authors) output += "Authors:<br>"+jsondata.authors+"<br><br>"; // if(jsondata.authors) output += "Authors:<br>"+jsondata.authors+"<br><br>";
if(jsondata.fields) output += "Keywords:<br>"+jsondata.fields+"<br><br>"; // if(jsondata.fields) output += "Keywords:<br>"+jsondata.fields+"<br><br>";
if(jsondata.abstract) output += "Abstract:<br>"+jsondata.abstract+"<br><br>"; // if(jsondata.abstract) output += "Abstract:<br>"+jsondata.abstract+"<br><br>";
// <div class='jumbotron'>
output += "<div class='nodeinfo-container'>"
if(jsondata.title) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Title</div> <div class='nodeinfo-content'>"+jsondata.title+"</div> </div>";
if(jsondata.publication_date) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Publication Date</div> <div class='nodeinfo-content'>"+jsondata.publication_date.split(" ")[0]+"</div> </div>";
if(jsondata.authors && jsondata.authors!="not found") output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Authors</div> <div class='nodeinfo-content'>"+jsondata.authors+"</div> </div>";
else {
if(jsondata.source) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Published in</div> <div class='nodeinfo-content'>"+jsondata.source+"</div> </div>";
}
if(jsondata.fields) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Keywords</div> <div class='nodeinfo-content'>"+jsondata.fields+"</div> </div>";
if(jsondata.abstract) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Abstract</div> <div class='nodeinfo-content'>"+jsondata.abstract+"</div> </div>";
else {
if(jsondata.text) output += "<div class='nodeinfo-elem'> <div class='nodeinfo-head'>Abstract</div> <div class='nodeinfo-content'>"+jsondata.text+"</div> </div>";
}
output += "</div>"
$("#metadata").html(output); $("#metadata").html(output);
}, },
......
templates/subcorpus.html 0 → 100644
View file @ 9d3682eb
{% if date %}
<p>Today: {{date}}</p>
{% endif %}
<div class="pagination">
<span class="step-links">
{% if documents.has_previous %}
<a onclick="updateDocuments({{ documents.previous_page_number }},true);">previous</a>
{% endif %}
<span class="current">
Page {{ documents.number }} of {{ documents.paginator.num_pages }}.
</span>
{% if documents.has_next %}
<a onclick="updateDocuments({{ documents.next_page_number }},true);">next</a>
{% endif %}
</span>
</div>
{% if documents %}
<p>Paginator stuff</p>
<ul>
{% for doc in documents %}
{% if doc.date %}
<li> <b>{{ doc.date }}</b>, <a href="/admin/node/document/{{doc.id}}">id:{{ doc.id}} title:{{ doc.name}}</a></li>
{% endif %}
{% endfor %}
</ul>
{% endif %}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment