Skip to content

gargantext
gargantext
Commit e5c57cba authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'master' into mat-master 

Conflicts:
	.gitignore
	analysis/functions.py
	gargantext_web/views.py
	init/README.rst
	init/dependances.sh
	init/requirements.txt
	templates/corpus.html
	templates/matrix.html
parents aee4893c ccaeea62
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 260 additions and 27 deletions
.gitignore
View file @ e5c57cba
__pycache__/
parsing/Taggers/treetagger/
.ipynb_checkpoints/
<<<<<<< HEAD
*.pyc
=======
data_samples
VENV
>>>>>>> master
analysis/functions.py
View file @ e5c57cba
......@@ -69,7 +69,11 @@ def create_whitelist(user, corpus, size=100):
return white_list
#def create_cooc(user, corpus, whitelist, blacklist, synonymes):
<<<<<<< HEAD
def create_cooc(user=None, corpus=None, whitelist=None, size=150):
=======
def create_cooc(user=None, corpus=None, whitelist=None, size=150, year_start=None, year_end=None):
>>>>>>> master
cursor = connection.cursor()
try:
......@@ -133,7 +137,11 @@ def create_cooc(user=None, corpus=None, whitelist=None, size=150):
cursor.execute(query_cooc)
return cooc
<<<<<<< HEAD
def get_cooc(request=None, corpus_id=None, cooc_id=None, type="node_link"):
=======
def get_cooc(request=None, corpus_id=None, cooc_id=None, type=None, n=150):
>>>>>>> master
import pandas as pd
from copy import copy
import numpy as np
......@@ -152,11 +160,14 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type="node_link"):
if Node.objects.filter(type=type_cooc, parent=corpus).first() is None:
print("Coocurrences do not exist yet, create it.")
<<<<<<< HEAD
if type == "node_link":
n = 150
elif type == "adjacency":
n = 50
=======
>>>>>>> master
whitelist = create_whitelist(request.user, corpus, size=n)
cooccurrence_node = create_cooc(user=request.user, corpus=corpus, whitelist=whitelist, size=n)
print(cooccurrence_node.id, "Cooc created")
......@@ -174,7 +185,11 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type="node_link"):
weight[cooccurrence.ngramx.terms] = weight.get(cooccurrence.ngramx.terms, 0) + cooccurrence.score
<<<<<<< HEAD
df = pd.DataFrame(matrix).T.fillna(0)
=======
df = pd.DataFrame(matrix).fillna(0)
>>>>>>> master
x = copy(df.values)
x = x / x.sum(axis=1)
......@@ -194,8 +209,16 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type="node_link"):
partition = best_partition(G)
<<<<<<< HEAD
if type == "node_link":
=======
if type == "node_link":
for community in set(partition.values()):
#print(community)
G.add_node("cluster " + str(community), hidden=1)
>>>>>>> master
for node in G.nodes():
try:
#node,type(labels[node])
......@@ -203,7 +226,11 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type="node_link"):
G.node[node]['name'] = node
G.node[node]['size'] = weight[node]
G.node[node]['group'] = partition[node]
<<<<<<< HEAD
#G.add_edge(node, partition[node], weight=3)
=======
G.add_edge(node, "cluster " + str(partition[node]), weight=3)
>>>>>>> master
# G.node[node]['color'] = '19,180,300'
except Exception as error:
print(error)
......
gargantext_web/settings.py
View file @ e5c57cba
......@@ -41,7 +41,7 @@ TEMPLATE_DIRS = (
#ALLOWED_HOSTS = ['*',]
ALLOWED_HOSTS = ['localhost', 'master.polemic.be', 'mat.polemic.be', 'alexandre.polemic.be']
ALLOWED_HOSTS = ['localhost', 'master.polemic.be', 'beta.gargantext.org']
......
gargantext_web/views.py
View file @ e5c57cba
......@@ -5,11 +5,10 @@ from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.template.loader import get_template
from django.template import Context
#from documents.models import Project, Corpus, Document
from node.models import Language, ResourceType, Resource, \
Node, NodeType, Node_Resource, Project, Corpus, \
Node_Ngram, NodeNgramNgram
from node.admin import CorpusForm, ProjectForm, ResourceForm
from django.contrib.auth.models import User
......@@ -143,6 +142,7 @@ def project(request, project_id):
type_corpus = NodeType.objects.get(name='Corpus')
type_document = NodeType.objects.get(name='Document')
type_whitelist = NodeType.objects.get(name='WhiteList')
type_blacklist = NodeType.objects.get(name='BlackList')
type_cooclist = NodeType.objects.get(name='Cooccurrence')
......@@ -158,17 +158,17 @@ def project(request, project_id):
# List of resources
# filter for each project here
whitelists = Node.objects.filter( type=type_whitelist)
blacklists = Node.objects.filter( type=type_blacklist)
cooclists = Node.objects.filter( type=type_cooclist)
whitelists = ""#.children.filter(type=type_whitelist)
blacklists = ""#.children.filter(type=type_blacklist)
cooclists = ""#.children.filter(type=type_cooclist)
for corpus in corpora:
docs_count = corpus.children.count()
docs_total += docs_count
corpus_view = dict()
corpus_view['id'] = corpus.pk
corpus_view['name'] = corpus.name
corpus_view['id'] = corpus.pk
corpus_view['name'] = corpus.name
corpus_view['count'] = corpus.children.count()
for node_resource in Node_Resource.objects.filter(node=corpus):
......@@ -237,12 +237,17 @@ def project(request, project_id):
)
try:
<<<<<<< HEAD
#corpus.parse_resources.apply_async((), countdown=1)
corpus.parse_resources()
# async
corpus.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
#corpus.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
=======
corpus.parse_and_extract_ngrams()
#corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
>>>>>>> master
except Exception as error:
print(error)
......@@ -471,7 +476,10 @@ def send_csv(request, corpus_id):
return response
<<<<<<< HEAD
=======
>>>>>>> master
# To get the data
from gargantext_web.api import JsonHttpResponse
from analysis.functions import get_cooc
......@@ -505,6 +513,7 @@ def graph_it(request):
}))
return HttpResponse(html)
<<<<<<< HEAD
def tests_mvc(request):
'''Just a test page for Javascript MVC.'''
t = get_template('tests/mvc.html')
......@@ -516,6 +525,8 @@ def tests_mvc(request):
}))
return HttpResponse(html)
=======
>>>>>>> master
def ngrams(request):
'''The ngrams list.'''
t = get_template('ngrams.html')
......
init/README.rst
View file @ e5c57cba
......@@ -59,6 +59,7 @@ Populate the database
python manage.py syncdb
<<<<<<< HEAD
Last steps of configuration
---------------------------
......@@ -93,6 +94,40 @@ Last steps of configuration
Extras
=======
Last steps of configuration:
----------------------------
1) If your project is not in /srv/gargantext:
ln -s [the project folder] /srv/gargantext
2) build gargantext_lib
wget http://docs.delanoe.org/gargantext_lib.tar.bz2
cd /srv/
sudo tar xvjf gargantext_lib.tar.bz2
sudo chown user:user /srv/gargantext_lib
3) Explorer:
create mkdir /srv/gargantext_lib/js
sudo chown -R user:user /srv/gargantext_lib/
cd /srv/gargantext_lib/js
git clone git@github.com:PkSM3/garg.git
4) Adapt all symlinks:
ln -s [your folder for tree tagger] [the project folder]/parsing/Tagger/treetagger
Warning: for ln, path has to be absolute!
5) patch CTE
patch /srv/gargantext_env/lib/python3.4/site-packages/cte_tree/models.py /srv/gargantext/init/cte_tree.models.diff
6) init nodetypes and main variables
/srv/gargantext/manage.py shell < /srv/gargantext/init/init.py
Extras:
>>>>>>> master
======
Start the Python Notebook server
......@@ -106,6 +141,6 @@ Start the Python Notebook server
Start the Django server
-----------------------
in bash to launch python env : /srv/gargantext_env/bin/activate
In Pyvenv:
python manage.py runserver
init/dependances.sh
View file @ e5c57cba
......@@ -30,3 +30,8 @@ sudo apt-get install libopenblas-dev
sudo apt-get install liblapack-dev
<<<<<<< HEAD
=======
source /srv/gargantext_env/bin/activate
pip install git+https://github.com/mathieurodic/aldjemy.git
>>>>>>> master
init/requirements.txt
View file @ e5c57cba
......@@ -6,7 +6,11 @@ Pillow==2.5.3
Pygments==1.6
SQLAlchemy==0.9.8
South==1.0
<<<<<<< HEAD
# aldjemy==0.3.51
=======
aldjemy==0.3.10
>>>>>>> master
amqp==1.4.6
anyjson==0.3.3
billiard==3.3.0.18
......@@ -26,6 +30,7 @@ django-hstore==1.3.1
django-mptt==0.6.1
django-nested-inlines==0.1
django-treebeard==2.0
djangorestframework==3.0.0
graphviz==0.4
ipython==2.2.0
kombu==3.0.23
......
init/update_20141204.sh 0 → 100755
View file @ e5c57cba
source /srv/gargantext_env/bin/activate
pip install git+https://github.com/mathieurodic/aldjemy.git
pip install djangorestframework==3.0.0
cd /srv/gargantext/
./manage.py syncdb
echo "Ok runserver!"
node/models.py
View file @ e5c57cba
......@@ -157,7 +157,6 @@ class Node(CTENode):
node_resource.save()
return resource
@current_app.task(filter=task_method)
def parse_resources(self, verbose=False):
# parse all resources into a list of metadata
metadata_list = []
......@@ -235,6 +234,13 @@ class Node(CTENode):
for ngram_text, weight in associations.items()
])
@current_app.task(filter=task_method)
def parse_and_extract_ngrams(self, keys=None, ngramsextractorscache=None, ngramscaches=None, verbose=False):
self.parse_resources()
type_document = NodeType.objects.get(name='Document')
self.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
class Node_Metadata(models.Model):
node = models.ForeignKey(Node)
metadata = models.ForeignKey(Metadata)
......
parsing/Taggers/treetagger
View file @ e5c57cba
/home/mat/projects/gargantext/old/gargantext-stable-2/shared/treetagger
\ No newline at end of file
/srv/gargantext_lib/treetagger
\ No newline at end of file
static/favicon.ico
View file @ e5c57cba
No preview for this file type
static/img/logo.svg 0 → 100644
View file @ e5c57cba
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="48px"
height="48px"
id="svg4362"
version="1.1"
inkscape:version="0.48.5 r10040"
sodipodi:docname="logo.svg">
<defs
id="defs4364" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="5.6897594"
inkscape:cx="-11.235831"
inkscape:cy="3.8560006"
inkscape:current-layer="layer1"
showgrid="true"
inkscape:grid-bbox="true"
inkscape:document-units="px"
inkscape:window-width="1360"
inkscape:window-height="762"
inkscape:window-x="0"
inkscape:window-y="0"
inkscape:window-maximized="0" />
<metadata
id="metadata4367">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
id="layer1"
inkscape:label="Layer 1"
inkscape:groupmode="layer">
<rect
style="fill:#fffcfc;fill-opacity:1;stroke:none"
id="rect3755"
width="29.70249"
height="31.108515"
x="0"
y="-0.1566938"
inkscape:export-filename="/srv/gargantext/static/img/logo.png"
inkscape:export-xdpi="53"
inkscape:export-ydpi="53" />
<g
inkscape:export-ydpi="53.799999"
inkscape:export-xdpi="53.799999"
inkscape:export-filename="/srv/gargantext/static/img/logo.png"
style="fill:#ff8080;fill-opacity:0.82014388"
id="g3835"
transform="matrix(0.2422549,0,0,0.23374214,-49.789462,-7.9055988)">
<path
inkscape:export-ydpi="100"
inkscape:export-xdpi="100"
inkscape:export-filename="/home/alexandre/projets/gargantext.py/gargantext_core/shared/LogoSimple.png"
id="path3837"
d="m 206.24721,35.28586 0,129.5 67.78125,0 0,-8.625 c -9.86526,-0.47262 -18.57934,-2.63259 -25.5625,-6.28125 -18.65918,-9.74237 -29.875,-28.26535 -29.875,-49.1875 0,-31.71741 21.11877,-52.8149 55.4375,-55.1875 l 0,-10.21875 -67.78125,0 z m 67.78125,10.21875 0,8.5 c 1.74191,-0.16369 3.53543,-0.28125 5.37499,-0.28125 6.91081,0 13.295,1.44116 19.6875,4.15625 l 2.40625,2.875 2.59375,14.53125 9.6875,0 0,-25.375 c -11.40283,-3.03451 -22.61727,-4.65625 -33.15625,-4.65625 -2.24526,0 -4.44959,0.10177 -6.59374,0.25 z m 0,8.5 c -23.28864,2.18852 -37.65625,18.81513 -37.65625,45.562503 0,27.600037 14.44681,45.025437 37.65625,47.812497 l 0,-93.375 z m 0,93.375 0,8.78125 c 1.36224,0.0653 2.75177,0.0937 4.15624,0.0937 10.19344,0 22.1324,-1.88915 35.78125,-5.5625 l 0,-38.1875 2.9375,-2.21875 9.5,-0.8125 0,-6.5625 -43.21875,0 0,6.5625 12.28125,0.8125 2.9375,2.21875 0,33.21875 c -6.73804,1.4374 -12.61466,2.09375 -17.625,2.09375 -2.32322,0 -4.57592,-0.17643 -6.74999,-0.4375 z"
style="font-size:166.11251831px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#ff8080;fill-opacity:0.82014388;stroke:none;font-family:Bitstream Charter;-inkscape-font-specification:Bitstream Charter"
inkscape:connector-curvature="0" />
<path
inkscape:export-ydpi="100"
inkscape:export-xdpi="100"
transform="translate(611.62306,-400.10238)"
sodipodi:open="true"
sodipodi:end="6.1660663"
sodipodi:start="0"
d="m -312.87112,480.17926 c 0,4.97881 -4.03612,9.01493 -9.01493,9.01493 -4.97881,0 -9.01493,-4.03612 -9.01493,-9.01493 0,-4.97881 4.03612,-9.01493 9.01493,-9.01493 4.57131,0 8.41901,3.42153 8.95317,7.96152"
sodipodi:ry="9.0149298"
sodipodi:rx="9.0149298"
sodipodi:cy="480.17926"
sodipodi:cx="-321.88605"
id="path3839"
style="fill:#ff8080;fill-opacity:0.82014388;stroke:none"
sodipodi:type="arc" />
</g>
</g>
</svg>
static/img/miserables.json deleted 100644 → 0
View file @ aee4893c
This diff is collapsed.
templates/corpus.html
View file @ e5c57cba
......@@ -116,9 +116,9 @@
<div class="jumbotron">
<h3><a href="/ngrams">Dictionaries</a></h3>
<ol>
<li>White Lists</li>
<li>Black Lists</li>
<li>Synonyms</li>
<li>Black Lists</li>
<li>White Lists</li>
</ol>
</div>
</div>
......@@ -129,7 +129,11 @@
<ol>
<li><a href="/corpus/{{ corpus.id }}/matrix">Adjacency matrix</a></li>
<li><a href="/corpus/{{ corpus.id }}/explorer">Static maps</a></li>
<<<<<<< HEAD
<li>Dynamics maps</li>
=======
<li>Dynamic maps</li>
>>>>>>> master
</ol>
</div>
</div>
......
templates/explorer.html
View file @ e5c57cba
......@@ -78,13 +78,21 @@
<ul id="category-A" class="nav navbar-nav navbar-right">
<li>
<ul style="list-style-type: none; margin:0em 1em;">
<ul >
<li><div id="sliderANodeWeight"></div></li>
<li><div id="sliderAEdgeWeight"></div></li>
<li><div id="sliderANodeSize"></div></li>
</ul>
</li>
</ul>
<!--
a
a
a
a
a
a
-->
<ul id="filterslegend" class="nav navbar-nav navbar-right">
<li>
......@@ -99,7 +107,7 @@
<ul id="category-B" class="nav navbar-nav navbar-right">
<li>
<ul style="list-style-type: none; margin:0em 1em;">
<ul>
<li><div id="sliderBNodeWeight"></div></li>
<li><div id="sliderBEdgeWeight"></div></li>
<li><div id="sliderBNodeSize"></div></li>
......
templates/matrix.html
View file @ e5c57cba
......@@ -5,7 +5,11 @@
{% load staticfiles %}
<link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" href="{% static "css/bootstrap-theme.min.css" %}">
<<<<<<< HEAD
<link rel="stylesheet" href="{% static "css/d3matrix.css" %}">
=======
<!-- <link rel="stylesheet" href="{% static "css/d3matrix.css" %}"> -->
>>>>>>> master
<style>
.background {
......@@ -29,10 +33,15 @@ text.active {
<div class="container theme-showcase" role="main">
<div class="jumbotron">
<h1>Adjancy Matrix</h1>
<h1>Adjacency Matrix</h1>
</div>
</div>
<<<<<<< HEAD
=======
<div class="container">
>>>>>>> master
<div id="graphid" style="visibility: hidden;">/corpus/{{ corpus.id }}/adjacency.json</div>
......@@ -46,6 +55,8 @@ text.active {
</select>
</p>
</div>
<script>
var margin = {top: 80, right: 0, bottom: 10, left: 80},
......@@ -59,12 +70,20 @@ var x = d3.scale.ordinal().rangeBands([0, width]),
var svg = d3.select("body").append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
<<<<<<< HEAD
//.style("margin-left", -margin.left + "px")
=======
.style("margin-left", "25%")
>>>>>>> master
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var filename = document.getElementById("graphid").innerHTML
d3.json(filename, function(miserables) {
<<<<<<< HEAD
=======
>>>>>>> master
var matrix = [],
nodes = miserables.nodes,
n = nodes.length;
......
templates/menu.html
View file @ e5c57cba
......@@ -6,6 +6,7 @@
</head>
{% load staticfiles %}
<body>
<!-- Fixed navbar -->
<div id="dafixedtop" class="navbar navbar-inverse navbar-fixed-top" role="navigation">
......@@ -16,7 +17,7 @@
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="/">Gargantext</a>
<a class="navbar-brand" style="line-height:15px; height:10px; padding: 10px 10px;" href="/"><img src="{% static "img/logo.svg" %}"></a>
</div>
<div class="navbar-collapse collapse">
......
templates/project.html
View file @ e5c57cba
......@@ -68,17 +68,20 @@
<!-- Add jumbotron container for each type of corpus (presse, science etc.) --!>
<div class="container">
<h1>Resources</h1>
<h2>Corpora</h2>
{% if list_corpora %}
{% if list_corpora %}
<h1>Resources</h1>
<h2>Corpora</h2>
<ul>
{% for key, corpora in list_corpora.items %}
<li>{{ key }}</li>
<ul>
{% for corpus in corpora %}
<li>
<li> {% ifnotequal corpus.count 0 %}
<a href="/project/{{project.id}}/corpus/{{corpus.id}}">{{corpus.name}}</a>
, {{ corpus.count }} Documents
{% else %}
{{corpus.name}} : Processing, drink a cup of tea, and refresh the page :)
{% endifnotequal %}
<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
data-content='
<ul>
......@@ -92,7 +95,7 @@
</ul>
{% endfor %}
</ul>
{% endif %}
{% endif %}
{% if list_corporax %}
......@@ -105,9 +108,9 @@
</div>
{% endif %}
<h2>Lists of Ngrams</h2>
{% if whitelists %}
<h3>White Lists</h2>
<h2>Lists of Ngrams</h2>
<h3>White Lists</h2>
{% for list in whitelists %}
<ul>
<li> {{list.name }}
......@@ -126,8 +129,8 @@
<h2>Results (graphs)</h2>
{% if cooclists %}
{% if cooclists %}
<h2>Results (graphs)</h2>
<h3>Cooccurrences Lists</h2>
{% for list in cooclists %}
<ul>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment