Commit 64ead7e6 authored by Administrator's avatar Administrator

Merge branch 'samuel' into merge

parents 3edc8ef3 c5f3fa91
......@@ -6,4 +6,5 @@ urlpatterns = patterns('',
url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view
url(r'^corpus/(?P<corpus_id>[0-9]+)/document/(?P<doc_id>[0-9]+)$', views.NgramList.as_view()), # the list associated with an ngram
url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.NgramEdit.as_view()), #
url(r'^lists/(?P<list_id>[0-9]+)/multiple?$', views.deleteMultiple ), #
)
......@@ -16,6 +16,8 @@ from rest_framework.authentication import SessionAuthentication, BasicAuthentica
from node.models import Node
from gargantext_web.db import *
from ngram.lists import listIds, listNgramIds, ngramList
from gargantext_web.api import JsonHttpResponse
import json
@login_required
......@@ -109,6 +111,26 @@ class NgramEdit(APIView):
session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).delete()
return Response(None, 204)
def deleteMultiple(request, list_id):
results = ["hola","mundo"]
user = request.user
if not user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
if request.POST:
todel_ids = json.loads(request.POST['to_delete'])
for ngram_id in todel_ids:
# add the ngram to the list if not already done
node_ngram = session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).first()
if node_ngram is None:
node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
session.add(node_ngram)
session.commit()
return JsonHttpResponse(results)
class Document(APIView):
"""
......
......@@ -85,7 +85,10 @@ urlpatterns = patterns('',
url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', samtest.test_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams)
url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams),
url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', samtest.test_journals),
url(r'^project/(\d+)/corpus/(\d+)/journals$', samtest.get_journals),
url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
)
......
......@@ -98,8 +98,8 @@ from django import forms
from django.utils.translation import ugettext_lazy as _
class CustomForm(forms.Form):
name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
type = ModelChoiceField( ResourceType.objects.all() , widget=forms.Select(attrs={'onchange':'CustomForSelect( $("option:selected", this).text() );'}) )
name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
file = forms.FileField()
# Description: clean_file()
......
......@@ -269,11 +269,10 @@ def testISTEX(request , project_id):
type_id = resourcetype.id,
file = filename,
)
dwnldsOK+=1
# print(urlreqs)
dwnldsOK+=1
if dwnldsOK == 0: return JsonHttpResponse(["fail"])
###########################
###########################
try:
if not DEBUG:
apply_workflow.apply_async((corpus.id,),)
......@@ -287,50 +286,6 @@ def testISTEX(request , project_id):
return HttpResponseRedirect('/project/' + str(project_id))
# resource_type = ResourceType.objects.get(name="istext" )
# parent = Node.objects.get(id=project_id)
# node_type = NodeType.objects.get(name='Corpus')
# type_id = NodeType.objects.get(name='Document').id
# user_id = User.objects.get( username=request.user ).id
# corpus = Node(
# user=request.user,
# parent=parent,
# type=node_type,
# name=query,
# )
# corpus.save()
# # configuring your queue with the event
# for i in range(8):
# t = threading.Thread(target=tasks.worker2) #thing to do
# t.daemon = True # thread dies when main thread (only non-daemon thread) exits.
# t.start()
# for url in urlreqs:
# filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.now().microsecond))
# tasks.q.put( [url , filename]) #put a task in th queue
# tasks.q.join() # wait until everything is finished
# for filename in tasks.firstResults:
# corpus.add_resource( user=request.user, type=resource_type, file=filename )
# corpus.save()
# print("DEBUG:",DEBUG)
# # do the WorkFlow
# try:
# if DEBUG is True:
# corpus.workflow()
# else:
# corpus.workflow.apply_async((), countdown=3)
# return JsonHttpResponse(["workflow","finished"])
# except Exception as error:
# print(error)
data = [query_string,query,N]
return JsonHttpResponse(data)
......@@ -119,11 +119,15 @@ function Final_UpdateTable( action ) {
var current_docs = {}
var BIS_dict = {}
var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\//);
var projectid = path[1]
var corpusid = path[2]
var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
var url_elems = window.location.href.split("/")
var url_mainIDs = {}
for(var i=0; i<url_elems.length; i++) {
// if the this element is a number:
if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
}
}
var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
// $.ajax({
// url: theurl,
// success: function(data) {
......@@ -204,6 +208,38 @@ function transformContent(rec_id , header , content) {
} else return content;
}
$("#move2trash")
.click(function(){
var ids2trash = []
for(var i in Garbage) {
ids2trash.push(AjaxRecords[i].id);
}
console.log("ids to the trash:")
console.log(ids2trash)
$.ajax({
url: "/tests/move2trash/",
data: "nodeids="+JSON.stringify(ids2trash),
type: 'POST',
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data) {
console.log("in #move2trash")
console.log(data)
location.reload();
},
error: function(result) {
console.log("Data not found in #move2trash");
console.log(result)
}
});
})
.hide();
//generic enough
function ulWriter(rowIndex, record, columns, cellWriter) {
// pr("\tulWriter: "+record.id)
......@@ -224,9 +260,30 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
return '<tr>' + tr + '</tr>';
}
// var div__filter_for_search = ''
// div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple data-max-options="1">';
// div__filter_for_search += ' <optgroup label="All" data-max-options="1" selected>';
// div__filter_for_search += ' <option>Title</option>';
// div__filter_for_search += ' <option>Date</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Category" data-max-options="1">';
// div__filter_for_search += ' <option>Title</option>';
// div__filter_for_search += ' <option>Date</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += ' <optgroup label="Duplicates" data-max-options="1">';
// div__filter_for_search += ' <option>by DOI</option>';
// div__filter_for_search += ' <option>by Title</option>';
// div__filter_for_search += ' </optgroup>';
// div__filter_for_search += '</select>';
// $("#supmofos").html(div__filter_for_search)
// (3) Get records and hyperdata for paginator
$.ajax({
url: '/tests/paginator/corpus/'+corpusid,
url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
success: function(data){
console.log(data)
......@@ -235,7 +292,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
var orig_id = parseInt(data.records[i].id)
var arr_id = parseInt(i)
RecDict[orig_id] = arr_id;
data.records[i]["name"] = '<a target="_blank" href="/project/'+projectid+'/corpus/'+ corpusid + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
data.records[i]["del"] = false
var date = data.records[i]["date"];
......@@ -247,36 +304,6 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
// $("#move2trash").prop('disabled', true);
$("#move2trash")
.click(function(){
var ids2trash = []
for(var i in Garbage) {
ids2trash.push(AjaxRecords[i].id);
}
console.log("ids to the trash:")
console.log(ids2trash)
$.ajax({
url: "/tests/move2trash/",
data: "nodeids="+JSON.stringify(ids2trash),
type: 'POST',
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data) {
console.log("in #move2trash")
console.log(data)
location.reload();
},
error: function(result) {
console.log("Data not found in #move2trash");
console.log(result)
}
});
})
.hide();
var t0 = AjaxRecords[0].date.split("-").map(Number)
......@@ -429,6 +456,13 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
$('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
$(".dynatable-record-count").insertAfter(".imadiv")
$(".dynatable-pagination-links").insertAfter(".imadiv")
// console.log(RecDict)
var the_content = $("#supmofos").html();
$(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
$("#supmofos").remove()
// .insertAfter("#dynatable-query-search-my-ajax-table")
}
});
This diff is collapsed.
......@@ -159,8 +159,16 @@ function Final_UpdateTable( action ) {
// Get all the duplicates using the Django-Garg API
var current_docs = {}
var BIS_dict = {}
var corpusid = window.location.href.split("corpus")[1].replace(/\//g, '')//replace all the slashes
var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
var url_elems = window.location.href.split("/")
var url_mainIDs = {}
for(var i=0; i<url_elems.length; i++) {
// if the this element is a number:
if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
}
}
var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
// $.ajax({
// url: theurl,
// success: function(data) {
......@@ -364,14 +372,46 @@ $("#Clean_All").click(function(){
$("#Save_All").click(function(){
var sum__selected_elems = 0;
var poubelle = []
for(var i in FlagsBuffer)
if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
for(var i in poubelle)
delete FlagsBuffer[poubelle[i]];
if ( sum__selected_elems>0 ) {
console.log("")
console.log("Do the ajax conexion with API and send this array to be processed:")
for(var i in FlagsBuffer) {
var real_ids = []
for (var j in FlagsBuffer[i])
real_ids.push( AjaxRecords[j].id );
FlagsBuffer[i] = real_ids
}
console.log(FlagsBuffer)
console.log("")
var list_id = $("#list_id").val()
// '/annotations/lists/'+list_id+'/ngrams/108642'
console.log(window.location.origin+'/annotations/lists/'+list_id+"/multiple")
console.log(real_ids)
$.ajax({
method: "POST",
url: window.location.origin+'/annotations/lists/'+list_id+"/multiple",
data: "to_delete="+JSON.stringify(real_ids),
beforeSend: function(xhr) {
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data){
console.log(data)
},
error: function(result) {
console.log("Data not found in #Save_All");
console.log(result)
}
});
// console.log("")
}
});
......
......@@ -5,6 +5,8 @@
{% load staticfiles %}
<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
......@@ -132,7 +134,6 @@ th a {
</div>
</div>
<div class="container">
<div class="row">
<div class="col-md-4">
......@@ -178,12 +179,29 @@ th a {
</div>
</div>
<div id="supmofos">
<select data-width="100px" dir="ltr" class="selectpicker">
<option selected>All</option>
<option>Title</option>
<option>Date</option>
<optgroup label="Duplicates">
<option>by DOI</option>
<option>by Title</option>
</optgroup>
</select>
</div>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js -->
<script type="text/javascript" src="{% static "js/dyna_chart_and_table.js" %}"></script>
<script type="text/javascript" src="{% static "js/Docs_dyna_chart_and_table.js" %}"></script>
......
......@@ -264,7 +264,7 @@
xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
},
success: function(data) {
console.log("in doTheQuery()")
console.log("in doTheQuery() Ajax.Success:")
console.log(data)
location.reload();
},
......@@ -487,7 +487,6 @@
var pubmedifiedQuery = { query : query , string: query }
// console.log(pubmedifiedQuery)
var projectid = window.location.href.split("project")[1].replace(/\//g, '')//replace all the slashes
......
{% extends "menu.html" %}
{% block css %}
{% load staticfiles %}
<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
<link rel="stylesheet" type="text/css" href="{% static "css/dc.css"%}"/>
<link rel="stylesheet" type="text/css" href="{% static "css/jquery.dynatable.css"%}"/>
<script type="text/javascript" src="{% static "js/charts/d3.js"%}"></script>
<script type="text/javascript" src="{% static "js/charts/crossfilter.js"%}"></script>
<!-- <script type="text/javascript" src="http://localhost/i7/js_examples/DC_linear/dc.js"></script> -->
<script type="text/javascript" src="{% static "js/charts/dc.js"%}"></script>
<style>
.no-transition {
-webkit-transition: height 0.1s;
-moz-transition: height 0.1s;
-ms-transition: height 0.1s;
-o-transition: height 0.1s;
transition: height 0.1s;
}
th { color: #fff; }
th a {
color: #fff;
font-weight: normal;
font-style: italic;
font-size: 0.9em;
}
/*tr:hover {
cursor: pointer;
color: #000;
font-weight: bold;
}*/
.to_delete {
color:red;
opacity: 0.8;
}
.to_group {
color:blue;
}
.to_keep {
color:green;
}
.dynatable-record-count {
font-size: 0.7em;
}
.dynatable-pagination-links {
font-size: 0.7em;
}
input[type=radio] {
display:none;
}
input[type=radio] + label {
display:inline-block;
margin:-2px;
padding: 4px 12px;
margin-bottom: 0;
font-size: 14px;
line-height: 20px;
color: #333;
text-align: center;
text-shadow: 0 1px 1px rgba(255,255,255,0.75);
vertical-align: middle;
cursor: pointer;
background-color: #f5f5f5;
background-image: -moz-linear-gradient(top,#fff,#e6e6e6);
background-image: -webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));
background-image: -webkit-linear-gradient(top,#fff,#e6e6e6);
background-image: -o-linear-gradient(top,#fff,#e6e6e6);
background-image: linear-gradient(to bottom,#fff,#e6e6e6);
background-repeat: repeat-x;
border: 1px solid #ccc;
border-color: #e6e6e6 #e6e6e6 #bfbfbf;
border-color: rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);
border-bottom-color: #b3b3b3;
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);
filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
-moz-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
}
input[type=radio]:checked + label {
background-image: none;
outline: 0;
-webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
-moz-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
background-color:#e0e0e0;
}
</style>
{% endblock %}
{% block content %}
<div class="container theme-showcase" role="main">
<div class="jumbotron">
{% if project %}
<h1>{{ project.name }}, {{ corpus.name }}
</h1>
{% endif %}
{% if corpus %}
<p>
Created on {{ corpus.date }}
</p>
<div id="stats"></div>
{% endif %}
<!-- <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
<a class="btn btn-primary btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save as</a>
<a class="btn btn-primary btn-lg" role="button" href="/delete/{{ corpus.id }}">Delete</a></p>
{% if number == 0 %}
<a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add documents</a></p>
{% endif %}
</div>
</div>
<div class="container">
<div class="container">
<div class="row">
<center>
<div id="ScoresBox"></div> <br>(Blue bars: all, Green line: zoom)
<a class="reset" href="javascript:volumeChart.filterAll();dc.redrawAll();"
style="display: none;">reset</a>
<div class="clearfix"></div>
</center>
<div id="monthly-move-chart"></div>
</div>
<div class="row">
<div id="monthly-volume-chart"></div>
<p class="muted pull-left" style="margin-right: 15px;">Select a time range to zoom in</p>
</div>
</div>
<p align="center">
<a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a></p>
<div class="container">
<div class="jumbotron">
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
<div class="panel panel-body">
<div class="panel-heading">
<h4 class="panel-title">
<a data-toggle="collapse" data-parent="#accordion" href="#collapseOne">
<p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg" style="width:200px; margin:0 auto; display:block;">Open Folder</h2></p>
</a>
</h4>
</div>
<div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
<div class="panel-body">
<div id="div-table"></div>
<!-- <p align="right">
<table id="my-ajax-table" class="table table-bordered">
<thead>
<th data-dynatable-column="name">Title</th>
<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>
<th data-dynatable-column="del" data-dynatable-no-sort="true">
<input type="checkbox" id="multiple_selection" onclick="SelectAll(this);" /> Select
</th>
</thead>
<tbody>
</tbody>
</table>
</p> -->
</div>
</div>
</div>
</div>
</div>
</div>
<div class="container">
<div class="row">
<div class="col-md-4">
<div class="jumbotron">
<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
<ol>
<li>Count</li> <!-- read, compute -->
<li>Filter</li> <!-- count, compute -->
<li>Compare</li> <!-- select, cut -->
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
<div class="col-md-4">
<div class="jumbotron">
<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3>
<ol>
<li>Sort</li>
<li>Group</li>
<li>Cluster</li>
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
<div class="col-md-4">
<div class="jumbotron">
{% if processing > 0 %}
<h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
{% else %}
<h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
{% endif %}
<ol>
<li>Visualize</li>
<li>Explore</li>
<li>Read</li>
</ol>
<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
</div>
</div>
</div>
</div>
<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js -->
<script type="text/javascript" src="{% static "js/Journals_dyna_chart_and_table.js" %}"></script>
{% endblock %}
......@@ -281,7 +281,7 @@ input[type=radio]:checked + label {
<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
<!-- custom-lib for dynatable.js and dc.js -->
<script type="text/javascript" src="{% static "js/test_dyna_chart_and_table.js" %}"></script>
<script type="text/javascript" src="{% static "js/NGrams_dyna_chart_and_table.js" %}"></script>
......
......@@ -50,6 +50,46 @@ from gargantext_web import about
from gargantext_web.api import JsonHttpResponse
from ngram.lists import listIds, listNgramIds, ngramList , doList
def test_page(request , project_id , corpus_id):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = int(project_id)
offset = int(corpus_id)
except ValueError:
raise Http404()
t = get_template('tests/test_select-boostrap.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
project = cache.Node[int(project_id)]
corpus = cache.Node[int(corpus_id)]
type_doc_id = cache.NodeType['Document'].id
number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
try:
processing = corpus.hyperdata['Processing']
except Exception as error:
print(error)
processing = 0
html = t.render(Context({
'debug': settings.DEBUG,
'user': user,
'date': date,
'project': project,
'corpus' : corpus,
'processing' : processing,
'number' : number,
}))
return HttpResponse(html)
def get_ngrams(request , project_id , corpus_id ):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
......@@ -68,6 +108,80 @@ def get_ngrams(request , project_id , corpus_id ):
corpus = cache.Node[int(corpus_id)]
type_doc_id = cache.NodeType['Document'].id
number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
lists = dict()
for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
try:
processing = corpus.hyperdata['Processing']
except Exception as error:
print(error)
processing = 0
html = t.render(Context({
'debug': settings.DEBUG,
'user': user,
'date': date,
'project': project,
'corpus' : corpus,
'processing' : processing,
'number' : number,
'list_id': list_id[0][0],
}))
return HttpResponse(html)
def test_test(request , corpus_id , doc_id):
"""Get All for a doc id"""
corpus_id = int(corpus_id)
doc_id = int(doc_id)
lists = dict()
for list_type in ['StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
print(list_id[0][0])
# # # ngrams of list_id of corpus_id:
# commeca = "StopList"
doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
to_del = {}
for n in doc_ngram_list:
to_del[ n[0] ] = True
print( to_del.keys() )
results = [ "hola" , "mundo" ]
return JsonHttpResponse(results)
def get_journals(request , project_id , corpus_id ):
if not request.user.is_authenticated():
return redirect('/login/?next=%s' % request.path)
try:
offset = int(project_id)
offset = int(corpus_id)
except ValueError:
raise Http404()
t = get_template('tests/journals.html')
user = cache.User[request.user.username].id
date = datetime.datetime.now()
project = cache.Node[int(project_id)]
corpus = cache.Node[int(corpus_id)]
type_doc_id = cache.NodeType['Document'].id
number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
try:
processing = corpus.hyperdata['Processing']
except Exception as error:
......@@ -86,6 +200,21 @@ def get_ngrams(request , project_id , corpus_id ):
return HttpResponse(html)
def test_journals(request , project_id, corpus_id ):
results = ["hola" , "mundo"]
JournalsDict = {}
user_id = request.user.id
document_type_id = cache.NodeType['Document'].id
documents = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
for doc in documents:
if "journal" in doc.hyperdata:
journal = doc.hyperdata["journal"]
if journal not in JournalsDict:
JournalsDict [journal] = 0
JournalsDict[journal] += 1
return JsonHttpResponse(JournalsDict)
def test_ngrams(request , project_id, corpus_id ):
results = ["hola" , "mundo"]
......@@ -94,6 +223,17 @@ def test_ngrams(request , project_id, corpus_id ):
whitelist_type_id = cache.NodeType['WhiteList'].id
document_type_id = cache.NodeType['Document'].id
corpus_id = int(corpus_id)
lists = dict()
for list_type in ['StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
StopList = {}
for n in doc_ngram_list:
StopList[ n[0] ] = True
# # 13099 clinical benefits
# # 7492 recent data
# # 14279 brain development
......@@ -145,15 +285,16 @@ def test_ngrams(request , project_id, corpus_id ):
for doc in documents:
NgramOccs = session.query(Node_Ngram).filter( Node_Ngram.node_id==doc.id).all()
for ngram in NgramOccs:
if ngram.ngram_id not in Ngrams_Scores:
Ngrams_Scores[ngram.ngram_id] = {}
Ngrams_Scores[ngram.ngram_id]["scores"] = {
"occ_sum": 0.0,
"occ_uniq": 0.0,
"tfidf_sum": 0.0
}
Ngrams_Scores[ngram.ngram_id]["scores"]["occ_sum"]+=ngram.weight
Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]+=1
if ngram.ngram_id not in StopList:
if ngram.ngram_id not in Ngrams_Scores:
Ngrams_Scores[ngram.ngram_id] = {}
Ngrams_Scores[ngram.ngram_id]["scores"] = {
"occ_sum": 0.0,
"occ_uniq": 0.0,
"tfidf_sum": 0.0
}
Ngrams_Scores[ngram.ngram_id]["scores"]["occ_sum"]+=ngram.weight
Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]+=1
# print("\t" , ngram.ngram_id , "\t" , ngram.weight )
## Getting the Effective nro de OCCS / >##
......@@ -177,9 +318,10 @@ def test_ngrams(request , project_id, corpus_id ):
Sum = 0
NgramTFIDF = session.query(NodeNodeNgram).filter( NodeNodeNgram.nodex_id==corpus_id ).all()
for ngram in NgramTFIDF:
Ngrams_Scores[ngram.ngram_id]["scores"]["tfidf_sum"] += ngram.score
Sum += Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]
# print( "docid:", ngram.nodey_id , ngram.ngram_id , ngram.score)
if ngram.ngram_id not in StopList:
Ngrams_Scores[ngram.ngram_id]["scores"]["tfidf_sum"] += ngram.score
Sum += Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]
# print( "docid:", ngram.nodey_id , ngram.ngram_id , ngram.score)
# import pprint
......@@ -210,13 +352,14 @@ def test_ngrams(request , project_id, corpus_id ):
query = session.query(Ngram).filter(Ngram.id.in_( ngrams_ids ))
ngrams_data = query.all()
for ngram in ngrams_data:
occ_uniq = Ngrams_Scores[ngram.id]["scores"]["occ_uniq"]
if occ_uniq > occs_threshold:
Ngrams_Scores[ngram.id]["name"] = ngram.terms
Ngrams_Scores[ngram.id]["id"] = ngram.id
Ngrams_Scores[ngram.id]["scores"]["tfidf"] = Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"] / occ_uniq
del Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"]
Metrics["ngrams"].append( Ngrams_Scores[ngram.id] )
if ngram.id not in StopList:
occ_uniq = Ngrams_Scores[ngram.id]["scores"]["occ_uniq"]
if occ_uniq > occs_threshold:
Ngrams_Scores[ngram.id]["name"] = ngram.terms
Ngrams_Scores[ngram.id]["id"] = ngram.id
Ngrams_Scores[ngram.id]["scores"]["tfidf"] = Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"] / occ_uniq
del Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"]
Metrics["ngrams"].append( Ngrams_Scores[ngram.id] )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment