Commit 32754ad0 authored by delanoe's avatar delanoe

[FEAT] merge and Agent Based simulation presentation at AFS.

parents e9ce6b85 344f989f
This diff is collapsed.
......@@ -9,7 +9,8 @@
"angular-loader": "~1.2.x",
"angular-resource": "~1.2.x",
"bootstrap": "~3.x",
"angular-cookies": "1.2"
"angular-cookies": "1.2",
"bootstrap-select": "silviomoreto/bootstrap-select#~1.7.3"
},
"resolutions": {
"angular": "~1.2.x"
......
(function () {
'use strict';
var annotationsAppActiveLists = angular.module('annotationsAppActiveLists', []);
annotationsAppActiveLists.controller('ActiveListsController',
['$scope', '$rootScope', '$timeout',
function ($scope, $rootScope, $timeout) {
$scope.activeListsChange = function() {
var selected = $('.selectpicker option:selected').val();
var newActive = {};
$('.selectpicker option:selected').each(function(item, value) {
var id = value.id.split("---", 2)[1];
newActive[id] = value.value;
});
$rootScope.activeLists = newActive;
};
$rootScope.$watchCollection('activeLists', function (newValue, oldValue) {
if (newValue === undefined) return;
$timeout(function() {
$('.selectpicker').selectpicker('refresh');
});
});
$rootScope.$watchCollection('lists', function (newValue, oldValue) {
if (newValue === undefined) return;
// reformat lists to allListsSelect
var allListsSelect = [];
angular.forEach($rootScope.lists, function(value, key) {
this.push({
'id': key,
'label': value
});
// initialize activeLists with the MiamList by default
if (value == 'MiamList') {
$rootScope.activeLists = {};
$rootScope.activeLists[key] = value;
}
}, allListsSelect);
$rootScope.allListsSelect = allListsSelect;
$timeout(function() {
$('.selectpicker').selectpicker();
$('.selectpicker').selectpicker('val', ['MiamList']);
});
});
}]);
})(window);
/* app css stylesheet */
/*
* Class names corresponding to server-side list names
* To display another list name, add a new class under this
*/
.MiamList {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.StopList {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
}
.delete-keyword, .occurrences {
vertical-align: super;
font-size: 70%;
......@@ -27,47 +45,32 @@
border-bottom: none;
}
.miamword {
color: black;
background-color: rgba(60, 118, 61, 0.5);
cursor: pointer;
}
.stopword {
color: black;
background-color: rgba(169, 68, 66, 0.2);
cursor: pointer;
.main-panel, .text-panel, .words-panel {
margin: 10px 0;
}
.global-stopword {
color: black;
background-color: rgba(169, 68, 66, 0.05);
cursor: pointer;
#annotationsApp {
min-width: 780px;
}
.main-panel, .text-panel, .words-panel {
height: 800px;
margin: 10px 0px;
.words-panel {
min-width: 220px;
}
.text-panel {
overflow-y: auto;
min-width: 400px;
}
.words-list {
margin-bottom: 5px;
height: 250px;
}
.keyword-container {
/*display: inline-block;*/
}
.keyword {
.keyword-text {
word-break: break-all;
}
.list-group-item {
.keyword-group-item {
display: inline-block;
float: left;
padding: 5px;
......@@ -145,3 +148,7 @@
border-right: solid thin #CCC;
margin-right: 5px;
}
.float-right {
float: right;
}
This diff is collapsed.
(function () {
'use strict';
var annotationsAppDocument = angular.module('annotationsAppDocument', ['annotationsAppHttp']);
annotationsAppDocument.controller('DocController',
['$scope', '$rootScope', '$timeout', 'NgramListHttpService', 'DocumentHttpService',
function ($scope, $rootScope, $timeout, NgramListHttpService, DocumentHttpService) {
$rootScope.documentResource = DocumentHttpService.get(
{'docId': $rootScope.docId},
function(data, responseHeaders) {
$scope.authors = data.authors;
$scope.journal = data.journal;
$scope.publication_date = data.publication_date;
//$scope.current_page_number = data.current_page_number;
//$scope.last_page_number = data.last_page_number;
$rootScope.title = data.title;
$rootScope.docId = data.id;
$rootScope.full_text = data.full_text;
$rootScope.abstract_text = data.abstract_text;
// GET the annotationss
NgramListHttpService.get(
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.lists = data[$rootScope.corpusId.toString()].lists;
},
function(data) {
console.error("unable to get the list of ngrams");
}
);
});
// TODO setup article pagination
$scope.onPreviousClick = function () {
DocumentHttpService.get($scope.docId - 1);
};
$scope.onNextClick = function () {
DocumentHttpService.get($scope.docId + 1);
};
}]);
annotationsAppDocument.controller('DocFavoriteController',
['$scope', '$rootScope', 'DocumentHttpService',
function ($scope, $rootScope, DocumentHttpService) {
$scope.onStarClick = function($event) {
console.log("TODO");
};
$scope.isFavorite = false;
}]);
})(window);
This diff is collapsed.
......@@ -57,11 +57,11 @@
{
post: {
method: 'POST',
params: {'listId': '@listId', 'ngramId': ''}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
},
delete: {
method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@id'}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
}
}
);
......
<span ng-if="keyword.category == 'miamlist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="miamlist">×</span>
<span ng-if="keyword.category == 'miamlist'" data-toggle="tooltip" class="keyword miamword">{[{keyword.text}]}</span>
<span ng-if="keyword.category == 'stoplist'" ng-click='onDeleteClick()' class="delete-keyword" data-keyword-id="{[{keyword.uuid}]}" data-keyword-text="{[{keyword.text}]}" data-keyword-category="stoplist">×</span>
<span ng-if="keyword.category == 'stoplist'" data-toggle="tooltip" class="keyword stopword">{[{keyword.text}]}</span>
<span ng-click='onDeleteClick()' class="delete-keyword">×</span>
<span data-toggle="tooltip" class="keyword-text {[{keyword.listName}]}">{[{keyword.text}]}</span>
<span class="occurrences" data-keyword-id="{[{keyword.uuid}]}">{[{keyword.occurrences}]}</span>
......@@ -19,6 +19,7 @@ var S = window.STATIC_URL;
$script([
S + 'bower_components/angular/angular.min.js',
S + 'bower_components/bootstrap/dist/js/bootstrap.min.js',
S + 'bower_components/bootstrap-select/dist/js/bootstrap-select.min.js',
S + 'bower_components/angular-loader/angular-loader.min.js',
S + 'bower_components/underscore/underscore-1.5.2.js',
//'bower_components/angular-route/angular-route.js',
......@@ -26,7 +27,10 @@ $script([
$script([
S + 'bower_components/angular-cookies/angular-cookies.min.js',
S + 'bower_components/angular-resource/angular-resource.min.js'], function() {
$script([S + 'annotations/http.js', S + 'annotations/app.js'], function() {
$script([S + 'annotations/http.js', S + 'annotations/highlight.js',
S + 'annotations/document.js', S + 'annotations/ngramlist.js',
S + 'annotations/activelists.js', S + 'annotations/ngramlist.js',
S + 'annotations/utils.js', S + 'annotations/app.js'], function() {
// when all is done, execute bootstrap angular application (replace ng-app directive)
angular.bootstrap(document.getElementById("annotationsApp"), ['annotationsApp']);
});
......
(function () {
'use strict';
var annotationsAppNgramList = angular.module('annotationsAppNgramList', ['annotationsAppHttp']);
/*
* Controls one Ngram displayed in the flat lists (called "extra-text")
*/
annotationsAppNgramList.controller('NgramController',
['$scope', '$rootScope', 'NgramHttpService', 'NgramListHttpService',
function ($scope, $rootScope, NgramHttpService, NgramListHttpService) {
/*
* Click on the 'delete' cross button
*/
$scope.onDeleteClick = function () {
NgramHttpService.delete({
'listId': $scope.keyword.list_id,
'ngramId': $scope.keyword.uuid
}, function(data) {
// Refresh the annotationss
NgramListHttpService.get(
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.refreshDisplay();
},
function(data) {
console.error("unable to refresh the list of ngrams");
}
);
}, function(data) {
console.error("unable to remove the Ngram " + $scope.keyword.text);
});
};
}]);
/*
* Controller for the list panel displaying extra-text ngram
*/
annotationsAppNgramList.controller('NgramListPaginationController',
['$scope', '$rootScope', function ($scope, $rootScope) {
$rootScope.$watchCollection('extraNgramList', function (newValue, oldValue) {
$scope.currentListPage = 0;
$scope.pageSize = 15;
$scope.nextListPage = function() {
$scope.currentListPage = $scope.currentListPage + 1;
};
$scope.previousListPage = function() {
$scope.currentListPage = $scope.currentListPage - 1;
};
$scope.totalListPages = function(listId) {
if ($rootScope.extraNgramList[listId] === undefined) return 0;
return Math.ceil($rootScope.extraNgramList[listId].length / $scope.pageSize);
};
});
}]);
/*
* Template of the ngram element displayed in the flat lists
*/
annotationsAppNgramList.directive('keywordTemplate', function () {
return {
templateUrl: function ($element, $attributes) {
return S + 'annotations/keyword_tpl.html';
}
};
});
/*
* new NGram from the user input
*/
annotationsAppNgramList.controller('NgramInputController',
['$scope', '$rootScope', '$element', 'NgramHttpService', 'NgramListHttpService',
function ($scope, $rootScope, $element, NgramHttpService, NgramListHttpService) {
/*
* Add a new NGram from the user input in the extra-text list
*/
$scope.onListSubmit = function ($event, listId) {
var inputEltId = "#"+ listId +"-input";
if ($event.keyCode !== undefined && $event.keyCode != 13) return;
var value = angular.element(inputEltId).val().trim();
if (value === "") return;
NgramHttpService.post(
{
'listId': listId,
'ngramId': 'create'
},
{
'text': value
},
function(data) {
// on success
if (data) {
angular.element(inputEltId).val("");
// Refresh the annotationss
NgramListHttpService.get(
{
'corpusId': $rootScope.corpusId,
'docId': $rootScope.docId
},
function(data) {
$rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
$rootScope.refreshDisplay();
},
function(data) {
console.error("unable to get the list of ngrams");
}
);
}
}, function(data) {
// on error
angular.element(inputEltId).parent().addClass("has-error");
console.error("error adding Ngram "+ value);
}
);
};
}]);
})(window);
<ul class="noselection">
<li class="miamword" ng-if="local_miamlist === true" ng-click="onClick($event, 'post', miamListId, 'local')">add to miam-list</li>
<li class="miamword" ng-if="local_miamlist === false" ng-click="onClick($event, 'delete', miamListId, 'local')">remove from miam-list</li>
<li class="stopword" ng-if="local_stoplist === true" ng-click="onClick($event, 'post', stopListId, 'local')">add to stop-list</li>
<li class="stopword" ng-if="local_stoplist === false" ng-click="onClick($event, 'delete', stopListId, 'local')">remove from stop-list</li>
<!--<li class="stopword" ng-if="global_stoplist === true" ng-click="onClick($event, 'post', 'stoplist', 'global')">add to global stop-list</li>
<li class="stopword" ng-if="global_stoplist === false" ng-click="onClick($event, 'delete', 'stoplist', 'global')">remove from global stop-list</li>-->
</ul>
(function () {
'use strict';
var annotationsAppUtils = angular.module('annotationsAppUtils', []);
/*
* Filter used in lists pagination (extra-text panel)
*/
annotationsAppUtils.filter('startFrom', function () {
return function (input, start) {
if (input === undefined) return;
start = +start; //parse to int
return input.slice(start);
};
});
})(window);
/*!
* Bootstrap-select v1.7.3 (http://silviomoreto.github.io/bootstrap-select)
*
* Copyright 2013-2015 bootstrap-select
* Licensed under MIT (https://github.com/silviomoreto/bootstrap-select/blob/master/LICENSE)
*/.bootstrap-select{width:220px \0}.bootstrap-select>.dropdown-toggle{width:100%;padding-right:25px}.error .bootstrap-select .dropdown-toggle,.has-error .bootstrap-select .dropdown-toggle{border-color:#b94a48}.bootstrap-select.fit-width{width:auto!important}.bootstrap-select:not([class*=col-]):not([class*=form-control]):not(.input-group-btn){width:220px}.bootstrap-select .dropdown-toggle:focus{outline:thin dotted #333!important;outline:5px auto -webkit-focus-ring-color!important;outline-offset:-2px}.bootstrap-select.form-control{margin-bottom:0;padding:0;border:none}.bootstrap-select.form-control:not([class*=col-]){width:100%}.bootstrap-select.form-control.input-group-btn{z-index:auto}.bootstrap-select.btn-group:not(.input-group-btn),.bootstrap-select.btn-group[class*=col-]{float:none;display:inline-block;margin-left:0}.bootstrap-select.btn-group.dropdown-menu-right,.bootstrap-select.btn-group[class*=col-].dropdown-menu-right,.row .bootstrap-select.btn-group[class*=col-].dropdown-menu-right{float:right}.form-group .bootstrap-select.btn-group,.form-horizontal .bootstrap-select.btn-group,.form-inline .bootstrap-select.btn-group{margin-bottom:0}.form-group-lg .bootstrap-select.btn-group.form-control,.form-group-sm .bootstrap-select.btn-group.form-control{padding:0}.form-inline .bootstrap-select.btn-group .form-control{width:100%}.bootstrap-select.btn-group.disabled,.bootstrap-select.btn-group>.disabled{cursor:not-allowed}.bootstrap-select.btn-group.disabled:focus,.bootstrap-select.btn-group>.disabled:focus{outline:0!important}.bootstrap-select.btn-group .dropdown-toggle .filter-option{display:inline-block;overflow:hidden;width:100%;text-align:left}.bootstrap-select.btn-group .dropdown-toggle .caret{position:absolute;top:50%;right:12px;margin-top:-2px;vertical-align:middle}.bootstrap-select.btn-group[class*=col-] .dropdown-toggle{width:100%}.bootstrap-select.btn-group .dropdown-menu{min-width:100%;z-index:1035;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.bootstrap-select.btn-group .dropdown-menu.inner{position:static;float:none;border:0;padding:0;margin:0;border-radius:0;-webkit-box-shadow:none;box-shadow:none}.bootstrap-select.btn-group .dropdown-menu li{position:relative}.bootstrap-select.btn-group .dropdown-menu li.active small{color:#fff}.bootstrap-select.btn-group .dropdown-menu li.disabled a{cursor:not-allowed}.bootstrap-select.btn-group .dropdown-menu li a{cursor:pointer}.bootstrap-select.btn-group .dropdown-menu li a.opt{position:relative;padding-left:2.25em}.bootstrap-select.btn-group .dropdown-menu li a span.check-mark{display:none}.bootstrap-select.btn-group .dropdown-menu li a span.text{display:inline-block}.bootstrap-select.btn-group .dropdown-menu li small{padding-left:.5em}.bootstrap-select.btn-group .dropdown-menu .notify{position:absolute;bottom:5px;width:96%;margin:0 2%;min-height:26px;padding:3px 5px;background:#f5f5f5;border:1px solid #e3e3e3;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05);pointer-events:none;opacity:.9;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.bootstrap-select.btn-group .no-results{padding:3px;background:#f5f5f5;margin:0 5px;white-space:nowrap}.bootstrap-select.btn-group.fit-width .dropdown-toggle .filter-option{position:static}.bootstrap-select.btn-group.fit-width .dropdown-toggle .caret{position:static;top:auto;margin-top:-1px}.bootstrap-select.btn-group.show-tick .dropdown-menu li.selected a span.check-mark{position:absolute;display:inline-block;right:15px;margin-top:5px}.bootstrap-select.btn-group.show-tick .dropdown-menu li a span.text{margin-right:34px}.bootstrap-select.show-menu-arrow.open>.dropdown-toggle{z-index:1036}.bootstrap-select.show-menu-arrow .dropdown-toggle:before{content:'';border-left:7px solid transparent;border-right:7px solid transparent;border-bottom:7px solid rgba(204,204,204,.2);position:absolute;bottom:-4px;left:9px;display:none}.bootstrap-select.show-menu-arrow .dropdown-toggle:after{content:'';border-left:6px solid transparent;border-right:6px solid transparent;border-bottom:6px solid #fff;position:absolute;bottom:-4px;left:10px;display:none}.bootstrap-select.show-menu-arrow.dropup .dropdown-toggle:before{bottom:auto;top:-3px;border-top:7px solid rgba(204,204,204,.2);border-bottom:0}.bootstrap-select.show-menu-arrow.dropup .dropdown-toggle:after{bottom:auto;top:-3px;border-top:6px solid #fff;border-bottom:0}.bootstrap-select.show-menu-arrow.pull-right .dropdown-toggle:before{right:12px;left:auto}.bootstrap-select.show-menu-arrow.pull-right .dropdown-toggle:after{right:13px;left:auto}.bootstrap-select.show-menu-arrow.open>.dropdown-toggle:after,.bootstrap-select.show-menu-arrow.open>.dropdown-toggle:before{display:block}.bs-actionsbox,.bs-donebutton,.bs-searchbox{padding:4px 8px}.bs-actionsbox{float:left;width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.bs-actionsbox .btn-group button{width:50%}.bs-donebutton{float:left;width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.bs-donebutton .btn-group button{width:100%}.bs-searchbox+.bs-actionsbox{padding:0 8px 4px}.bs-searchbox .form-control{margin-bottom:0;width:100%}select.bs-select-hidden,select.selectpicker{display:none!important}select.mobile-device{position:absolute!important;top:0;left:0;display:block!important;width:100%;height:100%!important;opacity:0}
\ No newline at end of file
This diff is collapsed.
......@@ -5,6 +5,7 @@ from annotations import views
urlpatterns = patterns('',
url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view
url(r'^corpus/(?P<corpus_id>[0-9]+)/document/(?P<doc_id>[0-9]+)$', views.NgramList.as_view()), # the list associated with an ngram
url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.NgramEdit.as_view()), #
url(r'^lists/(?P<list_id>[0-9]+)/multiple?$', views.deleteMultiple ), #
url(r'^lists/(?P<list_id>[0-9]+)/ngrams/(?P<ngram_id>[0-9]+)$', views.NgramEdit.as_view()), #
url(r'^lists/(?P<list_id>[0-9]+)/ngrams/create$', views.NgramCreate.as_view()), #
url(r'^lists/(?P<list_id>[0-9]+)/multiple?$', views.deleteMultiple ), # FIXME What's this ?
)
......@@ -17,7 +17,6 @@ from node.models import Node
from gargantext_web.db import *
from ngram.lists import listIds, listNgramIds, ngramList
from gargantext_web.api import JsonHttpResponse
import json
@login_required
......@@ -40,16 +39,13 @@ class NgramList(APIView):
"""Get All for a doc id"""
corpus_id = int(corpus_id)
doc_id = int(doc_id)
lists = dict()
lists = {}
for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
lists["%s" % list_id[0][0]] = list_type
# ngrams of list_id of corpus_id:
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=doc_id, user_id=request.user.id)
#doc_ngram_list = [(1, 'miam', 2, 1931), (2, 'stop', 2, 1932), (3, 'Potassium channels', 4, 1931)]
# ngrams for the corpus_id (ignoring doc_id for the moment):
doc_ngram_list = listNgramIds(corpus_id=corpus_id, doc_id=None, user_id=request.user.id)
data = { '%s' % corpus_id : {
'%s' % doc_id : [
{
......@@ -66,50 +62,83 @@ class NgramList(APIView):
class NgramEdit(APIView):
"""
Actions on one Ngram in one list
Actions on one existing Ngram in one list
"""
renderer_classes = (JSONRenderer,)
authentication_classes = (SessionAuthentication, BasicAuthentication)
def post(self, request, list_id, ngram_id):
"""
Add a ngram in a list
Edit an existing NGram in a given list
"""
list_id = int(list_id)
ngram_id = int(ngram_id)
# TODO remove the node_ngram from another conflicting list
# FIXME session.query(Node_Ngram).filter(Node_Ngram.ngram_id==ngram_id).delete()
# add the ngram to the list
node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
session.add(node_ngram)
session.commit()
# return the response
return Response({
'uuid': ngram_id,
'list_id': list_id,
})
def delete(self, request, list_id, ngram_id):
"""
Delete a ngram from a list
"""
session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).delete()
session.commit()
return Response(None, 204)
class NgramCreate(APIView):
"""
Create a new Ngram in one list
"""
renderer_classes = (JSONRenderer,)
authentication_classes = (SessionAuthentication, BasicAuthentication)
def post(self, request, list_id):
"""
create NGram in a given list
"""
# TODO - if Ngram is in miam-list, and adding it to stop-list,
# then remove it from the previous list
list_id = int(list_id)
# format the ngram's text
ngram_text = request.data.get('annotation', {}).get('text', None)
ngram_text = request.data.get('text', None)
if ngram_text is None:
raise APIException("Could not create a new Ngram without one \
text key in the json body")
ngram_text = ngram_text.strip().lower()
ngram_text = ' '.join(ngram_text.split())
# retrieve the ngram's id
# check if the ngram exists with the same terms
ngram = session.query(Ngram).filter(Ngram.terms == ngram_text).first()
if ngram is None:
ngram = Ngram(n=len(ngram_text.split()), terms=ngram_text)
session.add(ngram)
session.commit()
else:
# make sure the n value is correct
ngram.n = len(ngram_text.split())
session.add(ngram)
session.commit()
ngram_id = ngram.id
# add the ngram to the list if not already done
node_ngram = session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).first()
if node_ngram is None:
node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
session.add(node_ngram)
session.commit()
ngram_occurrences = node_ngram.weight
# create the new node_ngram relation
# TODO check existing Node_Ngram ?
node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
session.add(node_ngram)
session.commit()
# return the response
return Response({
'uuid': ngram_id,
'text': ngram_text,
'occurrences': ngram_occurrences,
'list_id': list_id,
})
def delete(self, request, list_id, ngram_id):
"""
Delete a ngram from a list
"""
session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).delete()
return Response(None, 204)
def deleteMultiple(request, list_id):
results = ["hola","mundo"]
......
This diff is collapsed.
......@@ -48,7 +48,6 @@ urlpatterns = patterns('',
# Corpus management
# Document view (main)
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
url(r'^project/(\d+)/corpus/(\d+)/documents/?$', views.corpus),
# Journals view
url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals),
......@@ -60,6 +59,7 @@ urlpatterns = patterns('',
# Update corpus
url(r'^project/(\d+)/corpus/(\d+)/(\w+)/update$', views.update_nodes),
url(r'^project/(\d+)/corpus/(\d+)/update$', views.update_nodes),
############################################################################
# annotations App
......
from rest_framework.exceptions import APIException
from datetime import datetime
__all__ = ['validate']
_types_names = {
bool: 'boolean',
int: 'integer',
float: 'float',
str: 'string',
dict: 'object',
list: 'array',
datetime: 'datetime',
}
class ValidationException(APIException):
status_code = 400
default_detail = 'Bad request!'
def validate(value, expected, path='input'):
# Is the expected type respected?
if 'type' in expected:
expected_type = expected['type']
if not isinstance(value, expected_type):
if expected_type in (bool, int, float, str, datetime, ):
try:
if expected_type == bool:
value = value not in {0, 0.0, '', '0', 'false'}
elif expected_type == datetime:
value = value + '2000-01-01T00:00:00Z'[len(value):]
value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
else:
value = expected_type(value)
except ValueError:
raise ValidationException('%s should be a JSON %s, but could not be parsed as such' % (path, _types_names[expected_type], ))
else:
raise ValidationException('%s should be a JSON %s' % (path, _types_names[expected_type], ))
else:
expected_type = type(value)
# Is the value in the expected range?
if 'range' in expected:
expected_range = expected['range']
if isinstance(expected_range, tuple):
if expected_type in (int, float):
tested_value = value
tested_name = 'value'
elif expected_type in (str, list):
tested_value = len(value)
tested_name = 'length'
if tested_value < expected_range[0]:
raise ValidationException('%s should have a minimum %s of %d' % (path, tested_name, expected_range[0], ))
if len(expected_range) > 1 and tested_value > expected_range[1]:
raise ValidationException('%s should have a maximum %s of %d' % (path, tested_name, expected_range[1], ))
elif isinstance(expected_range, (list, set, )) and value not in expected_range:
expected_values = expected_range if isinstance(expected_range, list) else expected_range
expected_values = [str(value) for value in expected_values if isinstance(value, expected_type)]
if len(expected_values) < 16:
expected_values_str = '", "'.join(expected_values)
expected_values_str = '"' + expected_values_str + '"'
else:
expected_values_str = '", "'.join(expected_values[:16])
expected_values_str = '"' + expected_values_str + '"...'
raise ValidationException('%s should take one of the following values: %s' % (path, expected_values_str, ))
# Do we have to translate through a dictionary?
if 'translate' in expected:
translate = expected['translate']
if callable(translate):
value = translate(value)
if value is None and expected.get('required', False):
raise ValidationException('%s has been given an invalid value' % (path, ))
return value
try:
value = expected['translate'][value]
except KeyError:
if expected.get('translate_fallback_keep', False):
return value
if expected.get('required', False):
raise ValidationException('%s has been given an invalid value' % (path, ))
else:
return expected.get('default', value)
# Are we handling an iterable?
if expected_type in (list, dict):
if 'items' in expected:
expected_items = expected['items']
if expected_type == list:
for i, element in enumerate(value):
value[i] = validate(element, expected_items, '%s[%d]' % (path, i, ))
elif expected_type == dict:
if expected_items:
for key in value:
if key not in expected_items:
raise ValidationException('%s should not have a "%s" key.' % (path, key, ))
for expected_key, expected_value in expected_items.items():
if expected_key in value:
value[expected_key] = validate(value[expected_key], expected_value, '%s["%s"]' % (path, expected_key, ))
elif 'required' in expected_value and expected_value['required']:
raise ValidationException('%s should have a "%s" key.' % (path, expected_key, ))
elif 'default' in expected_value:
value[expected_key] = expected_value['default']
# Let's return the proper value!
return value
......@@ -252,7 +252,7 @@ def projects(request):
})
def update_nodes(request, project_id, corpus_id, view):
def update_nodes(request, project_id, corpus_id, view=None):
'''
update function:
- remove previous computations (temporary lists and coocurrences)
......@@ -265,7 +265,8 @@ def update_nodes(request, project_id, corpus_id, view):
try:
offset = int(project_id)
offset = int(corpus_id)
offset = str(view)
if view is not None:
offset = str(view)
except ValueError:
raise Http404()
......@@ -307,7 +308,10 @@ def update_nodes(request, project_id, corpus_id, view):
#return redirect(request.path.replace('update', ''))
return redirect('/project/%s/corpus/%s/%s' % (project_id, corpus_id, view))
if view is None:
return redirect('/project/%s/corpus/%s/' % (project_id, corpus_id))
else:
return redirect('/project/%s/corpus/%s/%s' % (project_id, corpus_id, view))
#
# return render_to_response(
# request.path,
......@@ -385,7 +389,7 @@ def newpaginatorJSON(request , corpus_id):
for doc in documents:
if "publication_date" in doc.hyperdata:
try:
realdate = doc.hyperdata["publication_date"].split(" ")[0] # in database is = (year-month-day = 2015-01-06 00:00:00 = 06 jan 2015 00 hrs)
realdate = doc.hyperdata["publication_date"].replace('T',' ').split(" ")[0] # in database is = (year-month-day = 2015-01-06 00:00:00 = 06 jan 2015 00 hrs)
realdate = datetime.datetime.strptime(str(realdate), '%Y-%m-%d').date() # finalform = (yearmonthday = 20150106 = 06 jan 2015)
# doc.date = realdate
resdict = {}
......
......@@ -8,7 +8,7 @@ from admin.utils import PrintException
##from node import models
#
## SQLA models
from gargantext_web.db import *
from gargantext_web.db import session
################################################################################
## If you need to reset all data
......@@ -33,17 +33,15 @@ hyperdata = {
}
for name_, type_ in hyperdata.items():
data = (session.query(Hyperdata).filter(
data_ = (session.query(Hyperdata).filter(
Hyperdata.name == str(name_),
Hyperdata.type == str(type_)
).first()
)
if data is None:
if data_ is None:
print('Hyper Data' + name_ + 'does not existe, creating it')
hyperdata = Hyperdata(name=name_, type=type_)
session.add(hyperdata)
session.commit()
......
......@@ -103,6 +103,7 @@ def listNgramIds(list_id=None, typeList=None,
.group_by(Ngram.id, ListNgram.node_id)
)
# FIXME this is only used to filter on 1 document
if doc_id is not None:
Doc = aliased(Node)
DocNgram = aliased(NodeNgram)
......
from .NgramsExtractor import NgramsExtractor
from ..Taggers import NltkTagger
from ..Taggers import NltkTagger, MeltTagger
class EnglishNgramsExtractor(NgramsExtractor):
def start(self):
self.tagger = NltkTagger()
# self.tagger = NltkTagger()
self.tagger = MeltTagger(language='en')
\ No newline at end of file
from .NgramsExtractor import NgramsExtractor
from ..Taggers import TreeTagger
from ..Taggers import TreeTagger, MeltTagger
class FrenchNgramsExtractor(NgramsExtractor):
def start(self):
self.tagger = TreeTagger()
#self.tagger = TreeTagger()
self.tagger = MeltTagger(language='fr')
......@@ -14,24 +14,24 @@ class NgramsExtractor:
self._label = "NP"
self._rule = self._label + ": " + rule
self._grammar = nltk.RegexpParser(self._rule)
def __del__(self):
self.stop()
def start(self):
self.tagger = TurboTagger()
def stop(self):
pass
"""Extracts a list of ngrams.
Returns a list of the ngrams found in the given text.
"""
def extract_ngrams(self, contents):
tagged_ngrams = self.tagger.tag_text(contents)
if len(tagged_ngrams):
grammar_parsed = self._grammar.parse(tagged_ngrams)
tagged_tokens = list(self.tagger.tag_text(contents))
if len(tagged_tokens):
grammar_parsed = self._grammar.parse(tagged_tokens)
for subtree in grammar_parsed.subtrees():
if subtree.label() == self._label:
yield subtree.leaves()
from .FrenchNgramsExtractor import FrenchNgramsExtractor
from .TurboNgramsExtractor import TurboNgramsExtractor as EnglishNgramsExtractor
# from .EnglishNgramsExtractor import EnglishNgramsExtractor
#from .TurboNgramsExtractor import TurboNgramsExtractor as EnglishNgramsExtractor
from .EnglishNgramsExtractor import EnglishNgramsExtractor
from .NgramsExtractor import NgramsExtractor
......@@ -12,7 +12,8 @@ import os
class identity_dict(dict):
def __missing__(self, key):
return key
_tag_replacements = identity_dict({
_tag_replacements = dict()
_tag_replacements['fr'] = identity_dict({
'DET': 'DT',
'NC': 'NN',
'NPP': 'NNP',
......@@ -46,11 +47,18 @@ _tag_replacements = identity_dict({
# 'PREF': '',
# 'ADJWH': '',
})
_tag_replacements['en'] = identity_dict()
class MeltTagger(Tagger):
def start(self, language='fr', melt_data_path='lib/melttagger'):
def __init__(self, *args, **kwargs):
self.language = kwargs.pop('language', 'fr')
self._tag_replacements = _tag_replacements[self.language]
super(self.__class__, self).__init__(*args, **kwargs)
def start(self, melt_data_path='lib/melttagger'):
language = self.language
basepath = os.path.dirname(os.path.realpath(__file__))
path = os.path.join(basepath, melt_data_path)
self._pos_tagger = POSTagger()
......@@ -94,12 +102,12 @@ class MeltTagger(Tagger):
if len(token.string):
yield (token.string, token.label, )
def tag_text(self, text, lemmatize=True):
def tag_text(self, text, lemmatize=False):
tagged_tokens = self._tag(text)
# without lemmatization
if not lemmatize:
for form, tag in tagged_tokens:
yield (form, _tag_replacements[tag])
yield (form, self._tag_replacements[tag])
return
# with lemmatization
command_input = ' '.join(
......@@ -110,4 +118,4 @@ class MeltTagger(Tagger):
for token in lemmatized.split():
if len(token):
values = token.split('/')
yield (values[0], _tag_replacements[values[1]], values[2].replace('*', ''))
yield (values[0], self._tag_replacements[values[1]], values[2].replace('*', ''))
// Pre-defined constants
var operators = {
'text': [
{'label': 'contains', 'key': 'contains'}
{'label': 'contains', 'key': 'contains'},
{'label': 'does not contain', 'key': 'doesnotcontain'},
],
'string': [
{'label': 'starts with', 'key': 'startswith'},
{'label': 'contains', 'key': 'contains'},
{'label': 'does not contain', 'key': 'doesnotcontain'},
{'label': 'ends with', 'key': 'endswith'},
{'label': 'is', 'key': '='},
{'label': 'is before', 'key': '<'},
......@@ -259,11 +261,21 @@ gargantext.controller("DatasetController", function($scope, $http) {
$scope.corpora = [];
$http.get('/api/nodes?type=Project', {cache: true}).success(function(response){
$scope.projects = response.data;
// Initially set to what is indicated in the URL
if (/^\/project\/\d+\/corpus\/\d+/.test(location.pathname)) {
$scope.projectId = parseInt(location.pathname.split('/')[2]);
$scope.updateCorpora();
}
});
// update corpora according to the select parent project
$scope.updateCorpora = function() {
$http.get('/api/nodes?type=Corpus&parent=' + $scope.projectId, {cache: true}).success(function(response){
$scope.corpora = response.data;
// Initially set to what is indicated in the URL
if (/^\/project\/\d+\/corpus\/\d+/.test(location.pathname)) {
$scope.corpusId = parseInt(location.pathname.split('/')[4]);
$scope.updateEntities();
}
});
};
// update entities depending on the selected corpus
......@@ -522,8 +534,8 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
filters: query.filters,
sort: ['hyperdata.publication_date.day'],
retrieve: {
type: 'aggregates',
list: ['hyperdata.publication_date.day', query.mesured]
aggregate: true,
fields: ['hyperdata.publication_date.day', query.mesured]
}
};
// request to the server
......@@ -588,4 +600,4 @@ setTimeout(function(){
// // $('button.refresh').first().click();
}, 500);
}, 250);
*/
\ No newline at end of file
*/
......@@ -229,7 +229,7 @@
<hr/>
<div class="corpus">
<button ng-click="removeDataset($index)" title="remove this dataset">X</button>
<select ng-model="mesured" style="background-color:{{ getColor($index, datasets.length) }}" ng-options="value as key for (key, value) in {'Documents count': 'nodes.count', 'Ngrams count': 'ngrams.count', 'Simulation par Agents': 'agents.count'}" ng-change="updateQuery()"></select>
<select ng-model="mesured" style="background-color:{{ getColor($index, datasets.length) }}" ng-options="value as key for (key, value) in {'Documents count': 'nodes.count', 'Ngrams count': 'ngrams.count', 'Simulation par Agents': 'haskell.test'}" ng-change="updateQuery()"></select>
in the project
<select ng-model="projectId" ng-change="updateCorpora()" ng-options="project.id as project.name for project in projects"></select>,
corpus
......
......@@ -33,7 +33,7 @@
<div class="col-md-4">
<br>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{corpus.id}}/{{view}}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{corpus.id}}{% if view %}/{{view}}{% endif %}/update">Update</a>
<a class="btn btn-default btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Download</a>
<a type="button" class="btn btn-default btn-lg" data-container="body" data-toggle="popover" data-placement="bottom"
......
from parsing.Taggers import MeltTagger
# from parsing.Taggers.melttagger.tagger import POSTagger, Token, DAGParser, DAGReader
# # references:
# # - http://cs.nyu.edu/grishman/jet/guide/PennPOS.html
# # - http://www.lattice.cnrs.fr/sites/itellier/SEM.html
# class identity_dict(dict):
# def __missing__(self, key):
# return key
# _tag_replacements = identity_dict({
# 'DET': 'DT',
# 'NC': 'NN',
# 'NPP': 'NNP',
# 'ADJ': 'JJ',
# 'PONCT': '.',
# 'ADVWH': 'WRB',
# 'ADV': 'RB',
# 'DETWH': 'WDT',
# 'PROWH': 'WP',
# 'ET': 'FW',
# 'VINF': 'VB',
# 'I': 'UH',
# 'CS': 'IN',
# # 'CLS': '',
# # 'CLR': '',
# # 'CLO': '',
# # 'PRO': '',
# # 'PROREL': '',
# # 'P': '',
# # 'P+D': '',
# # 'P+PRO': '',
# # 'V': '',
# # 'VPR': '',
# # 'VPP': '',
# # 'VS': '',
# # 'VIMP': '',
# # 'PREF': '',
# # 'ADJWH': '',
# })
# import subprocess
# class MeltTagger:
# def __init__(self, language='fr', melt_data_path='./parsing/Taggers/melttagger'):
# path = '%s/%s' % (melt_data_path, language)
# self.pos_tagger = POSTagger()
# self.pos_tagger.load_tag_dictionary('%s/tag_dict.json' % path)
# self.pos_tagger.load_lexicon('%s/lexicon.json' % path)
# self.pos_tagger.load_model('%s' % path)
# self._preprocessing_commands = (
# # ('/usr/local/bin/clean_noisy_characters.sh', ),
# # ('/usr/local/bin/MElt_normalizer.pl', '-nc', '-c', '-d', '/usr/local/share/melt/normalization/%s' % language, '-l', language, ),
# ('/usr/local/share/melt/segmenteur.pl', '-a', '-ca', '-af=/usr/local/share/melt/pctabr', '-p', 'r'),
# )
# self._lemmatization_commands = (
# ('/usr/local/bin/MElt_postprocess.pl', '-npp', '-l', language),
# ('MElt_lemmatizer.pl', '-m', '/usr/local/share/melt/%s' % language),
# )
# def pipe(self, text, commands, encoding='utf8'):
# text = text.encode(encoding)
# # print(text.decode(encoding))
# for command in commands:
# # print(command)
# process = subprocess.Popen(
# command,
# bufsize=0,
# stdin=subprocess.PIPE,
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
# )
# text, err = process.communicate(text)
# # print()
# # print(text.decode(encoding))
# if len(err):
# print(err.decode(encoding))
# return text.decode(encoding)
# def tag(self, text, encoding='utf8', lemmatize=True):
# preprocessed = self.pipe(text, self._preprocessing_commands)
# if lemmatize:
# result = ''
# for sentence in preprocessed.split('\n'):
# words = sentence.split(' ')
# tokens = [Token(word) for word in words]
# tagged_tokens = self.pos_tagger.tag_token_sequence(tokens)
# # result += ' '.join(token.__str__() for token in tagged_tokens)
# for token in tagged_tokens:
# if len(token.string):
# result += '%s/%s ' % (token.string, token.label, )
# result += '\n'
# lemmatized = self.pipe(result, self._lemmatization_commands)
# for sentence in lemmatized.split('\n'):
# for token in sentence.split(' '):
# if len(token):
# yield tuple(token.split('/'))
# else:
# for sentence in preprocessed.split('\n'):
# words = sentence.split(' ')
# tokens = [Token(word) for word in words]
# tagged_tokens = self.pos_tagger.tag_token_sequence(tokens)
# for token in tagged_tokens:
# if len(token.string):
# yield (token.string, _tag_replacements[token.label], )
if __name__ == '__main__':
from time import time
t0 = time()
tagger = MeltTagger()
print(time() - t0)
print()
text = """Le vieil hôtel de ville, construit de 1608 à 1610 est le plus ancien bâtiment de la ville de Wiesbaden. Il se dresse sur la place centrale de la vieille ville, la Place du Palais, qui abrite aujourd'hui le Parlement de l'État de Hesse, l'église et l'hôtel de ville.
texts = {
'en':
"""Air raids on Japan by the Allies in World War II caused extensive destruction and casualties; the most commonly cited estimates are 333,000 killed and 473,000 wounded.
During the first years of the Pacific War, these attacks were limited to the Doolittle Raid in April 1942 and small-scale raids on military positions in the Kuril Islands starting in mid-1943. Strategic bombing raids began in June 1944 and were greatly expanded in November. The raids initially attempted to target industrial facilities, but from March 1945 onwards were generally directed against urban areas. Aircraft flying from aircraft carriers and the Ryukyu Islands also frequently struck targets in Japan during 1945 in preparation for an Allied invasion planned for October. In early August, the cities of Hiroshima and Nagasaki were struck and mostly destroyed by atomic bombs. Japan's military and civil defenses were not capable of protecting the country, and the Allied forces generally suffered few losses. The bombing campaign was one of the main factors in the Japanese government's decision to surrender in mid-August 1945. Nevertheless, there has been a long-running debate over the attacks on Japanese cities, and the decision to use atomic weapons has been particularly controversial.
""",
'fr':
"""Le vieil hôtel de ville, construit de 1608 à 1610 est le plus ancien bâtiment de la ville de Wiesbaden. Il se dresse sur la place centrale de la vieille ville, la Place du Palais, qui abrite aujourd'hui le Parlement de l'État de Hesse, l'église et l'hôtel de ville.
Il a été construit dans le style Renaissance. On a ajouté, en 1828, un étage de style romantique historié. Sur les bas-reliefs des cinq fenêtres de l'étage, en bois, étaient représentées les vertus de la force, la justice, la charité, de prudence et de modération, alors que la pierre a remplacé par des copies. Le pièces de chêne d'origine peut être visitées aujourd'hui au Musée de Wiesbaden. Aujourd'hui, le bâtiment sert de bureau de la ville de Wiesbaden.
Devant le porche, entre l'hôtel de Ville et l'Ancien hôtel de ville, se trouve la colonne centrale de Nassau, un lion couronné avec bouclier.
Il s'agit de construire progressivement, à partir des données initiales, un sous-graphe dans lequel sont classés les différents sommets par ordre croissant de leur distance minimale au sommet de départ. La distance correspond à la somme des poids des arêtes empruntées.
......@@ -129,7 +16,18 @@ if __name__ == '__main__':
Le plus proche des sommets adjacents est alors ajouté au sous-graphe.
La seconde étape consiste à mettre à jour les distances des sommets adjacents à ce dernier. Encore une fois, on recherche alors le sommet doté de la distance la plus faible. Comme tous les sommets n'avaient plus une valeur infinie, il est donc possible que le sommet choisi ne soit pas un des derniers mis à jour.
On l'ajoute au sous-graphe, puis on continue ainsi à partir du dernier sommet ajouté, jusqu'à épuisement des sommets ou jusqu'à sélection du sommet d'arrivée.
"""
""",
}
language = 'en'
text = texts[language]
if __name__ == '__main__':
from time import time
t0 = time()
tagger = MeltTagger(language=language)
print(time() - t0)
print()
i = 0
t0 = time()
for x in tagger.tag_text(text, lemmatize=True):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment