Skip to content

gargantext
gargantext
Commit bde91b6e authored by Elias's avatar Elias
Browse files
Options

Connecting Elias' annotations app with the backend

parent 5fc5153d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 231 additions and 461 deletions
annotations/static/annotations/app.css
View file @ bde91b6e
...@@ -46,7 +46,7 @@ ...@@ -46,7 +46,7 @@
} }
.main-panel, .text-panel, .words-panel { .main-panel, .text-panel, .words-panel {
height: 400px; height: 800px;
margin: 10px 0px; margin: 10px 0px;
} }
......
annotations/static/annotations/app.js
View file @ bde91b6e
...@@ -382,9 +382,9 @@ ...@@ -382,9 +382,9 @@
$rootScope.abstract_text = data.abstract_text; $rootScope.abstract_text = data.abstract_text;
// GET the annotations // GET the annotations
$rootScope.annotationsResource = NgramListHttpService.get( $rootScope.annotationsResource = NgramListHttpService.get(
{'listId': $rootScope.listId, 'docId': $rootScope.docId} {'corpusId': $rootScope.corpusId, 'docId': $rootScope.docId}
).$promise.then(function(data) { ).$promise.then(function(data) {
$rootScope.annotations = data[$rootScope.listId.toString()][$rootScope.docId.toString()]; $rootScope.annotations = data[$rootScope.corpusId.toString()][$rootScope.docId.toString()];
}); });
}); });
//}); //});
...@@ -399,9 +399,10 @@ ...@@ -399,9 +399,10 @@
window.annotationsApp.run(function ($rootScope) { window.annotationsApp.run(function ($rootScope) {
/* GET the document node and all the annotations in the list associated */ /* GET the document node and all the annotations in the list associated */
// TODO debug var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\/document\/(.*)\//)
$rootScope.docId = 4; $rootScope.projectId = path[1];
$rootScope.listId = 1; $rootScope.corpusId = path[2];
$rootScope.docId = path[3];
}); });
})(window); })(window);
annotations/static/annotations/http.js
View file @ bde91b6e
(function () { (function () {
'use strict'; 'use strict';
var http = angular.module('annotationsAppHttp', ['ngResource']); var http = angular.module('annotationsAppHttp', ['ngResource']);
/*
/* * Read Document
* Read Document */
*/ http.factory('DocumentHttpService', function($resource) {
http.factory('DocumentHttpService', function($resource) { return $resource(
return $resource( window.ANNOTATION_API_URL + "document/:docId/",
window.ANNOTATION_API_URL + "document" + '/:docId/', {
{ docId: '@docId'
docId: '@docId' },
}, {
{ get: {
get: { method: 'GET',
method: 'GET', params: {docId: '@docId'}
params: {docId: '@docId'}
}
} }
); }
}); );
});
/* /*
* Read Ngram Lists * Read all Ngrams
*/ */
http.factory('NgramListHttpService', function ($resource) { http.factory('NgramListHttpService', function ($resource) {
return $resource( return $resource(
window.ANNOTATION_API_URL + 'lists' + '/:listId/', window.ANNOTATION_API_URL + 'corpus/:corpusId/document/:docId',
{ {
listId: '@listId' corpusId: '@corpusId',
}, docId: '@docId'
{ },
get: { {
method: 'GET', get: {
params: {listId: '@listId'} method: 'GET',
} params: {}
} }
); }
}); );
});
/* /*
* Create, modify or delete on Ngram of a list * Create, modify or delete 1 Ngram
*/ */
http.factory('NgramHttpService', function ($resource) { http.factory('NgramHttpService', function ($resource) {
return $resource( return $resource(
window.ANNOTATION_API_URL + 'lists' + '/:listId/ngrams/' + ':ngramId/', window.ANNOTATION_API_URL + 'lists/:listId/ngrams/:ngramId/',
{ {
listId: '@listId' listId: '@listId',
ngramId: '@ngramID'
},
{
post: {
method: 'POST',
params: {'listId': '@listId', 'ngramId': '@ngramId'}
}, },
{ delete: {
post: { method: 'DELETE',
method: 'POST', params: {'listId': '@listId', 'ngramId': '@ngramId'}
params: {'listId': '@listId', 'ngramId': '@ngramId'}
},
delete: {
method: 'DELETE',
params: {'listId': '@listId', 'ngramId': '@ngramId'}
}
} }
); }
}); );
// return { });
// newAnnotationObject: function(text, category, level) {
// return {
// 'text': text.trim(),
// 'category': category,
// 'level': level
// };
// },
// create: function(keyword, $rootScope) {
// if ($rootScope.annotations === undefined) $rootScope.annotations = [];
// // find duplicate by text
// var existing = _.find(
// $rootScope.annotations,
// function(annotation) { return annotation.text.trim().toLowerCase() === keyword.text.trim().toLowerCase(); }
// );
// // delete existing conflicting data before adding new
// if (existing) {
// if (existing.category == keyword.category && existing.level == keyword.level) return;
// this.delete(existing, $rootScope);
// }
// // TODO remove server mocking
// var mock = _.extend(keyword, {
// 'uuid': jQuery.now().toString(),
// 'occurrences': 322
// });
//
// $timeout(function() {
// $rootScope.$apply(function() {
// $rootScope.annotations.push(mock);
// });
// });
//
// return mock;
// },
// delete: function(keyword, $rootScope) {
// var filtered = _.filter($rootScope.annotations, function(item) {
// if (item.uuid == keyword.uuid) {
// return false;
// } else {
// return true;
// }
// });
// $timeout(function() {
// $rootScope.$apply(function() {
// $rootScope.annotations = filtered;
// });
// });
// }
// };
})(window); })(window);
annotations/templates/annotations/demo.html → annotations/templates/annotations/main.html
View file @ bde91b6e
...@@ -102,6 +102,7 @@ ...@@ -102,6 +102,7 @@
<script type="application/javascript"> <script type="application/javascript">
window.STATIC_URL = "{% static '' %}"; window.STATIC_URL = "{% static '' %}";
window.ANNOTATION_API_URL = "{{ api_url }}"; window.ANNOTATION_API_URL = "{{ api_url }}";
window.NODES_API_URL = "{{ nodes_api_url }}";
</script> </script>
<script src="{% static 'annotations/main.js' %}"></script> <script src="{% static 'annotations/main.js' %}"></script>
......
annotations/urls.py
View file @ bde91b6e
...@@ -3,9 +3,8 @@ from annotations import views ...@@ -3,9 +3,8 @@ from annotations import views
urlpatterns = patterns('', urlpatterns = patterns('',
url(r'^demo/$', views.demo),
url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view
#url(r'^document/(?P<doc_id>[0-9]+)/ngrams/(?P<ngram_id>[0-9]+)$', views.DocumentNgram.as_view()), # actions on ngram from a document url(r'^corpus/(?P<corpus_id>[0-9]+)$', views.CorpusList.as_view()), # the list associated with an ngram
url(r'^lists/(?P<list_id>[0-9]+)$', views.NgramList.as_view()), # actions on list filtered by document url(r'^corpus/(?P<corpus_id>[0-9]+)/document/(?P<doc_id>[0-9]+)$', views.NgramList.as_view()), # the list associated with an ngram
url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.Ngram.as_view()), # actions on ngram from a list optionally filtered by document url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.Ngram.as_view()), #
) )
annotations/views.py
View file @ bde91b6e
This diff is collapsed.
gargantext_web/api.py
View file @ bde91b6e
...@@ -562,6 +562,7 @@ class NodesList(APIView): ...@@ -562,6 +562,7 @@ class NodesList(APIView):
for node in query.all() for node in query.all()
]}) ]})
class Nodes(APIView): class Nodes(APIView):
def get(self, request, node_id): def get(self, request, node_id):
...@@ -652,39 +653,3 @@ class CorpusController: ...@@ -652,39 +653,3 @@ class CorpusController:
) )
else: else:
raise ValidationError('Unrecognized "format=%s", should be "csv" or "json"' % (format, )) raise ValidationError('Unrecognized "format=%s", should be "csv" or "json"' % (format, ))
from ngram.lists import listIds, ngramList
class ListManagement(APIView):
#authentication_classes = (SessionAuthentication, BasicAuthentication)
# TODO: Be carefull need authentication!
def get(self, request, corpus_id):
user_id = session.query(User.id).filter(User.username==str(request.user)).first()[0]
lists = dict()
for list_type in ['MiamList', 'StopList']:
list_id = list()
list_id = listIds(user_id=user_id, corpus_id=int(corpus_id), typeList=list_type)
lists[list_type] = int(list_id[0][0])
# lists[list_type]['id']['name'] = r[0][1]
return JsonHttpResponse({
'MiamList' : lists['MiamList'],
'StopList' : lists['StopList']
})
def post(self, request, corpus_id):
list_id = request.POST.get('list_id')
ngram_ids = request.POST.get('ngram_ids')
ngramList(do='add', ngram_ids=ngram_ids, list_id=list_id)
def delete(self, request, corpus_id):
list_id = request.POST.get('list_id')
ngram_ids = request.POST.get('ngram_ids')
ngramList(do='del', ngram_ids=ngram_ids, list_id=list_id)
gargantext_web/urls.py
View file @ bde91b6e
...@@ -5,6 +5,8 @@ from django.contrib.auth.views import login ...@@ -5,6 +5,8 @@ from django.contrib.auth.views import login
from gargantext_web import views, views_optimized from gargantext_web import views, views_optimized
from annotations import urls as annotations_urls from annotations import urls as annotations_urls
from annotations.views import main as annotations_main_view
import gargantext_web.api import gargantext_web.api
import scrappers.scrap_pubmed.views as pubmedscrapper import scrappers.scrap_pubmed.views as pubmedscrapper
...@@ -39,6 +41,11 @@ urlpatterns = patterns('', ...@@ -39,6 +41,11 @@ urlpatterns = patterns('',
# Corpus management # Corpus management
url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus), url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
# annotations App
url(r'^project/(\d+)/corpus/(\d+)/document/(\d+)/$', annotations_main_view),
url(r'^annotations/', include(annotations_urls)),
#
url(r'^project/(\d+)/corpus/(\d+)/corpus.csv$', views.corpus_csv), url(r'^project/(\d+)/corpus/(\d+)/corpus.csv$', views.corpus_csv),
url(r'^project/(\d+)/corpus/(tests_mvc_listdocuments+)/corpus.tests_mvc_listdocuments$', views.corpus_csv), url(r'^project/(\d+)/corpus/(tests_mvc_listdocuments+)/corpus.tests_mvc_listdocuments$', views.corpus_csv),
...@@ -63,13 +70,8 @@ urlpatterns = patterns('', ...@@ -63,13 +70,8 @@ urlpatterns = patterns('',
url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()), url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()), url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
# url(r'^api/nodes/(\d+)/children/duplicates/delete$', gargantext_web.api.NodesChildrenDuplicates.delete ), # url(r'^api/nodes/(\d+)/children/duplicates/delete$', gargantext_web.api.NodesChildrenDuplicates.delete ),
url(r'^api/corpus/(\d+)/lists$', gargantext_web.api.ListManagement.as_view()),
url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams), url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
url(r'^annotations/', include(annotations_urls)),
# Provisory tests # Provisory tests
url(r'^ngrams$', views.ngrams), # to be removed url(r'^ngrams$', views.ngrams), # to be removed
url(r'^nodeinfo/(\d+)$', views.nodeinfo), # to be removed ? url(r'^nodeinfo/(\d+)$', views.nodeinfo), # to be removed ?
...@@ -100,12 +102,12 @@ if settings.DEBUG: ...@@ -100,12 +102,12 @@ if settings.DEBUG:
if settings.MAINTENANCE: if settings.MAINTENANCE:
urlpatterns = patterns('', urlpatterns = patterns('',
url(r'^img/logo.svg$', views.logo), url(r'^img/logo.svg$', views.logo),
url(r'^css/bootstrap.css$', views.css), url(r'^css/bootstrap.css$', views.css),
url(r'^$', views.home_view), url(r'^$', views.home_view),
url(r'^about/', views.get_about), url(r'^about/', views.get_about),
url(r'^admin/', include(admin.site.urls)), url(r'^admin/', include(admin.site.urls)),
url(r'^.*', views.get_maintenance), url(r'^.*', views.get_maintenance),
) )
ngram/lists.py
View file @ bde91b6e
...@@ -11,9 +11,9 @@ from sqlalchemy import desc, asc, or_, and_, Date, cast, select ...@@ -11,9 +11,9 @@ from sqlalchemy import desc, asc, or_, and_, Date, cast, select
from sqlalchemy import literal_column from sqlalchemy import literal_column
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
# from gargantext_web.db import Node, get_cursor
def listIds(user_id=None, corpus_id=None, typeList='MiamList'):
def listIds(typeList=None, user_id=None, corpus_id=None):
''' '''
nodeList : get or create NodeList. nodeList : get or create NodeList.
nodeList :: Integer -> Integer -> String -> [Node] nodeList :: Integer -> Integer -> String -> [Node]
...@@ -22,6 +22,9 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'): ...@@ -22,6 +22,9 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'):
typeList :: String, Type of the Node that should be created typeList :: String, Type of the Node that should be created
[Node] :: List of Int, returned or created by the function [Node] :: List of Int, returned or created by the function
''' '''
if typeList is None:
typeList = 'MiamList'
if corpus_id is not None and user_id is not None: if corpus_id is not None and user_id is not None:
# Nodes are either in root_list or user_list # Nodes are either in root_list or user_list
...@@ -39,9 +42,7 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'): ...@@ -39,9 +42,7 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'):
Node.type_id == cache.NodeType[typeList].id Node.type_id == cache.NodeType[typeList].id
).order_by(desc(Node.id)).all() ).order_by(desc(Node.id)).all()
else: else:
print('typeList not supported yet') raise Exception("typeList %s not supported yet" % typeList)
sys.exit(0)
if nodes == []: if nodes == []:
node = Node(user_id = user_id, node = Node(user_id = user_id,
...@@ -56,12 +57,12 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'): ...@@ -56,12 +57,12 @@ def listIds(user_id=None, corpus_id=None, typeList='MiamList'):
return([(node.id, node.name) for node in nodes]) return([(node.id, node.name) for node in nodes])
else: else:
print("Usage (Warning): Need corpus_id and user_id") raise Exception("Usage (Warning): Need corpus_id and user_id")
# Some functions to manage ngrams according to the lists # Some functions to manage ngrams according to the lists
def listNgramIds(list_id=None, typeList=None, def listNgramIds(list_id=None, typeList=None,
corpus_id=None, doc_id=None, user_id=None): corpus_id=None, doc_id=None, user_id=None):
''' '''
listNgramsIds :: Int | String, Int, Int, Int -> [(Int, String, Int)] listNgramsIds :: Int | String, Int, Int, Int -> [(Int, String, Int)]
return has types: [(ngram_id, ngram_terms, occurrences)] return has types: [(ngram_id, ngram_terms, occurrences)]
...@@ -75,49 +76,49 @@ def listNgramIds(list_id=None, typeList=None, ...@@ -75,49 +76,49 @@ def listNgramIds(list_id=None, typeList=None,
doc_id : to get specific ngrams related to a document with Node.id=doc_id doc_id : to get specific ngrams related to a document with Node.id=doc_id
user_id : needed to create list if it does not exist user_id : needed to create list if it does not exist
''' '''
if typeList is None:
typeList = ['MiamList', 'StopList']
elif isinstance(typeList, string):
typeList = [typeList]
if list_id is None : if list_id is None and corpus_id is None:
if corpus_id is not None : raise Exception('Need a listId or corpusId to query')
if typeList is not None :
if user_id is not None :
try:
list_id = listIds(user_id=user_id,
corpus_id=corpus_id,
typeList=typeList)[0][0]
except:
PrintException()
else:
print('Need a user_id to create list if needed')
sys.exit()
else:
print('Need a typeList parameter')
sys.exit()
else:
print('Need a node_id to take default list of type' + typeList)
sys.exit()
else:
ListNgram = aliased(NodeNgram)
query = (session.query(Ngram.id, Ngram.terms, func.count())
.join(ListNgram, ListNgram.ngram_id == Ngram.id)
.filter(ListNgram.node_id == list_id)
.group_by(Ngram.id)
)
if doc_id is not None :
Doc = aliased(Node)
DocNgram = aliased(NodeNgram)
query = (query
.join(DocNgram, DocNgram.ngram_id == Ngram.id)
.join(Doc, Doc.id == doc_id)
.filter(DocNgram.node_id == Doc.id)
)
return(query.all()) if user_id is None:
raise Exception("Need a user_id to create list if needed")
# iterate over every list in a corpus
try:
allLists = []
for aType in typeList:
allLists += listIds(user_id=user_id, corpus_id=corpus_id, typeList=aType)
except Exception as exc:
PrintException()
raise exc
ListNgram = aliased(NodeNgram)
or_args = [ListNgram.node_id == l[0] for l in allLists]
query = (session.query(Ngram.id, Ngram.terms, func.count(), ListNgram.node_id)
.join(ListNgram, ListNgram.ngram_id == Ngram.id)
.filter(or_(*or_args))
.group_by(Ngram.id, ListNgram)
)
if doc_id is not None:
Doc = aliased(Node)
DocNgram = aliased(NodeNgram)
query = (query
.join(DocNgram, DocNgram.ngram_id == Ngram.id)
.join(Doc, Doc.id == doc_id)
.filter(DocNgram.node_id == Doc.id)
)
return(query.all())
def ngramList(do=None, ngram_ids=[], list_id=None) :
def ngramList(do=None, ngram_ids=None, list_id=None) :
''' '''
,gramList :: ([Int], Int, String) -> Bool ngramList :: ([Int], Int, String) -> Bool
Do (delete | add) [ngram_id] (from | to) the list_id Do (delete | add) [ngram_id] (from | to) the list_id
options: options:
...@@ -125,40 +126,42 @@ def ngramList(do=None, ngram_ids=[], list_id=None) : ...@@ -125,40 +126,42 @@ def ngramList(do=None, ngram_ids=[], list_id=None) :
ngram_id = [Int] : list of Ngrams id (Ngrams.id) ngram_id = [Int] : list of Ngrams id (Ngrams.id)
list_id = Int : list id (Node.id) list_id = Int : list id (Node.id)
''' '''
if do is None or ngram_ids == [] or list_id is None : if ngram_ids is None:
print('Need more options: do, ngram_id, list_id') raise Exception('Need at least one ngram id in ngram_ids')
sys.exit(0)
else:
try:
node_type_id = (session.query(Node.type_id)
.filter(Node.id == list_id)
.first()
)
for ngram_id in ngram_ids:
# First we test to know if ngram exist in database already
#ngram = (session.query(Ngram).filter(Ngram.id == ngram_id).first()
# Need to be optimized with list of ids
node_ngram = (session.query(NodeNgram)
.filter(NodeNgram.ngram_id == ngram_id)
.filter(NodeNgram.node_id == list_id)
.first()
)
if node_ngram is None :
node_ngram = NodeNgram(node_id = list_id,
ngram_id=ngram_id,
weight=1)
if do == 'add' :
session.add(node_ngram)
elif do == 'del' :
session.delete(node_ngram)
session.commit() if do is None or list_id is None :
return(True) raise Exception('Need more options: do, ngram_id, list_id')
try:
# node_type_id = (session.query(Node.type_id)
# .filter(Node.id == list_id)
# .first()
# )
for ngram_id in ngram_ids:
# First we test to know if ngram exist in database already
#ngram = (session.query(Ngram).filter(Ngram.id == ngram_id).first()
# Need to be optimized with list of ids
node_ngram = (session.query(NodeNgram)
.filter(NodeNgram.ngram_id == ngram_id)
.filter(NodeNgram.node_id == list_id)
.first()
)
if node_ngram is None :
node_ngram = NodeNgram(node_id = list_id,
ngram_id=ngram_id,
weight=1)
if do == 'add' :
session.add(node_ngram)
elif do == 'del' :
session.delete(node_ngram)
session.commit()
return(True)
except: except:
PrintException() PrintException()
return(False) return(False)
...@@ -174,7 +177,7 @@ def doStopList(user_id=None, corpus_id=None, ...@@ -174,7 +177,7 @@ def doStopList(user_id=None, corpus_id=None,
''' '''
if stop_id is None: if stop_id is None:
stop_id = nodeListIds(user_id=user_id, stop_id = listNgramIds(user_id=user_id,
corpus_id=corpus_id, corpus_id=corpus_id,
typeList='StopList')[0] typeList='StopList')[0]
# according to type of corpus, choose the right default stopList # according to type of corpus, choose the right default stopList
...@@ -182,12 +185,12 @@ def doStopList(user_id=None, corpus_id=None, ...@@ -182,12 +185,12 @@ def doStopList(user_id=None, corpus_id=None,
def doList( def doList(
type_list='miam', type_list='miam',
user_id=None, corpus_id=None, user_id=None, corpus_id=None,
miam_id=None, stop_id=None, main_id=None, miam_id=None, stop_id=None, main_id=None,
lem_id=None, stem_id=None, cvalue_id=None, group_id=None, lem_id=None, stem_id=None, cvalue_id=None, group_id=None,
reset=True, limit=None reset=True, limit=None
): ):
''' '''
Compute the miamList and returns its Node.id Compute the miamList and returns its Node.id
miamList = allList - stopList miamList = allList - stopList
...@@ -228,7 +231,7 @@ def doList( ...@@ -228,7 +231,7 @@ def doList(
for list_ in list_dict.keys(): for list_ in list_dict.keys():
if list_dict[list_]['id'] is None: if list_dict[list_]['id'] is None:
list_dict[list_]['id'] = nodeListIds(user_id=user_id, list_dict[list_]['id'] = listNgramIds(user_id=user_id,
corpus_id=corpus_id, corpus_id=corpus_id,
typeList=list_dict[list_]['type'])[0][0] typeList=list_dict[list_]['type'])[0][0]
# Delete previous List ? # Delete previous List ?
...@@ -241,8 +244,7 @@ def doList( ...@@ -241,8 +244,7 @@ def doList(
except: except:
PrintException() PrintException()
stopNgram = aliased(NodeNgram) stopNgram = aliased(NodeNgram)
if 'miam' == type_list: if 'miam' == type_list:
query = (session.query( query = (session.query(
...@@ -314,4 +316,3 @@ def doList( ...@@ -314,4 +316,3 @@ def doList(
bulk_insert(NodeNgram, ['node_id', 'ngram_id', 'weight'], query) bulk_insert(NodeNgram, ['node_id', 'ngram_id', 'weight'], query)
return(list_dict[type_list]['id']) return(list_dict[type_list]['id'])
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment