Skip to content

gargantext
gargantext
Commit 1676bf93 authored by sim's avatar sim
Browse files
Options

Remove old REST-API

parent 8d42b26a
Expand all Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 2881 deletions
gargantext/urls.py
View file @ 1676bf93
"""URL Configuration of GarganText
Views are shared between these modules:
- `api`, for JSON and CSV interaction with data
- `contents`, for Python-generated contents
"""
......@@ -10,11 +9,8 @@ from django.contrib import admin
from django.views.generic.base import RedirectView as Redirect
from django.contrib.staticfiles.storage import staticfiles_storage as static
import gargantext.views.api.urls
urlpatterns = [ url(r'^admin/' , admin.site.urls )
, url(r'^api/' , include( gargantext.views.api.urls ) )
, url(r'^favicon.ico$', Redirect.as_view( url=static.url('favicon.ico')
, permanent=False), name="favicon" )
]
gargantext/views/api/analytics.py deleted 100644 → 0
View file @ 8d42b26a
This diff is collapsed.
gargantext/views/api/api.py deleted 100644 → 0
View file @ 8d42b26a
from rest_framework.status import *
from rest_framework.exceptions import APIException
from rest_framework.response import Response
from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer
from rest_framework.views import APIView
from rest_framework.authentication import SessionAuthentication, BasicAuthentication
from rest_framework.permissions import IsAuthenticated
from gargantext.constants import RESOURCETYPES, NODETYPES, get_resource
from gargantext.models import Node, Ngram, NodeNgram, NodeNodeNgram, NodeNode
from gargantext.util.db import session, delete, func, bulk_insert
from gargantext.util.db_cache import cache, or_
from gargantext.util.files import upload
from gargantext.util.http import ValidationException, APIView, JsonHttpResponse, get_parameters
from gargantext.util.scheduling import scheduled
from gargantext.util.validation import validate
#import
#NODES format
_user_default_fields =["is_staff","is_superuser","is_active", "username", "email", "first_name", "last_name", "id"]
_api_default_fields = ['id', 'parent_id', 'name', 'typename', 'date']
_doc_default_fields = ['id', 'parent_id', 'name', 'typename', 'date', "hyperdata"]
#_resource_default_fields = [['id', 'parent_id', 'name', 'typename', "hyperdata.method"]
#_corpus_default_fields = ['id', 'parent_id', 'name', 'typename', 'date', "hyperdata","resource"]
def format_parent(node):
'''format the parent'''
try:
#USER
if node.username != "":
return {field: getattr(node, field) for field in _user_default_fields}
except:
#DOC
if node.typename == "DOCUMENT":
return {field: getattr(node, field) for field in _doc_default_fields}
elif node.typename == "CORPUS":
parent = {field: getattr(node, field) for field in _doc_default_fields}
#documents
#parent["documents"] = {"count":node.children("DOCUMENT").count()}
#resources
#parent["resources"] = {"count":node.children("RESOURCE").count()}
#status
#return {field: getattr(node, field) for field in _doc_default_fields}
parent["status_msg"] = status_message
return parent
#PROJECT, RESOURCES?
else:
return {field: getattr(node, field) for field in _api_default_fields}
def format_records(node_list):
'''format the records list'''
if len(node_list) == 0:
return []
node1 = node_list[0]
#USER
if node1.typename == "USER":
return [{field: getattr(node, field) for field in _user_default_fields} for node in node_list]
#DOCUMENT
elif node1.typename == "DOCUMENT":
return [{field: getattr(node, field) for field in _doc_default_fields} for node in node_list]
#CORPUS, PROJECT, RESOURCES?
elif node1.typename == "CORPUS":
records = []
for node in node_list:
#PROJECTS VIEW SHOULD NE BE SO DETAILED
record = {field: getattr(node, field) for field in _doc_default_fields}
record["resources"] = [n.id for n in node.children("RESOURCE")]
record["documents"] = [n.id for n in node.children("DOCUMENT")]
#record["resources"] = format_records([n for n in node.children("RESOURCE")])
#record["documents"] = format_records([n for n in node.children("DOCUMENT")])
status = node.status()
if status is not None and not status['complete']:
if not status['error']:
status_message = '(in progress: %s, %d complete)' % (
status['action'].replace('_', ' '),
status['progress'],
)
else:
status_message = '(aborted: "%s" after %i docs)' % (
status['error'][-1],
status['progress']
)
else:
status_message = ''
record["status"] = status_message
records.append(record)
return records
else:
return [{field: getattr(node, field) for field in _api_default_fields} for node in node_list]
def check_rights(request, node_id):
'''check that the node belong to USER'''
node = session.query(Node).filter(Node.id == node_id).first()
if node is None:
raise APIException("403 Unauthorized")
# return Response({'detail' : "Node #%s not found" %(node_id) },
# status = status.HTTP_404_NOT_FOUND)
elif node.user_id != request.user.id:
#response_data = {"log": "Unauthorized"}
#return JsonHttpResponse(response_data, status=403)
raise APIException("403 Unauthorized")
else:
return node
def format_response(parent, records):
#print(records)
return { "parent": format_parent(parent),
"records": format_records(records),
"count":len(records)
}
gargantext/views/api/corpora.py deleted 100644 → 0
View file @ 8d42b26a
from django.core.exceptions import *
from .api import * #APIView, APIException entre autres
from gargantext.util.db import session
from gargantext.models import Node
from gargantext.util.http import *
class CorpusView(APIView):
'''API endpoint that represent a corpus'''
def get(self, request, project_id, corpus_id, view = "DOCUMENT"):
'''GET corpus detail
default view full documents
'''
params = get_parameters(request)
if "view" in params.keys():
filter_view = params["view"].upper()
if view in ["DOCUMENT", "JOURNAL", "TITLE", "ANALYTICS", "RESSOURCE"]:
view = filter_view
project = session.query(Node).filter(Node.id == project_id, Node.typename == "PROJECT").first()
check_rights(request, project.id)
if project is None:
return Response({'detail' : "PROJECT Node #%s not found" %(project_id) },
status = status.HTTP_404_NOT_FOUND)
corpus = session.query(Node).filter(Node.id == corpus_id, Node.typename == "CORPUS").first()
if corpus is None:
return Response({'detail' : "CORPUS Node #%s not found" %(corpus_id) },
status = status.HTTP_404_NOT_FOUND)
documents = session.query(Node).filter(Node.parent_id == corpus_id, Node.typename == view).all()
context = format_response(corpus, documents)
return Response(context)
def delete(self, request, project_id, corpus_id):
'''DELETE corpus'''
print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>delete")
# project = session.query(Node).filter(Node.id == project_id, Node.typename == "PROJECT").first()
# check_rights(request, project.id)
# if project is None:
# return Response({'detail' : "PROJECT Node #%s not found" %(project_id) },
# status = status.HTTP_404_NOT_FOUND)
corpus = session.query(Node).filter(Node.id == corpus_id, Node.typename == "CORPUS").first()
if corpus is None:
return Response({'detail' : "CORPUS Node #%s not found" %(corpus_id) },
status = status.HTTP_404_NOT_FOUND)
documents = session.query(Node).filter(Node.parent_id == corpus_id).all()
session.delete(documents)
session.delete(corpus)
session.commit()
return Response(detail="Deleted corpus #%s" %str(corpus_id), status=HTTP_204_NO_CONTENT)
def put(self, request, project_id, corpus_id, view="DOCUMENT"):
'''UPDATE corpus'''
project = session.query(Node).filter(Node.id == project_id, Node.typename == "PROJECT").first()
project = check_rights(request, project.id)
if project is None:
return Response({'detail' : "PROJECT Node #%s not found" %(project_id) },
status = status.HTTP_404_NOT_FOUND)
corpus = session.query(Node).filter(Node.id == corpus_id, Node.typename == "CORPUS").first()
if corpus is None:
return Response({'detail' : "CORPUS Node #%s not found" %(corpus_id) },
status = status.HTTP_404_NOT_FOUND)
#documents = session.query(Node).filter(Node.parent_id == corpus_id, Node.typename= view).all()
for key, val in request.data.items():
if key in ["name", "date", "username", "hyperdata"]:
if key == "username":
#changement de propriétaire
#user = session.query(Node).filter(Node.typename=="USER", Node.username== username).first()
#print(user)
#set(node, user_id, user.id)
pass
elif key == "hyperdata":
#updating some contextualvalues of the corpus
pass
else:
setattr(node, key, val)
session.add(node)
session.commit()
'''#updating children???
'''
return Response({"detail":"Updated corpus #" %str(corpus.id)}, status=HTTP_202_ACCEPTED)
def post(self, request, project_id, corpus_id):
'''ADD a new RESOURCE to CORPUS'''
project = session.query(Node).filter(Node.id == project_id, Node.typename == "PROJECT").first()
check_rights(request, project.id)
if project is None:
return Response({'detail' : "PROJECT Node #%s not found" %(project_id) },
status = status.HTTP_404_NOT_FOUND)
corpus = session.query(Node).filter(Node.id == corpus_id, Node.typename == "CORPUS").first()
if corpus is None:
return Response({'detail' : "CORPUS Node #%s not found" %(corpus_id) },
status = status.HTTP_404_NOT_FOUND)
gargantext/views/api/metrics.py deleted 100644 → 0
View file @ 8d42b26a
from gargantext.util.db_cache import cache
from gargantext.util.http import ValidationException, APIView \
, HttpResponse, JsonHttpResponse
from gargantext.util.toolchain.main import recount
from gargantext.util.scheduling import scheduled
from datetime import datetime
class CorpusMetrics(APIView):
def patch(self, request, corpusnode_id):
"""
PATCH triggers recount of metrics for the specified corpus.
ex PATCH http://localhost:8000/api/metrics/14072
-----
corpus_id
"""
print("==> update metrics request on ", corpusnode_id)
if not request.user.is_authenticated():
# can't use @requires_auth because of positional 'self' within class
return HttpResponse('Unauthorized', status=401)
try:
corpus = cache.Node[int(corpusnode_id)]
except:
corpus = None
if corpus is None:
raise ValidationException("%s is not a valid corpus node id."
% corpusnode_id)
else:
t_before = datetime.now()
# =============
scheduled(recount)(corpus.id)
# =============
t_after = datetime.now()
return JsonHttpResponse({
'corpus_id' : corpusnode_id,
'took': "%f s." % (t_after - t_before).total_seconds()
})
gargantext/views/api/ngramlists.py deleted 100644 → 0
View file @ 8d42b26a
This diff is collapsed.
gargantext/views/api/ngrams.py deleted 100644 → 0
View file @ 8d42b26a
from gargantext.util.http import ValidationException, APIView \
, get_parameters, JsonHttpResponse\
, HttpResponse
from gargantext.util.db import session, func
from gargantext.util.db_cache import cache
from gargantext.models import Node, Ngram, NodeNgram, NodeNgramNgram
from sqlalchemy.orm import aliased
from re import findall
# ngrams put() will implement same text cleaning procedures as toolchain
from gargantext.util.toolchain.parsing import normalize_chars
from gargantext.util.toolchain.ngrams_extraction import normalize_forms
# for indexing
from gargantext.util.toolchain.ngrams_addition import index_new_ngrams
class ApiNgrams(APIView):
def get(self, request):
"""
Used for analytics
------------------
Get ngram listing + counts in a given scope
"""
# parameters retrieval and validation
startwith = request.GET.get('startwith', '').replace("'", "\\'")
# query ngrams
ParentNode = aliased(Node)
ngrams_query = (session
.query(Ngram.id, Ngram.terms, func.sum(NodeNgram.weight).label('count'))
.join(NodeNgram, NodeNgram.ngram_id == Ngram.id)
.join(Node, Node.id == NodeNgram.node_id)
.group_by(Ngram.id, Ngram.terms)
# .group_by(Ngram)
.order_by(func.sum(NodeNgram.weight).desc(), Ngram.terms)
)
# filters
if 'startwith' in request.GET:
ngrams_query = ngrams_query.filter(Ngram.terms.startswith(request.GET['startwith']))
if 'contain' in request.GET:
print("request.GET['contain']")
print(request.GET['contain'])
ngrams_query = ngrams_query.filter(Ngram.terms.contains(request.GET['contain']))
if 'corpus_id' in request.GET:
corpus_id_list = list(map(int, request.GET.get('corpus_id', '').split(',')))
if corpus_id_list and corpus_id_list[0]:
ngrams_query = ngrams_query.filter(Node.parent_id.in_(corpus_id_list))
if 'ngram_id' in request.GET:
ngram_id_list = list(map(int, request.GET.get('ngram_id', '').split(',')))
if ngram_id_list and ngram_id_list[0]:
ngrams_query = ngrams_query.filter(Ngram.id.in_(ngram_id_list))
# pagination
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 20))
total = ngrams_query.count()
# return formatted result
return JsonHttpResponse({
'pagination': {
'offset': offset,
'limit': limit,
'total': total,
},
'data': [
{
'id': ngram.id,
'terms': ngram.terms,
'count': ngram.count,
}
for ngram in ngrams_query[offset : offset+limit]
],
})
def put(self, request):
"""
Basic external access for *creating an ngram*
---------------------------------------------
1 - checks user authentication before any changes
2 - checks if ngram to Ngram table in DB
if yes returns ngram_id and optionally mainform_id
otherwise continues
3 - adds the ngram to Ngram table in DB
4 - (if corpus param is present)
adds the ngram doc counts to NodeNgram table in DB
(aka "index the ngram" throught the docs of the corpus)
5 - returns json with:
'msg' => a success msg
'text' => the initial text content
'term' => the normalized text content
'id' => the new ngram_id
'count' => the number of docs with the ngram in the corpus
(if corpus param is present)
'group' => the mainform_id if applicable
possible inline parameters
--------------------------
@param text=<ngram_string> [required]
@param corpus=<CORPUS_ID> [optional]
@param testgroup (true if present) [optional, requires corpus]
"""
# 1 - check user authentication
if not request.user.is_authenticated():
res = HttpResponse("Unauthorized")
res.status_code = 401
return res
# the params
params = get_parameters(request)
print("PARAMS", [(i,v) for (i,v) in params.items()])
if 'text' in params:
original_text = str(params.pop('text'))
ngram_str = normalize_forms(normalize_chars(original_text))
else:
raise ValidationException('The route PUT /api/ngrams/ is used to create a new ngram\
It requires a "text" parameter,\
for instance /api/ngrams?text=hydrometallurgy')
if ('testgroup' in params) and (not ('corpus' in params)):
raise ValidationException("'testgroup' param requires 'corpus' param")
# if we have a 'corpus' param (to do the indexing)...
do_indexation = False
if 'corpus' in params:
# we retrieve the corpus...
corpus_id = int(params.pop('corpus'))
corpus_node = cache.Node[corpus_id]
# and the user must also have rights on the corpus
if request.user.id == corpus_node.user_id:
do_indexation = True
else:
res = HttpResponse("Unauthorized")
res.status_code = 401
return res
# number of "words" in the ngram
ngram_size = len(findall(r' +', ngram_str)) + 1
# do the additions
try:
log_msg = ""
ngram_id = None
mainform_id = None
preexisting = session.query(Ngram).filter(Ngram.terms==ngram_str).first()
if preexisting is not None:
ngram_id = preexisting.id
log_msg += "ngram already existed (id %i)\n" % ngram_id
# in the context of a corpus we can also check if has mainform
# (useful for)
if 'testgroup' in params:
groupings_id = (session.query(Node.id)
.filter(Node.parent_id == corpus_id)
.filter(Node.typename == 'GROUPLIST')
.first()
)
had_mainform = (session.query(NodeNgramNgram.ngram1_id)
.filter(NodeNgramNgram.node_id == groupings_id)
.filter(NodeNgramNgram.ngram2_id == preexisting.id)
.first()
)
if had_mainform:
mainform_id = had_mainform[0]
log_msg += "ngram had mainform (id %i) in this corpus" % mainform_id
else:
log_msg += "ngram was not in any group for this corpus"
else:
# 2 - insert into Ngrams
new_ngram = Ngram(terms=ngram_str, n=ngram_size)
session.add(new_ngram)
session.commit()
ngram_id = new_ngram.id
log_msg += "ngram was added with new id %i\n" % ngram_id
# 3 - index the term
if do_indexation:
n_added = index_new_ngrams([ngram_id], corpus_node)
log_msg += 'ngram indexed in corpus %i\n' % corpus_id
return JsonHttpResponse({
'msg': log_msg,
'text': original_text,
'term': ngram_str,
'id' : ngram_id,
'group' : mainform_id,
'count': n_added if do_indexation else 'no corpus provided for indexation'
}, 200)
# just in case
except Exception as e:
return JsonHttpResponse({
'msg': str(e),
'text': original_text
}, 400)
gargantext/views/api/nodes.py deleted 100644 → 0
View file @ 8d42b26a
This diff is collapsed.
gargantext/views/api/projects.py deleted 100644 → 0
View file @ 8d42b26a
This diff is collapsed.
gargantext/views/api/urls.py deleted 100644 → 0
View file @ 8d42b26a
from django.conf.urls import url
from rest_framework_jwt.views import obtain_jwt_token
from . import nodes
from . import projects
from . import corpora
from . import users
from . import ngrams
from . import metrics
from . import ngramlists
from . import analytics
urlpatterns = [ url(r'^nodes$' , nodes.NodeListResource.as_view())
, url(r'^nodes/(\d+)$' , nodes.NodeResource.as_view())
, url(r'^nodes/(\d+)/having$' , nodes.NodeListHaving.as_view())
, url(r'^nodes/(\d+)/status$' , nodes.Status.as_view())
# Projects
, url(r'^projects$' , projects.ProjectList.as_view())
, url(r'^projects/(\d+)$' , projects.ProjectView.as_view())
# Corpora
, url(r'^projects/(\d+)/corpora/(\d+)$', corpora.CorpusView.as_view())
# Sources
#, url(r'^projects/(\d+)/corpora/(\d+)/sources$', corpora.CorpusSources.as_view())
#, url(r'^projects/(\d+)/corpora/(\d+)/sources/(\d+)$ , corpora.CorpusSourceView.as_view())
# Facets
, url(r'^projects/(\d+)/corpora/(\d+)/facets$', nodes.CorpusFacet.as_view())
# Favorites
, url(r'^projects/(\d+)/corpora/(\d+)/favorites$', nodes.CorpusFavorites.as_view())
# Metrics
, url(r'^projects/(\d+)/corpora/(\d+)/metrics$', metrics.CorpusMetrics.as_view())
# Ngrams
, url(r'^ngrams/?$' , ngrams.ApiNgrams.as_view())
# Analytics
, url(r'^nodes/(\d+)/histories$', analytics.NodeNgramsQueries.as_view())
, url(r'hyperdata$' , analytics.ApiHyperdata.as_view())
# get a list of ngram_ids or ngram_infos by list_id
# url(r'^ngramlists/(\d+)$', ngramlists.List.as_view()),
, url(r'^nodes/(\d+)/facets$' , nodes.CorpusFacet.as_view())
, url(r'^nodes/(\d+)/favorites$', nodes.CorpusFavorites.as_view())
# in these two routes the node is supposed to be a *corpus* node
, url(r'^metrics/(\d+)$' , metrics.CorpusMetrics.as_view())
# update all metrics for a corpus
# ex: PUT metrics/123
# \
# corpus id
, url(r'^ngramlists/export$', ngramlists.CSVLists.as_view())
# get a CSV export of the ngramlists of a corpus
# ex: GET ngramlists/export?corpus=43
# TODO : unify to a /api/ngrams?formatted=csv
# (similar to /api/nodes?formatted=csv)
, url(r'^ngramlists/import$', ngramlists.CSVLists.as_view())
# same handling class as export (CSVLists)
# but this route used only for POST + file
# or PATCH + other corpus id
, url(r'^ngramlists/change$', ngramlists.ListChange.as_view())
# add or remove ngram from a list
# ex: add <=> PUT ngramlists/change?list=42&ngrams=1,2
# rm <=> DEL ngramlists/change?list=42&ngrams=1,2
, url(r'^ngramlists/groups$', ngramlists.GroupChange.as_view())
# modify grouping couples of a group node
# ex: PUT/DEL ngramlists/groups?node=43
# & group data also in url: 767[]=209,640 & 779[]=436,265,385
, url(r'^ngramlists/family$', ngramlists.ListFamily.as_view())
# entire combination of lists from a corpus, dedicated to termtable
# (or any combination of lists that go together :
# - a mainlist
# - an optional stoplist
# - an optional maplist
# - an optional grouplist
, url(r'^ngramlists/maplist$', ngramlists.MapListGlance.as_view())
# fast access to maplist, similarly formatted for termtable
, url(r'^user/parameters/$', users.UserParameters.as_view())
, url('^auth/token$', obtain_jwt_token)
]
gargantext/views/api/users.py deleted 100644 → 0
View file @ 8d42b26a
from .api import * #notamment APIView, check_rights, format_response
from gargantext.util.http import *
from django.core.exceptions import *
from collections import defaultdict
from gargantext.util.toolchain import *
import copy
from gargantext.util.db import session
class UserParameters(APIView):
'''API endpoint that represent the parameters of the user'''
def get(self, request):
node_user = session.query(Node).filter(Node.user_id == request.user.id, Node.typename== "USER").first()
if node_user is None:
return Response({"detail":"Not Found"}, status=HTTP_404)
else:
#context = format_response(node_user, )
return Response(node_user.hyperdata)
def put(self, request):
if request.user.id is None:
raise TypeError("This API request must come from an authenticated user.")
else:
# we query among the nodes that belong to this user
user = cache.User[request.user.id]
node_user = session.query(Node).filter(Node.user_id == user.id, Node.typename== "USER").first()
if node_user is None:
return Response({"detail":"Not Allowed"}, status=HTTP_401_UNAUTHORIZED)
for k, v in request.data.items():
node_user.hyperdata[k] = v
# setattr(node_user.hyperdata, k, v)
# print(node_user.hyperdata)
node_user.save_hyperdata()
session.add(node_user)
session.commit()
node_user = session.query(Node).filter(Node.user_id == user.id, Node.typename== "USER").first()
print(node_user.hyperdata)
return Response({"detail":"Updated user parameters", "hyperdata": node_user.hyperdata}, status=HTTP_202_ACCEPTED)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment