Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
humanities
gargantext
Commits
8f529d4f
Commit
8f529d4f
authored
Jul 06, 2017
by
sim
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[DB] Add a revision to create OCC_HIST & OCC_HIST_PART SQL functions
parent
802a1aa2
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
208 additions
and
0 deletions
+208
-0
601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py
...ions/601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py
+96
-0
alembic.py
gargantext/tools/alembic.py
+112
-0
No files found.
alembic/versions/601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py
0 → 100644
View file @
8f529d4f
"""Add OCC_HIST & OCC_HIST_PART functions
Revision ID: 601e9d9baa4c
Revises: 932dbf3e8c43
Create Date: 2017-07-06 10:52:16.161118
"""
from
alembic
import
op
import
sqlalchemy
as
sa
from
gargantext.tools.alembic
import
ReplaceableObject
# revision identifiers, used by Alembic.
revision
=
'601e9d9baa4c'
down_revision
=
'932dbf3e8c43'
branch_labels
=
None
depends_on
=
None
# -- OCC_HIST_PART :: Corpus.id -> GroupList.id -> Start -> End
occ_hist_part
=
ReplaceableObject
(
"OCC_HIST_PART(int, int, timestamp, timestamp)"
,
"""
RETURNS TABLE (ng_id int, score float8)
AS $$
-- EXPLAIN ANALYZE
SELECT
COALESCE(gr.ngram1_id, ng1.ngram_id) as ng_id,
SUM(ng1.weight) as score
from nodes n
-- BEFORE
INNER JOIN nodes as n1 ON n1.id = n.id
INNER JOIN nodes_ngrams ng1 ON ng1.node_id = n1.id
-- Limit with timestamps: ]start, end]
INNER JOIN nodes_hyperdata nh1 ON nh1.node_id = n1.id
AND nh1.value_utc > $3
AND nh1.value_utc <= $4
-- Group List
LEFT JOIN nodes_ngrams_ngrams gr ON ng1.ngram_id = gr.ngram2_id
AND gr.node_id = $2
WHERE
n.typename = 4
AND n.parent_id = $1
GROUP BY 1
$$
LANGUAGE SQL;
"""
)
# -- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
# -- EXEMPLE USAGE
# -- SELECT * FROM OCC_HIST(182856, 183859, 183866, '1800-03-15 17:00:00+01', '2000-03-15 17:00:00+01', '2017-03-15 17:00:00+01')
occ_hist
=
ReplaceableObject
(
"OCC_HIST(int, int, int, timestamp, timestamp, timestamp)"
,
"""
RETURNS TABLE (ng_id int, score numeric)
AS $$
WITH OCC1 as (SELECT * from OCC_HIST_PART($1, $2, $4, $5))
, OCC2 as (SELECT * from OCC_HIST_PART($1, $2, $5, $6))
, GROWTH as (SELECT ml.ngram_id as ngram_id
, COALESCE(OCC1.score, null) as score1
, COALESCE(OCC2.score, null) as score2
FROM nodes_ngrams ml
LEFT JOIN OCC1 ON OCC1.ng_id = ml.ngram_id
LEFT JOIN OCC2 ON OCC2.ng_id = ml.ngram_id
WHERE ml.node_id = $3
ORDER by score2 DESC)
SELECT ngram_id, COALESCE(ROUND(CAST((100 * (score2 - score1) / COALESCE((score2 + score1), 1)) as numeric), 2), 0) from GROWTH
$$
LANGUAGE SQL;
"""
)
# -- BEHAVIORAL TEST (should be equal to occ in terms table)
# -- WITH OCC as (SELECT * from OCC_HIST(182856, 183859, '1800-03-15 17:00:00+01', '2300-03-15 17:00:00+01'))
# -- SELECT ng_id, score from OCC
# -- INNER JOIN nodes_ngrams ml on ml.ngram_id = ng_id
# -- AND ml.node_id = 183866
# -- ORDER BY score DESC;
def
upgrade
():
op
.
create_sp
(
occ_hist_part
)
op
.
create_sp
(
occ_hist
)
def
downgrade
():
op
.
drop_sp
(
occ_hist
)
op
.
drop_sp
(
occ_hist_part
)
gargantext/tools/alembic.py
0 → 100644
View file @
8f529d4f
"""Define ReplaceableObject and related operations
Implements operations to create/drop SQL objects such as views, stored
procedures and triggers that can't be "altered" but can be replaced -- hence
the name of "ReplaceableObject" class.
This recipe is directly borrowed from Alembic documentation, see
http://alembic.zzzcomputing.com/en/latest/cookbook.html#replaceable-objects
"""
from
alembic.operations
import
Operations
,
MigrateOperation
__all__
=
[
'ReplaceableObject'
]
class
ReplaceableObject
(
object
):
def
__init__
(
self
,
name
,
sqltext
):
self
.
name
=
name
self
.
sqltext
=
sqltext
class
ReversibleOp
(
MigrateOperation
):
def
__init__
(
self
,
target
):
self
.
target
=
target
@
classmethod
def
invoke_for_target
(
cls
,
operations
,
target
):
op
=
cls
(
target
)
return
operations
.
invoke
(
op
)
def
reverse
(
self
):
raise
NotImplementedError
()
@
classmethod
def
_get_object_from_version
(
cls
,
operations
,
ident
):
version
,
objname
=
ident
.
split
(
"."
)
module
=
operations
.
get_context
()
.
script
.
get_revision
(
version
)
.
module
obj
=
getattr
(
module
,
objname
)
return
obj
@
classmethod
def
replace
(
cls
,
operations
,
target
,
replaces
=
None
,
replace_with
=
None
):
if
replaces
:
old_obj
=
cls
.
_get_object_from_version
(
operations
,
replaces
)
drop_old
=
cls
(
old_obj
)
.
reverse
()
create_new
=
cls
(
target
)
elif
replace_with
:
old_obj
=
cls
.
_get_object_from_version
(
operations
,
replace_with
)
drop_old
=
cls
(
target
)
.
reverse
()
create_new
=
cls
(
old_obj
)
else
:
raise
TypeError
(
"replaces or replace_with is required"
)
operations
.
invoke
(
drop_old
)
operations
.
invoke
(
create_new
)
@
Operations
.
register_operation
(
"create_view"
,
"invoke_for_target"
)
@
Operations
.
register_operation
(
"replace_view"
,
"replace"
)
class
CreateViewOp
(
ReversibleOp
):
def
reverse
(
self
):
return
DropViewOp
(
self
.
target
)
@
Operations
.
register_operation
(
"drop_view"
,
"invoke_for_target"
)
class
DropViewOp
(
ReversibleOp
):
def
reverse
(
self
):
return
CreateViewOp
(
self
.
view
)
@
Operations
.
register_operation
(
"create_sp"
,
"invoke_for_target"
)
@
Operations
.
register_operation
(
"replace_sp"
,
"replace"
)
class
CreateSPOp
(
ReversibleOp
):
def
reverse
(
self
):
return
DropSPOp
(
self
.
target
)
@
Operations
.
register_operation
(
"drop_sp"
,
"invoke_for_target"
)
class
DropSPOp
(
ReversibleOp
):
def
reverse
(
self
):
return
CreateSPOp
(
self
.
target
)
@
Operations
.
implementation_for
(
CreateViewOp
)
def
create_view
(
operations
,
operation
):
operations
.
execute
(
"CREATE VIEW
%
s AS
%
s"
%
(
operation
.
target
.
name
,
operation
.
target
.
sqltext
))
@
Operations
.
implementation_for
(
DropViewOp
)
def
drop_view
(
operations
,
operation
):
operations
.
execute
(
"DROP VIEW
%
s"
%
operation
.
target
.
name
)
@
Operations
.
implementation_for
(
CreateSPOp
)
def
create_sp
(
operations
,
operation
):
operations
.
execute
(
"CREATE FUNCTION
%
s
%
s"
%
(
operation
.
target
.
name
,
operation
.
target
.
sqltext
)
)
@
Operations
.
implementation_for
(
DropSPOp
)
def
drop_sp
(
operations
,
operation
):
operations
.
execute
(
"DROP FUNCTION
%
s"
%
operation
.
target
.
name
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment