[DB] Add a revision to create OCC_HIST & OCC_HIST_PART SQL functions

alembic/versions/601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py

+"""Add OCC_HIST & OCC_HIST_PART functions
+
+Revision ID: 601e9d9baa4c
+Revises: 932dbf3e8c43
+Create Date: 2017-07-06 10:52:16.161118
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from gargantext.tools.alembic import ReplaceableObject
+
+
+# revision identifiers, used by Alembic.
+revision = '601e9d9baa4c'
+down_revision = '932dbf3e8c43'
+branch_labels = None
+depends_on = None
+
+
+# -- OCC_HIST_PART :: Corpus.id -> GroupList.id -> Start -> End
+
+occ_hist_part = ReplaceableObject(
+    "OCC_HIST_PART(int, int, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score float8)
+    AS $$
+-- EXPLAIN ANALYZE
+    SELECT
+    COALESCE(gr.ngram1_id, ng1.ngram_id) as ng_id,
+    SUM(ng1.weight) as score
+
+    from nodes n
+
+    -- BEFORE
+    INNER JOIN nodes as n1 ON n1.id = n.id
+
+    INNER JOIN nodes_ngrams ng1 ON ng1.node_id = n1.id
+
+    -- Limit with timestamps: ]start, end]
+    INNER JOIN nodes_hyperdata nh1 ON nh1.node_id = n1.id
+                                  AND nh1.value_utc >  $3
+                                  AND nh1.value_utc <= $4
+
+    -- Group List
+    LEFT JOIN  nodes_ngrams_ngrams gr ON ng1.ngram_id = gr.ngram2_id
+                               AND gr.node_id = $2
+
+    WHERE
+        n.typename  = 4
+    AND n.parent_id = $1
+    GROUP BY 1
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
+# -- EXEMPLE USAGE
+# --    SELECT * FROM OCC_HIST(182856, 183859, 183866, '1800-03-15 17:00:00+01', '2000-03-15 17:00:00+01', '2017-03-15 17:00:00+01')
+
+occ_hist = ReplaceableObject(
+    "OCC_HIST(int, int, int, timestamp, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score numeric)
+    AS $$
+    WITH OCC1 as (SELECT * from OCC_HIST_PART($1, $2, $4, $5))
+       , OCC2 as (SELECT * from OCC_HIST_PART($1, $2, $5, $6))
+       , GROWTH as (SELECT ml.ngram_id as ngram_id
+                 , COALESCE(OCC1.score, null) as score1
+                 , COALESCE(OCC2.score, null) as score2
+                    FROM nodes_ngrams ml
+                        LEFT JOIN OCC1 ON OCC1.ng_id = ml.ngram_id
+                        LEFT JOIN OCC2 ON OCC2.ng_id = ml.ngram_id
+                    WHERE ml.node_id = $3
+                    ORDER by score2 DESC)
+    SELECT ngram_id, COALESCE(ROUND(CAST((100 * (score2 - score1) / COALESCE((score2 + score1), 1)) as numeric), 2), 0) from GROWTH
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- BEHAVIORAL TEST (should be equal to occ in terms table)
+# --    WITH OCC as (SELECT * from OCC_HIST(182856, 183859, '1800-03-15 17:00:00+01', '2300-03-15 17:00:00+01'))
+# --    SELECT ng_id, score from OCC
+# --            INNER JOIN nodes_ngrams ml on ml.ngram_id = ng_id
+# --                                      AND ml.node_id = 183866
+# --            ORDER BY score DESC;
+
+def upgrade():
+    op.create_sp(occ_hist_part)
+    op.create_sp(occ_hist)
+
+
+def downgrade():
+    op.drop_sp(occ_hist)
+    op.drop_sp(occ_hist_part)

gargantext/tools/alembic.py

+"""Define ReplaceableObject and related operations
+
+Implements operations to create/drop SQL objects such as views, stored
+procedures and triggers that can't be "altered" but can be replaced -- hence
+the name of "ReplaceableObject" class.
+
+This recipe is directly borrowed from Alembic documentation, see
+http://alembic.zzzcomputing.com/en/latest/cookbook.html#replaceable-objects
+
+"""
+
+from alembic.operations import Operations, MigrateOperation
+
+
+__all__ = ['ReplaceableObject']
+
+
+class ReplaceableObject(object):
+    def __init__(self, name, sqltext):
+        self.name = name
+        self.sqltext = sqltext
+
+
+class ReversibleOp(MigrateOperation):
+    def __init__(self, target):
+        self.target = target
+
+    @classmethod
+    def invoke_for_target(cls, operations, target):
+        op = cls(target)
+        return operations.invoke(op)
+
+    def reverse(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def _get_object_from_version(cls, operations, ident):
+        version, objname = ident.split(".")
+
+        module = operations.get_context().script.get_revision(version).module
+        obj = getattr(module, objname)
+        return obj
+
+    @classmethod
+    def replace(cls, operations, target, replaces=None, replace_with=None):
+
+        if replaces:
+            old_obj = cls._get_object_from_version(operations, replaces)
+            drop_old = cls(old_obj).reverse()
+            create_new = cls(target)
+        elif replace_with:
+            old_obj = cls._get_object_from_version(operations, replace_with)
+            drop_old = cls(target).reverse()
+            create_new = cls(old_obj)
+        else:
+            raise TypeError("replaces or replace_with is required")
+
+        operations.invoke(drop_old)
+        operations.invoke(create_new)
+
+
+@Operations.register_operation("create_view", "invoke_for_target")
+@Operations.register_operation("replace_view", "replace")
+class CreateViewOp(ReversibleOp):
+    def reverse(self):
+        return DropViewOp(self.target)
+
+
+@Operations.register_operation("drop_view", "invoke_for_target")
+class DropViewOp(ReversibleOp):
+    def reverse(self):
+        return CreateViewOp(self.view)
+
+
+@Operations.register_operation("create_sp", "invoke_for_target")
+@Operations.register_operation("replace_sp", "replace")
+class CreateSPOp(ReversibleOp):
+    def reverse(self):
+        return DropSPOp(self.target)
+
+
+@Operations.register_operation("drop_sp", "invoke_for_target")
+class DropSPOp(ReversibleOp):
+    def reverse(self):
+        return CreateSPOp(self.target)
+
+
+@Operations.implementation_for(CreateViewOp)
+def create_view(operations, operation):
+    operations.execute("CREATE VIEW %s AS %s" % (
+        operation.target.name,
+        operation.target.sqltext
+    ))
+
+
+@Operations.implementation_for(DropViewOp)
+def drop_view(operations, operation):
+    operations.execute("DROP VIEW %s" % operation.target.name)
+
+
+@Operations.implementation_for(CreateSPOp)
+def create_sp(operations, operation):
+    operations.execute(
+        "CREATE FUNCTION %s %s" % (
+            operation.target.name, operation.target.sqltext
+        )
+    )
+
+
+@Operations.implementation_for(DropSPOp)
+def drop_sp(operations, operation):
+    operations.execute("DROP FUNCTION %s" % operation.target.name)