[REFACT] Move database indexes in models

gargantext/models/base.py

-from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint
+from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint, Index
 from sqlalchemy.orm import relationship
 from sqlalchemy.types import TypeDecorator, \
                              Integer, Float, Boolean, DateTime, String, Text

gargantext/models/hyperdata.py

 from gargantext.constants import INDEXED_HYPERDATA
 
-from .base import Base, Column, ForeignKey, TypeDecorator, \
+from .base import Base, Column, ForeignKey, TypeDecorator, Index, \
                   Integer, Double, DateTime, String, Text
 from .nodes import Node
 import datetime
@@ -65,6 +65,14 @@ class NodeHyperdata(Base):
         )
     """
     __tablename__ = 'nodes_hyperdata'
+    __table_args__ = (
+            Index('nodes_hyperdata_node_id_value_utc_idx', 'node_id', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_utc_idx', 'node_id', 'key', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_str_idx', 'node_id', 'key', 'value_str'),
+            Index('nodes_hyperdata_node_id_key_value_int_idx', 'node_id', 'key', 'value_int'),
+            Index('nodes_hyperdata_node_id_key_value_flt_idx', 'node_id', 'key', 'value_flt'),
+            Index('nodes_hyperdata_node_id_key_idx', 'node_id', 'key'))
+
     id        = Column( Integer, primary_key=True )
     node_id   = Column( Integer, ForeignKey(Node.id, ondelete='CASCADE'))
     key       = Column( HyperdataKey )

gargantext/models/ngrams.py

-from .base import Base, Column, ForeignKey, relationship, \
+from .base import Base, Column, ForeignKey, relationship, Index, \
                   Integer, Float, String
 from .nodes import Node
 
@@ -7,6 +7,9 @@ __all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
 
 class Ngram(Base):
     __tablename__ = 'ngrams'
+    __table_args__ = (
+            Index('ngrams_id_n_idx', 'id', 'n'),
+            Index('ngrams_n_idx', 'n'))
 
     id = Column(Integer, primary_key=True)
     terms = Column(String(255), unique=True)
@@ -21,6 +24,10 @@ class Ngram(Base):
 
 class NodeNgram(Base):
     __tablename__ = 'nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_node_id_ngram_id_idx', 'node_id', 'ngram_id'),
+            Index('nodes_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngram_id_idx', 'ngram_id'))
 
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
@@ -43,6 +50,9 @@ class NodeNodeNgram(Base):
     )
     """
     __tablename__ = 'nodes_nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_nodes_ngrams_node2_id_idx', 'node2_id'),
+            Index('nodes_nodes_ngrams_node1_id_idx', 'node1_id'))
 
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
@@ -70,6 +80,11 @@ class NodeNgramNgram(Base):
     )
     """
     __tablename__ = 'nodes_ngrams_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', 'node_id', 'ngram1_id', 'ngram2_id'),
+            Index('nodes_ngrams_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngrams_ngram1_id_idx', 'ngram1_id'),
+            Index('nodes_ngrams_ngrams_ngram2_id_idx', 'ngram2_id'))
 
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram1_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)

gargantext/models/nodes.py

@@ -4,7 +4,7 @@ from gargantext.constants import *
 
 from datetime import datetime
 
-from .base import Base, Column, ForeignKey, relationship, TypeDecorator, \
+from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, \
                   Integer, Float, String, DateTime, JSONB, \
                   MutableList, MutableDict
 from .users import User
@@ -53,6 +53,9 @@ class Node(Base):
     # Right: only user nodes are deleted.
     """
     __tablename__ = 'nodes'
+    __table_args__ = (
+            Index('nodes_user_id_typename_parent_id_idx', 'user_id', 'typename', 'parent_id'),
+            Index('nodes_hyperdata_idx', 'hyperdata'))
 
     id = Column(Integer, primary_key=True)
     typename = Column(NodeType, index=True)
@@ -240,6 +243,8 @@ class CorpusNode(Node):
 
 class NodeNode(Base):
     __tablename__ = 'nodes_nodes'
+    __table_args__ = (
+            Index('nodes_nodes_node1_id_node2_id_idx', 'node1_id', 'node2_id'),)
 
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)

install/gargamelle/psqlIndexes.sql

---                       ____ 
---                      / ___|
---                     | |  _ 
---                     | |_| |
---                      \____|arganTexT
-----------------------------------------------------------------------
---                  Gargantext optimization of Database             --
-----------------------------------------------------------------------
-
---> Manual optimization with indexes according to usages
--- Weakness and Strengths of indexes:
-    --> it can slow down the insertion(s)
-    --> it can speed up  the selection(s)
-
---> Conventions for this document:
-    --> indexes commented already have been created 
-    --> indexes not commented have not been created yet
-
-----------------------------------------------------------------------
--- Retrieve Nodes
-----------------------------------------------------------------------
-create INDEX on nodes (user_id, typename, parent_id) ;
-create INDEX on nodes_hyperdata (node_id, key);
-create INDEX on ngrams (id, n) ;
-create INDEX on ngrams (n) ;
-create INDEX on nodes_ngrams (node_id, ngram_id) ;
-create INDEX on nodes_ngrams (node_id) ;
-create INDEX on nodes_ngrams (ngram_id) ;
-create INDEX on nodes_ngrams_ngrams (node_id, ngram1_id, ngram2_id) ;
-
-create INDEX on nodes_ngrams_ngrams (node_id) ;
-create INDEX on nodes_ngrams_ngrams (ngram1_id) ;
-create INDEX on nodes_ngrams_ngrams (ngram2_id) ;
-
-----------------------------------------------------------------------
--- DELETE optimization of Nodes -- todo on dev
-create INDEX on nodes_nodes_ngrams (node1_id);
-create INDEX on nodes_nodes_ngrams (node2_id);
-
-create INDEX on nodes_nodes (node1_id, node2_id);
-
--- Maybe needed soon:
--- create INDEX on nodes_nodes_ngrams (node1_id, node2_id);
-----------------------------------------------------------------------
--- Analytics
-
-create INDEX on nodes_hyperdata (node_id,value_utc);  -- remove ?
-create INDEX on nodes_hyperdata (node_id,key,value_utc);
-create INDEX on nodes_hyperdata (node_id,key,value_int);
-create INDEX on nodes_hyperdata (node_id,key,value_flt);
-create INDEX on nodes_hyperdata (node_id,key,value_str);
-
-----------------------------------------------------------------------
-----------------------------------------------------------------------
-
-create index on nodes using GIN (hyperdata);
-----------------------------------------------------------------------