[GarGraph] Database triggers / inserts.

devops/postgres/create

 #!/bin/bash
 
-sudo su postgres
+# sudo su postgres
+
+# postgresql://$USER:$PW@localhost/$DB
 
 PW="password"
 DB="gargandbV5"
 USER="gargantua"
 
-psql -c "CREATE USER \"${USER}\"
+psql -c "CREATE USER \"${USER}\""
 psql -c "ALTER USER \"${USER}\" with PASSWORD \"${PW}\""
 
 psql -c "DROP DATABASE IF EXISTS \"${DB}\""
 createdb "${DB}"
 psql "${DB}" < schema.sql
 
-psql -c "ALTER DATABASE \"${DB}\" OWNER to \"${USER}\" ;"
+psql -c "ALTER DATABASE \"${DB}\" OWNER to \"${USER}\""
 
 
 

devops/postgres/schema.sql

@@ -80,6 +80,7 @@ ALTER TABLE public.ngrams OWNER TO gargantua;
 --ALTER TABLE public.nodes_ngrams_ngrams OWNER TO gargantua;
 
 ---------------------------------------------------------------
+-- TODO nodes_nodes(node1_id int, node2_id int, edge_type int , weight real)
 CREATE TABLE public.nodes_nodes (
     node1_id integer NOT NULL REFERENCES public.nodes(id) ON DELETE CASCADE,
     node2_id integer NOT NULL REFERENCES public.nodes(id) ON DELETE CASCADE,

src/Gargantext/API/Ngrams.hs

@@ -1153,8 +1153,6 @@ getTableNgramsDoc dId tabType listId limit_ offset listType minSize maxSize orde
 
 
 
-
-
 apiNgramsTableCorpus :: ( RepoCmdM env err m
                         , HasNodeError err
                         , HasInvalidError err

src/Gargantext/Database/Config.hs

@@ -94,4 +94,9 @@ nodeTypes = [ (n, nodeTypeId n) | n <- allNodeTypes ]
 
 fromNodeTypeId :: NodeTypeId -> NodeType
 fromNodeTypeId tId = fromMaybe (panic $ pack $ "Type Id " <> show tId <> " does not exist")
-                            (lookup tId nodeTypeInv)
+                               (lookup tId nodeTypeInv)
+
+
+
+
+

src/Gargantext/Database/Flow.hs

@@ -67,6 +67,7 @@ import Gargantext.Database.Schema.User (getUser, UserLight(..))
 import Gargantext.Database.TextSearch (searchInDatabase)
 import Gargantext.Database.Types.Node -- (HyperdataDocument(..), NodeType(..), NodeId, UserId, ListId, CorpusId, RootId, MasterCorpusId, MasterUserId)
 import Gargantext.Database.Utils (Cmd, CmdM)
+import Gargantext.Database.Triggers
 import Gargantext.Ext.IMT (toSchoolName)
 import Gargantext.Ext.IMTUser (deserialiseImtUsersFromFile)
 import Gargantext.Prelude
@@ -223,14 +224,16 @@ flowCorpusUser l userName corpusName ctype ids = do
   -- TODO: check if present already, ignore
   _ <- Doc.add userCorpusId ids
   tId <- mkNode NodeTexts userCorpusId userId
-  
+
   printDebug "Node Text Id" tId
 
   -- User List Flow
   --{-
-  (_masterUserId, _masterRootId, masterCorpusId) <- getOrMkRootWithCorpus userMaster (Left "") ctype
-  ngs         <- buildNgramsLists l 2 3 (StopSize 3) userCorpusId masterCorpusId
-  userListId  <- flowList userId userCorpusId ngs
+  (masterUserId, _masterRootId, masterCorpusId) <- getOrMkRootWithCorpus userMaster (Left "") ctype
+  ngs        <- buildNgramsLists l 2 3 (StopSize 3) userCorpusId masterCorpusId
+  userListId <- flowList userId userCorpusId ngs
+  mastListId <- getOrMkList masterCorpusId masterUserId
+  _ <- insertOccsUpdates userCorpusId mastListId
   printDebug "userListId" userListId
   -- User Graph Flow
   _ <- mkDashboard userCorpusId userId

src/Gargantext/Database/Flow/Pairing.hs

@@ -37,12 +37,20 @@ import Gargantext.Database.Schema.Ngrams -- (NgramsType(..))
 import Gargantext.Database.Node.Contact -- (HyperdataContact(..))
 import Gargantext.Database.Flow.Utils
 import Gargantext.Database.Utils (Cmd, runPGSQuery)
-import Gargantext.Database.Types.Node (AnnuaireId, CorpusId, ListId)
+import Gargantext.Database.Types.Node (AnnuaireId, CorpusId, ListId{-, DocId, ContactId-})
 import Gargantext.Database.Node.Children (getAllContacts)
 
 -- TODO mv this type in Types Main
 type Terms = Text
 
+{-
+pairing'' :: (CorpusId, CorpusId) -> (DocId -> DocId)
+pairing'' = undefined
+
+pairing' :: (CorpusId, AnnuaireId) -> (DocId -> ContactId)
+pairing' = undefined
+-}
+
 -- | TODO : add paring policy as parameter
 pairing :: AnnuaireId
         -> CorpusId

src/Gargantext/Database/Flow/Utils.hs

@@ -25,12 +25,17 @@ import Gargantext.Database.Utils (Cmd)
 import Gargantext.Database.Schema.NodeNodeNgrams
 import Gargantext.Database.Types.Node
 
-toMaps :: Hyperdata a => (a -> Map (NgramsT Ngrams) Int) -> [Node a] -> Map (NgramsT Ngrams) (Map NodeId Int)
+toMaps :: Hyperdata a
+       => (a -> Map (NgramsT Ngrams) Int)
+       -> [Node a]
+       -> Map (NgramsT Ngrams) (Map NodeId Int)
 toMaps fun ns = mapNodeIdNgrams $ documentIdWithNgrams fun ns'
   where
     ns' = map (\(Node nId _ _ _ _ _ json) -> DocumentWithId nId json) ns
 
-mapNodeIdNgrams :: Hyperdata a => [DocumentIdWithNgrams a] -> Map (NgramsT Ngrams) (Map NodeId Int)
+mapNodeIdNgrams :: Hyperdata a
+                => [DocumentIdWithNgrams a]
+                -> Map (NgramsT Ngrams) (Map NodeId Int)
 mapNodeIdNgrams ds = DM.map (DM.fromListWith (+)) $ DM.fromListWith (<>) xs
   where
     xs  = [(ng, [(nId, i)]) | (nId, n2i') <- n2i ds, (ng, i) <- DM.toList n2i']
@@ -72,14 +77,17 @@ data DocNgrams = DocNgrams { dn_doc_id :: DocId
 insertDocNgramsOn :: CorpusId
                   -> [DocNgrams]
                   -> Cmd err Int
-insertDocNgramsOn cId dn = insertNodeNodeNgrams $ (map (docNgrams2nodeNodeNgrams cId) dn)
+insertDocNgramsOn cId dn =
+  insertNodeNodeNgrams
+  $ (map (docNgrams2nodeNodeNgrams cId) dn)
 
 insertDocNgrams :: CorpusId
                 -> Map NgramsIndexed (Map NgramsType (Map NodeId Int))
                 -> Cmd err Int
-insertDocNgrams cId m = insertDocNgramsOn cId [ DocNgrams n (_ngramsId ng) (ngramsTypeId t) (fromIntegral i)
-                                        | (ng, t2n2i) <- DM.toList m
-                                        , (t,  n2i)   <- DM.toList t2n2i
-                                        , (n,  i)     <- DM.toList n2i
-                                        ]
+insertDocNgrams cId m =
+  insertDocNgramsOn cId [ DocNgrams n (_ngramsId ng) (ngramsTypeId t) (fromIntegral i)
+                          | (ng, t2n2i) <- DM.toList m
+                          , (t,  n2i)   <- DM.toList t2n2i
+                          , (n,  i)     <- DM.toList n2i
+                        ]
 

src/Gargantext/Database/Metrics/NgramsByNode.hs

@@ -68,21 +68,19 @@ getTficf' u m nt f = do
   pure $ toTficfData (countNodesByNgramsWith f u')
                      (countNodesByNgramsWith f m')
 
---{-
 getTficfWith :: UserCorpusId -> MasterCorpusId -> [ListId]
            -> NgramsType -> Map Text (Maybe Text)
            -> Cmd err (Map Text (Double, Set Text))
 getTficfWith u m ls nt mtxt = do
   u' <- getNodesByNgramsOnlyUser   u ls nt (Map.keys mtxt)
   m' <- getNodesByNgramsMaster     u m
-  
+
   let f x = case Map.lookup x mtxt of
         Nothing -> x
         Just x' -> maybe x identity x'
-  
+
   pure $ toTficfData (countNodesByNgramsWith f u')
                      (countNodesByNgramsWith f m')
---}
 
 
 type Context = (Double, Map Text (Double, Set Text))
@@ -121,7 +119,8 @@ groupNodesByNgramsWith f m =
                         $ toList m
 
 ------------------------------------------------------------------------
-getNodesByNgramsUser :: CorpusId -> NgramsType
+getNodesByNgramsUser :: CorpusId
+                     -> NgramsType
                      -> Cmd err (Map Text (Set NodeId))
 getNodesByNgramsUser cId nt =
   fromListWith (<>) <$> map (\(n,t) -> (t, Set.singleton n))
@@ -141,7 +140,6 @@ getNodesByNgramsUser cId nt =
 
       queryNgramsByNodeUser :: DPS.Query
       queryNgramsByNodeUser = [sql|
-
         SELECT nng.node2_id, ng.terms FROM node_node_ngrams nng
           JOIN ngrams ng      ON nng.ngrams_id = ng.id
           JOIN nodes_nodes nn ON nn.node2_id   = nng.node2_id
@@ -157,13 +155,19 @@ getNodesByNgramsUser cId nt =
         |]
 ------------------------------------------------------------------------
 -- TODO add groups
-getOccByNgramsOnlyFast :: CorpusId -> NgramsType -> [Text]
+getOccByNgramsOnlyFast :: CorpusId
+                       -> NgramsType
+                       -> [Text]
                        -> Cmd err (Map Text Int)
 getOccByNgramsOnlyFast cId nt ngs =
   fromListWith (+) <$> selectNgramsOccurrencesOnlyByNodeUser cId nt ngs
 
 -- just slower than getOccByNgramsOnlyFast
-getOccByNgramsOnlySlow :: NodeType -> CorpusId -> [ListId] -> NgramsType -> [Text]
+getOccByNgramsOnlySlow :: NodeType
+                       -> CorpusId
+                       -> [ListId]
+                       -> NgramsType
+                       -> [Text]
                        -> Cmd err (Map Text Int)
 getOccByNgramsOnlySlow t cId ls nt ngs =
   Map.map Set.size <$> getScore' t cId ls nt ngs
@@ -172,7 +176,10 @@ getOccByNgramsOnlySlow t cId ls nt ngs =
       getScore' NodeDocument = getNgramsByDocOnlyUser
       getScore' _            = getNodesByNgramsOnlyUser
 
-getOccByNgramsOnlySafe :: CorpusId -> [ListId] -> NgramsType -> [Text]
+getOccByNgramsOnlySafe :: CorpusId
+                       -> [ListId]
+                       -> NgramsType
+                       -> [Text]
                        -> Cmd err (Map Text Int)
 getOccByNgramsOnlySafe cId ls nt ngs = do
   printDebug "getOccByNgramsOnlySafe" (cId, nt, length ngs)
@@ -200,7 +207,23 @@ selectNgramsOccurrencesOnlyByNodeUser cId nt tms =
 -- equivalent ngrams intersections are not empty)
 queryNgramsOccurrencesOnlyByNodeUser :: DPS.Query
 queryNgramsOccurrencesOnlyByNodeUser = [sql|
+  WITH input_rows(terms) AS (?)
+  SELECT ng.terms, COUNT(nng.node2_id) FROM node_node_ngrams nng
+    JOIN ngrams ng      ON nng.ngrams_id = ng.id
+    JOIN input_rows  ir ON ir.terms      = ng.terms
+    JOIN nodes_nodes nn ON nn.node2_id   = nng.node2_id
+    JOIN nodes  n       ON nn.node2_id   = n.id
+    WHERE nn.node1_id     = ? -- CorpusId
+      AND n.typename      = ? -- NodeTypeId
+      AND nng.ngrams_type = ? -- NgramsTypeId
+      AND nn.category     > 0
+      GROUP BY nng.node2_id, ng.terms
+  |]
 
+
+
+queryNgramsOccurrencesOnlyByNodeUser' :: DPS.Query
+queryNgramsOccurrencesOnlyByNodeUser' = [sql|
   WITH input_rows(terms) AS (?)
   SELECT ng.terms, COUNT(nng.node2_id) FROM node_node_ngrams nng
     JOIN ngrams ng      ON nng.ngrams_id = ng.id
@@ -214,11 +237,16 @@ queryNgramsOccurrencesOnlyByNodeUser = [sql|
       GROUP BY nng.node2_id, ng.terms
   |]
 
+
+
+
 getNodesByNgramsOnlyUser :: NodeId -> [ListId] -> NgramsType -> [Text]
                          -> Cmd err (Map Text (Set NodeId))
-getNodesByNgramsOnlyUser cId ls nt ngs = Map.unionsWith (<>)
-                                    . map (fromListWith (<>) . map (second Set.singleton))
-                                   <$> mapM (selectNgramsOnlyByNodeUser cId ls nt) (splitEvery 1000 ngs)
+getNodesByNgramsOnlyUser cId ls nt ngs =
+  Map.unionsWith (<>)
+   . map (fromListWith (<>) . map (second Set.singleton))
+  <$> mapM (selectNgramsOnlyByNodeUser cId ls nt)
+           (splitEvery 1000 ngs)
 
 selectNgramsOnlyByNodeUser :: CorpusId -> [ListId] -> NgramsType -> [Text]
                            -> Cmd err [(Text, NodeId)]
@@ -235,7 +263,6 @@ selectNgramsOnlyByNodeUser cId ls nt tms =
 
 queryNgramsOnlyByNodeUser :: DPS.Query
 queryNgramsOnlyByNodeUser = [sql|
-
   WITH input_rows(terms) AS (?),
        input_list(id)    AS (?)
   SELECT ng.terms, nng.node2_id FROM node_node_ngrams nng
@@ -252,13 +279,12 @@ queryNgramsOnlyByNodeUser = [sql|
   |]
 
 
-
 getNgramsByDocOnlyUser :: NodeId -> [ListId] -> NgramsType -> [Text]
                          -> Cmd err (Map Text (Set NodeId))
-getNgramsByDocOnlyUser cId ls nt ngs = Map.unionsWith (<>)
-                                    . map (fromListWith (<>) . map (second Set.singleton))
-                                   <$> mapM (selectNgramsOnlyByDocUser cId ls nt) (splitEvery 1000 ngs)
-
+getNgramsByDocOnlyUser cId ls nt ngs =
+  Map.unionsWith (<>)
+  . map (fromListWith (<>) . map (second Set.singleton))
+  <$> mapM (selectNgramsOnlyByDocUser cId ls nt) (splitEvery 1000 ngs)
 
 
 selectNgramsOnlyByDocUser :: DocId -> [ListId] -> NgramsType -> [Text]
@@ -275,7 +301,6 @@ selectNgramsOnlyByDocUser dId ls nt tms =
 
 queryNgramsOnlyByDocUser :: DPS.Query
 queryNgramsOnlyByDocUser = [sql|
-
   WITH input_rows(terms) AS (?),
        input_list(id)    AS (?)
   SELECT ng.terms, nng.node2_id FROM node_node_ngrams nng
@@ -287,7 +312,6 @@ queryNgramsOnlyByDocUser = [sql|
       GROUP BY ng.terms, nng.node2_id
   |]
 
-
 ------------------------------------------------------------------------
 -- | TODO filter by language, database, any social field
 getNodesByNgramsMaster :: UserCorpusId -> MasterCorpusId -> Cmd err (Map Text (Set NodeId))
@@ -316,37 +340,36 @@ selectNgramsByNodeMaster n ucId mcId p = runPGSQuery
 -- | TODO fix node_node_ngrams relation
 queryNgramsByNodeMaster' :: DPS.Query
 queryNgramsByNodeMaster' = [sql|
+  WITH nodesByNgramsUser AS (
+
+  SELECT n.id, ng.terms FROM nodes n
+    JOIN nodes_nodes  nn  ON n.id = nn.node2_id
+    JOIN node_node_ngrams nng ON nng.node2_id   = n.id
+    JOIN ngrams       ng  ON nng.ngrams_id = ng.id
+    WHERE nn.node1_id     = ?   -- UserCorpusId
+      -- AND n.typename   = ?  -- NodeTypeId
+      AND nng.ngrams_type = ? -- NgramsTypeId
+      AND nn.category > 0
+      AND node_pos(n.id,?) >= ?
+      AND node_pos(n.id,?) <  ?
+    GROUP BY n.id, ng.terms
 
-WITH nodesByNgramsUser AS (
-
-SELECT n.id, ng.terms FROM nodes n
-  JOIN nodes_nodes  nn  ON n.id = nn.node2_id
-  JOIN node_node_ngrams nng ON nng.node2_id   = n.id
-  JOIN ngrams       ng  ON nng.ngrams_id = ng.id
-  WHERE nn.node1_id     = ?   -- UserCorpusId
-    -- AND n.typename   = ?  -- NodeTypeId
-    AND nng.ngrams_type = ? -- NgramsTypeId
-    AND nn.category > 0
-    AND node_pos(n.id,?) >= ?
-    AND node_pos(n.id,?) <  ?
-  GROUP BY n.id, ng.terms
-
-  ),
+    ),
 
-nodesByNgramsMaster AS (
+  nodesByNgramsMaster AS (
 
-SELECT n.id, ng.terms FROM nodes n TABLESAMPLE SYSTEM_ROWS(?)
-  JOIN node_node_ngrams nng  ON n.id  = nng.node2_id
-  JOIN ngrams       ng   ON ng.id = nng.ngrams_id
+  SELECT n.id, ng.terms FROM nodes n TABLESAMPLE SYSTEM_ROWS(?)
+    JOIN node_node_ngrams nng  ON n.id  = nng.node2_id
+    JOIN ngrams       ng   ON ng.id = nng.ngrams_id
 
-  WHERE n.parent_id  = ?     -- Master Corpus NodeTypeId
-    AND n.typename   = ?     -- NodeTypeId
-    AND nng.ngrams_type = ? -- NgramsTypeId
-  GROUP BY n.id, ng.terms
-  )
+    WHERE n.parent_id  = ?     -- Master Corpus NodeTypeId
+      AND n.typename   = ?     -- NodeTypeId
+      AND nng.ngrams_type = ? -- NgramsTypeId
+    GROUP BY n.id, ng.terms
+    )
 
-SELECT m.id, m.terms FROM nodesByNgramsMaster m
-  RIGHT JOIN nodesByNgramsUser u ON u.id = m.id
+  SELECT m.id, m.terms FROM nodesByNgramsMaster m
+    RIGHT JOIN nodesByNgramsUser u ON u.id = m.id
   |]
 
 

src/Gargantext/Database/Ngrams.hs

@@ -28,10 +28,10 @@ import Gargantext.Prelude
 import Opaleye
 import Control.Arrow (returnA)
 
-selectNgramsByDoc :: [CorpusId] -> DocId -> NgramsType -> Cmd err [Text]
-selectNgramsByDoc cIds dId nt = runOpaQuery (query cIds dId nt)
+selectNgramsByDoc :: [ListId] -> DocId -> NgramsType -> Cmd err [Text]
+selectNgramsByDoc lIds dId nt = runOpaQuery (query lIds dId nt)
   where
-    
+
     join :: Query (NgramsRead, NodeNodeNgramsReadNull)
     join = leftJoin queryNgramsTable queryNodeNodeNgramsTable on1
       where
@@ -42,7 +42,7 @@ selectNgramsByDoc cIds dId nt = runOpaQuery (query cIds dId nt)
       restrict -< foldl (\b cId -> ((toNullable $ pgNodeId cId) .== nnng^.nnng_node1_id) .|| b) (pgBool True) cIds'
       restrict -< (toNullable $ pgNodeId dId')    .== nnng^.nnng_node2_id
       restrict -< (toNullable $ pgNgramsType nt') .== nnng^.nnng_ngramsType
-      returnA -< ng^.ngrams_terms
+      returnA  -< ng^.ngrams_terms
 
 
 postNgrams :: CorpusId -> DocId -> [Text] -> Cmd err Int

src/Gargantext/Database/Schema/Node.hs

@@ -373,17 +373,17 @@ type JSONB = QueryRunnerColumnDefault PGJsonb
 
 getNode :: JSONB a => NodeId -> proxy a -> Cmd err (Node a)
 getNode nId _ = do
-    fromMaybe (error $ "Node does node exist: " <> show nId) . headMay
+    fromMaybe (error $ "Node does not exist: " <> show nId) . headMay
              <$> runOpaQuery (limit 1 $ selectNode (pgNodeId nId))
 
 getNodePhylo :: NodeId -> Cmd err (Node HyperdataPhylo)
 getNodePhylo nId = do
-    fromMaybe (error $ "Node does node exist: " <> show nId) . headMay
+    fromMaybe (error $ "Node Phylo does not exist: " <> show nId) . headMay
              <$> runOpaQuery (limit 1 $ selectNode (pgNodeId nId))
 
 
 getNode' :: NodeId -> Cmd err (Node Value)
-getNode' nId = fromMaybe (error $ "Node does node exist: " <> show nId) . headMay
+getNode' nId = fromMaybe (error $ "Node does not exist: " <> show nId) . headMay
              <$> runOpaQuery (limit 1 $ selectNode (pgNodeId nId))
 
 

src/Gargantext/Database/Tree.hs

@@ -108,7 +108,7 @@ dbTree rootId = map (\(nId, tId, pId, n) -> DbTreeNode nId tId pId n)
           FROM nodes AS c
 
           INNER JOIN tree AS s ON c.parent_id = s.id
---          WHERE c.typename IN (2,20,21,22,3,5,30,31,40,7,9,90)
+           WHERE c.typename IN (2,20,21,22,3,5,30,31,40,7,9,90)
         )
     SELECT * from tree;
     |] (Only rootId)