Commit 8ce01ee6 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[Ngrams] List and Group management, SQL queries fixed.

parent 32c27036
Pipeline #114 canceled with stage
...@@ -229,17 +229,17 @@ queryAuthorsDoc = leftJoin5 queryNodeTable queryNodeNgramTable queryNgramsTable ...@@ -229,17 +229,17 @@ queryAuthorsDoc = leftJoin5 queryNodeTable queryNodeNgramTable queryNgramsTable
where where
cond12 :: (NodeNgramRead, NodeRead) -> Column PGBool cond12 :: (NodeNgramRead, NodeRead) -> Column PGBool
cond12 (nodeNgram, doc) = _node_id doc cond12 (nodeNgram, doc) = _node_id doc
.== nodeNgram_node_id nodeNgram .== _nn_node_id nodeNgram
cond23 :: (NgramsRead, (NodeNgramRead, NodeReadNull)) -> Column PGBool cond23 :: (NgramsRead, (NodeNgramRead, NodeReadNull)) -> Column PGBool
cond23 (ngrams, (nodeNgram, _)) = ngrams_id ngrams cond23 (ngrams, (nodeNgram, _)) = ngrams_id ngrams
.== nodeNgram_ngrams_id nodeNgram .== _nn_ngrams_id nodeNgram
cond34 :: (NodeNgramRead, (NgramsRead, (NodeNgramReadNull, NodeReadNull))) -> Column PGBool cond34 :: (NodeNgramRead, (NgramsRead, (NodeNgramReadNull, NodeReadNull))) -> Column PGBool
cond34 (nodeNgram2, (ngrams, (_,_)))= ngrams_id ngrams .== nodeNgram_ngrams_id nodeNgram2 cond34 (nodeNgram2, (ngrams, (_,_)))= ngrams_id ngrams .== _nn_ngrams_id nodeNgram2
cond45 :: (NodeRead, (NodeNgramRead, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull)))) -> Column PGBool cond45 :: (NodeRead, (NodeNgramRead, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull)))) -> Column PGBool
cond45 (contact, (nodeNgram2, (_, (_,_)))) = _node_id contact .== nodeNgram_node_id nodeNgram2 cond45 (contact, (nodeNgram2, (_, (_,_)))) = _node_id contact .== _nn_node_id nodeNgram2
------------------------------------------------------------------------ ------------------------------------------------------------------------
......
...@@ -24,7 +24,7 @@ import Control.Monad.IO.Class (liftIO) ...@@ -24,7 +24,7 @@ import Control.Monad.IO.Class (liftIO)
import Data.Map (Map, lookup) import Data.Map (Map, lookup)
import Data.Maybe (Maybe(..), catMaybes) import Data.Maybe (Maybe(..), catMaybes)
import Data.Text (Text, splitOn, intercalate) import Data.Text (Text, splitOn, intercalate)
import Data.Tuple.Extra (both, second) import Data.Tuple.Extra (both)
import Data.List (concat) import Data.List (concat)
import GHC.Show (Show) import GHC.Show (Show)
import Gargantext.Core.Types (NodePoly(..), ListType(..), listTypeId, Terms(..)) import Gargantext.Core.Types (NodePoly(..), ListType(..), listTypeId, Terms(..))
...@@ -37,7 +37,7 @@ import Gargantext.Text.Terms (extractTerms) ...@@ -37,7 +37,7 @@ import Gargantext.Text.Terms (extractTerms)
import Gargantext.Database.Node.Document.Add (add) import Gargantext.Database.Node.Document.Add (add)
import Gargantext.Database.Node.Document.Insert (insertDocuments, ReturnId(..), addUniqIdsDoc, addUniqIdsContact, ToDbData(..)) import Gargantext.Database.Node.Document.Insert (insertDocuments, ReturnId(..), addUniqIdsDoc, addUniqIdsContact, ToDbData(..))
import Gargantext.Database.Root (getRoot) import Gargantext.Database.Root (getRoot)
import Gargantext.Database.Schema.Ngrams (insertNgrams, Ngrams(..), NgramsT(..), NgramsIndexed(..), indexNgramsT, NgramsType(..), text2ngrams) import Gargantext.Database.Schema.Ngrams (insertNgrams, Ngrams(..), NgramsT(..), NgramsIndexed(..), indexNgramsT, NgramsType(..), text2ngrams, ngramsTypeId)
import Gargantext.Database.Schema.Node (mkRoot, mkCorpus, getOrMkList, mkGraph, mkDashboard, mkAnnuaire, getCorporaWithParentId, HasNodeError, NodeError(..), nodeError) import Gargantext.Database.Schema.Node (mkRoot, mkCorpus, getOrMkList, mkGraph, mkDashboard, mkAnnuaire, getCorporaWithParentId, HasNodeError, NodeError(..), nodeError)
import Gargantext.Database.Schema.NodeNgram (NodeNgramPoly(..), insertNodeNgrams) import Gargantext.Database.Schema.NodeNgram (NodeNgramPoly(..), insertNodeNgrams)
import Gargantext.Database.Schema.NodeNgramsNgrams (NodeNgramsNgramsPoly(..), insertNodeNgramsNgramsNew) import Gargantext.Database.Schema.NodeNgramsNgrams (NodeNgramsNgramsPoly(..), insertNodeNgramsNgramsNew)
...@@ -276,10 +276,7 @@ flowList uId cId ngs = do ...@@ -276,10 +276,7 @@ flowList uId cId ngs = do
_ <- insertGroups lId groupEd _ <- insertGroups lId groupEd
-- compute Candidate / Map -- compute Candidate / Map
let lists = ngrams2list ngs is <- insertLists lId $ ngrams2list ngs
-- printDebug "lists:" lists
is <- insertLists lId lists
printDebug "listNgrams inserted :" is printDebug "listNgrams inserted :" is
pure lId pure lId
...@@ -306,13 +303,12 @@ insertGroups lId ngrs = ...@@ -306,13 +303,12 @@ insertGroups lId ngrs =
------------------------------------------------------------------------ ------------------------------------------------------------------------
-- TODO: verify NgramsT lost here -- TODO: verify NgramsT lost here
ngrams2list :: Map (NgramsT NgramsIndexed) (Map NodeId Int) -> [(ListType,NgramsIndexed)] ngrams2list :: Map (NgramsT NgramsIndexed) (Map NodeId Int) -> [(ListType, (NgramsType,NgramsIndexed))]
ngrams2list = zip (repeat CandidateList) . map (\(NgramsT _lost_t ng) -> ng) . DM.keys ngrams2list = zip (repeat GraphList) . map (\(NgramsT ngt ng) -> (ngt, ng)) . DM.keys
-- | TODO: weight of the list could be a probability -- | TODO: weight of the list could be a probability
insertLists :: HasNodeError err => ListId -> [(ListType,NgramsIndexed)] -> Cmd err Int insertLists :: HasNodeError err => ListId -> [(ListType, (NgramsType, NgramsIndexed))] -> Cmd err Int
insertLists lId lngs = insertLists lId lngs = insertNodeNgrams [ NodeNgram lId (_ngramsId ng) (fromIntegral $ ngramsTypeId ngt) (fromIntegral $ listTypeId l) 1
insertNodeNgrams [ NodeNgram Nothing lId ngr (fromIntegral $ listTypeId l) (listTypeId l) | (l,(ngt, ng)) <- lngs
| (l,ngr) <- map (second _ngramsId) lngs
] ]
------------------------------------------------------------------------ ------------------------------------------------------------------------
...@@ -23,6 +23,7 @@ import Gargantext.Database.Schema.Ngrams ...@@ -23,6 +23,7 @@ import Gargantext.Database.Schema.Ngrams
import Gargantext.Database.Types.Node (NodeId, Node, NodePoly(..), Hyperdata) import Gargantext.Database.Types.Node (NodeId, Node, NodePoly(..), Hyperdata)
import Gargantext.Database.Utils (Cmd) import Gargantext.Database.Utils (Cmd)
import Gargantext.Database.Schema.NodeNgram import Gargantext.Database.Schema.NodeNgram
import Gargantext.Core.Types.Main (ListType(..), listTypeId)
toMaps :: Hyperdata a => (a -> Map (NgramsT Ngrams) Int) -> [Node a] -> Map (NgramsT Ngrams) (Map NodeId Int) toMaps :: Hyperdata a => (a -> Map (NgramsT Ngrams) Int) -> [Node a] -> Map (NgramsT Ngrams) (Map NodeId Int)
toMaps fun ns = mapNodeIdNgrams $ documentIdWithNgrams fun ns' toMaps fun ns = mapNodeIdNgrams $ documentIdWithNgrams fun ns'
...@@ -53,12 +54,10 @@ data DocumentIdWithNgrams a = ...@@ -53,12 +54,10 @@ data DocumentIdWithNgrams a =
, document_ngrams :: Map (NgramsT Ngrams) Int , document_ngrams :: Map (NgramsT Ngrams) Int
} deriving (Show) } deriving (Show)
-- | TODO for now, list Type is CandidateList, why ?
insertToNodeNgrams :: Map (NgramsT NgramsIndexed) (Map NodeId Int) -> Cmd err Int insertToNodeNgrams :: Map (NgramsT NgramsIndexed) (Map NodeId Int) -> Cmd err Int
insertToNodeNgrams m = insertNodeNgrams [ NodeNgram Nothing nId ((_ngramsId . _ngramsT ) ng) insertToNodeNgrams m = insertNodeNgrams [ NodeNgram nId ((_ngramsId . _ngramsT) ng) ((ngramsTypeId . _ngramsType) ng) (listTypeId CandidateList) (fromIntegral n)
(fromIntegral n) ((ngramsTypeId . _ngramsType) ng)
| (ng, nId2int) <- DM.toList m | (ng, nId2int) <- DM.toList m
, (nId, n) <- DM.toList nId2int , (nId, n) <- DM.toList nId2int
] ]
...@@ -37,7 +37,7 @@ import Database.PostgreSQL.Simple.SqlQQ (sql) ...@@ -37,7 +37,7 @@ import Database.PostgreSQL.Simple.SqlQQ (sql)
import Database.PostgreSQL.Simple.ToField (toField) import Database.PostgreSQL.Simple.ToField (toField)
import Database.PostgreSQL.Simple.ToRow (toRow) import Database.PostgreSQL.Simple.ToRow (toRow)
import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..)) import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
import Debug.Trace (trace) --import Debug.Trace (trace)
import GHC.Generics (Generic) import GHC.Generics (Generic)
import Gargantext.Core.Types -- (fromListTypeId, ListType, NodePoly(Node)) import Gargantext.Core.Types -- (fromListTypeId, ListType, NodePoly(Node))
import Gargantext.Database.Config (nodeTypeId,userMaster) import Gargantext.Database.Config (nodeTypeId,userMaster)
...@@ -239,7 +239,7 @@ getNgramsTableData :: NodeType -> NgramsType ...@@ -239,7 +239,7 @@ getNgramsTableData :: NodeType -> NgramsType
-> Limit -> Offset -> Limit -> Offset
-> Cmd err [NgramsTableData] -> Cmd err [NgramsTableData]
getNgramsTableData nodeT ngrmT (NgramsTableParam ul uc) (NgramsTableParam ml mc) limit_ offset_ = getNgramsTableData nodeT ngrmT (NgramsTableParam ul uc) (NgramsTableParam ml mc) limit_ offset_ =
trace ("Ngrams table params" <> show params) <$> -- trace ("Ngrams table params" <> show params) <$>
map (\(t,n,nt,w) -> NgramsTableData t n (fromListTypeId nt) w) <$> map (\(t,n,nt,w) -> NgramsTableData t n (fromListTypeId nt) w) <$>
runPGSQuery querySelectTableNgrams params runPGSQuery querySelectTableNgrams params
where where
...@@ -254,20 +254,21 @@ querySelectTableNgrams :: PGS.Query ...@@ -254,20 +254,21 @@ querySelectTableNgrams :: PGS.Query
querySelectTableNgrams = [sql| querySelectTableNgrams = [sql|
WITH tableUser AS ( WITH tableUser AS (
SELECT ngs.terms, ngs.n, list.ngrams_type, corp.weight FROM ngrams ngs SELECT ngs.terms, ngs.n, list.list_type, corp.weight FROM ngrams ngs
JOIN nodes_ngrams list ON list.ngram_id = ngs.id JOIN nodes_ngrams list ON list.ngrams_id = ngs.id
JOIN nodes_ngrams corp ON corp.ngram_id = ngs.id JOIN nodes_ngrams corp ON corp.ngrams_id = ngs.id
JOIN nodes_nodes nn ON nn.node2_id = corp.node_id
JOIN nodes n ON n.id = corp.node_id JOIN nodes n ON n.id = corp.node_id
WHERE list.node_id = ? -- User listId WHERE list.node_id = ? -- User listId
AND n.parent_id = ? -- User CorpusId or AnnuaireId AND nn.node1_id = ? -- User CorpusId or AnnuaireId
AND n.typename = ? -- both type of childs (Documents or Contacts) AND n.typename = ? -- both type of childs (Documents or Contacts)
AND corp.ngrams_type = ? -- both type of ngrams (Authors or Terms or...) AND corp.ngrams_type = ? -- both type of ngrams (Authors or Terms or...)
) )
, tableMaster AS ( , tableMaster AS (
SELECT ngs.terms, ngs.n, list.ngrams_type, corp.weight FROM ngrams ngs SELECT ngs.terms, ngs.n, list.list_type, corp.weight FROM ngrams ngs
JOIN nodes_ngrams list ON list.ngram_id = ngs.id JOIN nodes_ngrams list ON list.ngrams_id = ngs.id
JOIN nodes_ngrams corp ON corp.ngram_id = ngs.id JOIN nodes_ngrams corp ON corp.ngrams_id = ngs.id
JOIN nodes n ON n.id = corp.node_id JOIN nodes n ON n.id = corp.node_id
JOIN nodes_nodes nn ON nn.node2_id = n.id JOIN nodes_nodes nn ON nn.node2_id = n.id
...@@ -280,10 +281,11 @@ querySelectTableNgrams = [sql| ...@@ -280,10 +281,11 @@ querySelectTableNgrams = [sql|
SELECT COALESCE(tu.terms,tm.terms) AS terms SELECT COALESCE(tu.terms,tm.terms) AS terms
, COALESCE(tu.n,tm.n) AS n , COALESCE(tu.n,tm.n) AS n
, COALESCE(tu.ngrams_type,tm.ngrams_type) AS ngrams_type , COALESCE(tu.list_type,tm.list_type) AS ngrams_type
, SUM(COALESCE(tu.weight,tm.weight)) AS weight , SUM(COALESCE(tu.weight,tm.weight)) AS weight
FROM tableUser tu RIGHT JOIN tableMaster tm ON tu.terms = tm.terms FROM tableUser tu RIGHT JOIN tableMaster tm ON tu.terms = tm.terms
GROUP BY tu.terms,tm.terms,tu.n,tm.n,tu.ngrams_type,tm.ngrams_type GROUP BY tu.terms,tm.terms,tu.n,tm.n,tu.list_type,tm.list_type
ORDER BY 1,2
LIMIT ? LIMIT ?
OFFSET ?; OFFSET ?;
...@@ -320,6 +322,6 @@ querySelectNgramsGroup = [sql| ...@@ -320,6 +322,6 @@ querySelectNgramsGroup = [sql|
) )
SELECT COALESCE(gu.t1,gm.t1) AS ngram1_id SELECT COALESCE(gu.t1,gm.t1) AS ngram1_id
, COALESCE(gu.t2,gm.t2) AS ngram2_id , COALESCE(gu.t2,gm.t2) AS ngram2_id
FROM groupUser gu RIGHT JOIN groupMaster gm ON gu.t1 = gm.t1 FROM groupUser gu LEFT JOIN groupMaster gm ON gu.t1 = gm.t1
|] |]
...@@ -30,8 +30,9 @@ if Node is a List then it is listing (either Stop, Candidate or Map) ...@@ -30,8 +30,9 @@ if Node is a List then it is listing (either Stop, Candidate or Map)
-- TODO NodeNgrams -- TODO NodeNgrams
module Gargantext.Database.Schema.NodeNgram where module Gargantext.Database.Schema.NodeNgram where
import Data.ByteString (ByteString)
import Data.Text (Text) import Data.Text (Text)
import Control.Lens.TH (makeLensesWith, abbreviatedFields) import Control.Lens.TH (makeLenses)
import Control.Monad (void) import Control.Monad (void)
import Data.Profunctor.Product.TH (makeAdaptorAndInstance) import Data.Profunctor.Product.TH (makeAdaptorAndInstance)
import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..)) import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
...@@ -42,57 +43,58 @@ import Gargantext.Database.Types.Node (NodeId, ListId) ...@@ -42,57 +43,58 @@ import Gargantext.Database.Types.Node (NodeId, ListId)
import Gargantext.Database.Schema.Node (pgNodeId) import Gargantext.Database.Schema.Node (pgNodeId)
import Gargantext.Database.Schema.NodeNgramsNgrams (NgramsChild, NgramsParent, ngramsGroup, Action(..)) import Gargantext.Database.Schema.NodeNgramsNgrams (NgramsChild, NgramsParent, ngramsGroup, Action(..))
import Gargantext.Prelude import Gargantext.Prelude
import Gargantext.Database.Utils (formatPGSQuery)
import Opaleye import Opaleye
import qualified Database.PostgreSQL.Simple as PGS (Only(..)) import qualified Database.PostgreSQL.Simple as PGS (Only(..), Query)
-- | TODO : remove id -- | TODO : remove id
data NodeNgramPoly id node_id ngram_id weight ngrams_type data NodeNgramPoly node_id ngrams_id ngrams_type list_type weight
= NodeNgram { nodeNgram_id :: id = NodeNgram { _nn_node_id :: node_id
, nodeNgram_node_id :: node_id , _nn_ngrams_id :: ngrams_id
, nodeNgram_ngrams_id :: ngram_id , _nn_ngramsType :: ngrams_type
, nodeNgram_weight :: weight , _nn_listType :: list_type
, nodeNgram_type :: ngrams_type , _nn_weight :: weight
} deriving (Show) } deriving (Show)
type NodeNgramWrite = type NodeNgramWrite =
NodeNgramPoly NodeNgramPoly
(Maybe (Column PGInt4 ))
(Column PGInt4 ) (Column PGInt4 )
(Column PGInt4 ) (Column PGInt4 )
(Column PGFloat8)
(Column PGInt4 ) (Column PGInt4 )
(Column PGInt4 )
(Column PGFloat8)
type NodeNgramRead = type NodeNgramRead =
NodeNgramPoly NodeNgramPoly
(Column PGInt4 ) (Column PGInt4 )
(Column PGInt4 ) (Column PGInt4 )
(Column PGInt4 ) (Column PGInt4 )
(Column PGFloat8)
(Column PGInt4 ) (Column PGInt4 )
(Column PGFloat8)
type NodeNgramReadNull = type NodeNgramReadNull =
NodeNgramPoly NodeNgramPoly
(Column (Nullable PGInt4 )) (Column (Nullable PGInt4 ))
(Column (Nullable PGInt4 )) (Column (Nullable PGInt4 ))
(Column (Nullable PGInt4 )) (Column (Nullable PGInt4 ))
(Column (Nullable PGFloat8))
(Column (Nullable PGInt4 )) (Column (Nullable PGInt4 ))
(Column (Nullable PGFloat8))
type NodeNgram = type NodeNgram =
NodeNgramPoly (Maybe NodeId) NodeId Int Double Int NodeNgramPoly NodeId Int Int Int Double
$(makeAdaptorAndInstance "pNodeNgram" ''NodeNgramPoly) $(makeAdaptorAndInstance "pNodeNgram" ''NodeNgramPoly)
$(makeLensesWith abbreviatedFields ''NodeNgramPoly) makeLenses ''NodeNgramPoly
nodeNgramTable :: Table NodeNgramWrite NodeNgramRead nodeNgramTable :: Table NodeNgramWrite NodeNgramRead
nodeNgramTable = Table "nodes_ngrams" nodeNgramTable = Table "nodes_ngrams"
( pNodeNgram NodeNgram ( pNodeNgram NodeNgram
{ nodeNgram_id = optional "id" { _nn_node_id = required "node_id"
, nodeNgram_node_id = required "node_id" , _nn_ngrams_id = required "ngrams_id"
, nodeNgram_ngrams_id = required "ngram_id" , _nn_ngramsType = required "ngrams_type"
, nodeNgram_weight = required "weight" , _nn_listType = required "list_type"
, nodeNgram_type = required "ngrams_type" , _nn_weight = required "weight"
} }
) )
...@@ -101,9 +103,12 @@ queryNodeNgramTable = queryTable nodeNgramTable ...@@ -101,9 +103,12 @@ queryNodeNgramTable = queryTable nodeNgramTable
insertNodeNgrams :: [NodeNgram] -> Cmd err Int insertNodeNgrams :: [NodeNgram] -> Cmd err Int
insertNodeNgrams = insertNodeNgramW insertNodeNgrams = insertNodeNgramW
. map (\(NodeNgram _ n g w t) -> . map (\(NodeNgram n g ngt lt w) ->
NodeNgram Nothing (pgNodeId n) (pgInt4 g) NodeNgram (pgNodeId n)
(pgDouble w) (pgInt4 t) (pgInt4 g)
(pgInt4 ngt)
(pgInt4 lt)
(pgDouble w)
) )
insertNodeNgramW :: [NodeNgramWrite] -> Cmd err Int insertNodeNgramW :: [NodeNgramWrite] -> Cmd err Int
...@@ -123,14 +128,30 @@ updateNodeNgrams' [] = pure () ...@@ -123,14 +128,30 @@ updateNodeNgrams' [] = pure ()
updateNodeNgrams' input = void $ execPGSQuery updateQuery (PGS.Only $ Values fields input) updateNodeNgrams' input = void $ execPGSQuery updateQuery (PGS.Only $ Values fields input)
where where
fields = map (\t-> QualifiedIdentifier Nothing t) ["int4","text","int4"] fields = map (\t-> QualifiedIdentifier Nothing t) ["int4","text","int4"]
updateQuery = [sql| UPDATE nodes_ngrams as old SET
ngrams_type = new.typeList updateNodeNgrams'' :: [(ListId, NgramsText, ListTypeId)] -> Cmd err ByteString
from (?) as new(node_id,terms,typeList) updateNodeNgrams'' input = formatPGSQuery updateQuery (PGS.Only $ Values fields input)
JOIN ngrams ON ngrams.terms = new.terms where
WHERE old.node_id = new.node_id fields = map (\t-> QualifiedIdentifier Nothing t) ["int4","text","int4"]
AND old.ngram_id = ngrams.id;
updateQuery :: PGS.Query
updateQuery = [sql|
WITH new(node_id,terms,typeList) as (?)
INSERT into nodes_ngrams (node_id,ngrams_id,ngrams_type,list_type,weight)
SELECT node_id,ngrams.id,4,typeList,1 FROM new
JOIN ngrams ON ngrams.terms = new.terms
ON CONFLICT (node_id, ngrams_id, ngrams_type) DO
-- DO NOTHING-
UPDATE SET list_type = excluded.list_type
;
|] |]
data NodeNgramsUpdate = NodeNgramsUpdate data NodeNgramsUpdate = NodeNgramsUpdate
{ _nnu_lists_update :: [(ListId, NgramsText, ListTypeId)] { _nnu_lists_update :: [(ListId, NgramsText, ListTypeId)]
, _nnu_add_children :: [(ListId, NgramsParent, NgramsChild, Maybe Double)] , _nnu_add_children :: [(ListId, NgramsParent, NgramsChild, Maybe Double)]
......
...@@ -36,12 +36,14 @@ import Control.Lens (view) ...@@ -36,12 +36,14 @@ import Control.Lens (view)
import Control.Lens.TH (makeLensesWith, abbreviatedFields) import Control.Lens.TH (makeLensesWith, abbreviatedFields)
import Control.Monad (void) import Control.Monad (void)
import Control.Monad.IO.Class (liftIO) import Control.Monad.IO.Class (liftIO)
import Data.ByteString (ByteString)
import Data.Text (Text) import Data.Text (Text)
import Data.Maybe (Maybe) import Data.Maybe (Maybe)
import Data.Profunctor.Product.TH (makeAdaptorAndInstance) import Data.Profunctor.Product.TH (makeAdaptorAndInstance)
import Database.PostgreSQL.Simple.SqlQQ (sql) import Database.PostgreSQL.Simple.SqlQQ (sql)
import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..)) import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
import Gargantext.Database.Utils (Cmd, runOpaQuery, execPGSQuery, connection) import Debug.Trace (trace)
import Gargantext.Database.Utils (Cmd, runOpaQuery, execPGSQuery, connection, formatPGSQuery)
import Gargantext.Database.Types.Node (ListId) import Gargantext.Database.Types.Node (ListId)
import Gargantext.Database.Schema.Node (pgNodeId) import Gargantext.Database.Schema.Node (pgNodeId)
import Gargantext.Prelude import Gargantext.Prelude
...@@ -130,7 +132,7 @@ type NgramsChild = Text ...@@ -130,7 +132,7 @@ type NgramsChild = Text
ngramsGroup :: Action -> [(ListId, NgramsParent, NgramsChild, Maybe Double)] ngramsGroup :: Action -> [(ListId, NgramsParent, NgramsChild, Maybe Double)]
-> Cmd err () -> Cmd err ()
ngramsGroup _ [] = pure () ngramsGroup _ [] = pure ()
ngramsGroup action ngs = runNodeNgramsNgrams q ngs ngramsGroup action ngs = trace (show ngs) $ runNodeNgramsNgrams q ngs
where where
q = case action of q = case action of
Del -> queryDelNodeNgramsNgrams Del -> queryDelNodeNgramsNgrams
...@@ -144,6 +146,14 @@ runNodeNgramsNgrams q ngs = void $ execPGSQuery q (PGS.Only $ Values fields ngs' ...@@ -144,6 +146,14 @@ runNodeNgramsNgrams q ngs = void $ execPGSQuery q (PGS.Only $ Values fields ngs'
fields = map (\t -> QualifiedIdentifier Nothing t) fields = map (\t -> QualifiedIdentifier Nothing t)
["int4","text","text","float8"] ["int4","text","text","float8"]
runNodeNgramsNgramsDebug :: PGS.Query -> [(ListId, NgramsParent, NgramsChild, Maybe Double)] -> Cmd err ByteString
runNodeNgramsNgramsDebug q ngs = formatPGSQuery q (PGS.Only $ Values fields ngs')
where
ngs' = map (\(n,ng1,ng2,w) -> (n,ng1,ng2,maybe 0 identity w)) ngs
fields = map (\t -> QualifiedIdentifier Nothing t)
["int4","text","text","float8"]
-------------------------------------------------------------------- --------------------------------------------------------------------
-- TODO: on conflict update weight -- TODO: on conflict update weight
queryInsertNodeNgramsNgrams :: PGS.Query queryInsertNodeNgramsNgrams :: PGS.Query
...@@ -159,12 +169,16 @@ queryInsertNodeNgramsNgrams = [sql| ...@@ -159,12 +169,16 @@ queryInsertNodeNgramsNgrams = [sql|
queryDelNodeNgramsNgrams :: PGS.Query queryDelNodeNgramsNgrams :: PGS.Query
queryDelNodeNgramsNgrams = [sql| queryDelNodeNgramsNgrams = [sql|
WITH input(nId,ng1,ng2,w) AS (?) WITH input(nId,ng1,ng2,w) AS (?)
DELETE FROM nodes_ngrams_ngrams nnn DELETE FROM nodes_ngrams_ngrams AS nnn
JOIN ngrams ngrams1 ON ngrams.terms = ng1 USING ngrams AS ngrams1,
JOIN ngrams ngrams2 ON ngrams.terms = ng2 ngrams AS ngrams2,
WHERE nnn.node_id = input.nId input AS input
WHERE
ngrams1.terms = input.ng1
AND ngrams2.terms = input.ng2
AND nnn.node_id = input.nId
AND nnn.ngram1_id = ngrams1.id AND nnn.ngram1_id = ngrams1.id
AND nnn.ngram2_id = ngrams2.id AND nnn.ngram2_id = ngrams2.id
;) ;
|] |]
...@@ -53,12 +53,13 @@ ALTER TABLE public.ngrams OWNER TO gargantua; ...@@ -53,12 +53,13 @@ ALTER TABLE public.ngrams OWNER TO gargantua;
CREATE TABLE public.nodes_ngrams ( CREATE TABLE public.nodes_ngrams (
id SERIAL, id SERIAL,
node_id integer NOT NULL, node_id integer NOT NULL,
ngram_id integer NOT NULL, ngrams_id integer NOT NULL,
weight double precision,
ngrams_type integer, ngrams_type integer,
list_type integer,
weight double precision,
FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE,
FOREIGN KEY (ngram_id) REFERENCES public.ngrams(id) ON DELETE CASCADE, FOREIGN KEY (ngrams_id) REFERENCES public.ngrams(id) ON DELETE CASCADE,
PRIMARY KEY (node_id,ngram_id) PRIMARY KEY (node_id,ngrams_id)
); );
ALTER TABLE public.nodes_ngrams OWNER TO gargantua; ALTER TABLE public.nodes_ngrams OWNER TO gargantua;
...@@ -88,7 +89,6 @@ CREATE TABLE public.nodes_nodes ( ...@@ -88,7 +89,6 @@ CREATE TABLE public.nodes_nodes (
ALTER TABLE public.nodes_nodes OWNER TO gargantua; ALTER TABLE public.nodes_nodes OWNER TO gargantua;
-- INDEXES -- INDEXES
CREATE UNIQUE INDEX ON public.auth_user(username); CREATE UNIQUE INDEX ON public.auth_user(username);
...@@ -107,9 +107,9 @@ CREATE INDEX nodes_user_id_typename_parent_id_idx ON public.nodes USING btree (u ...@@ -107,9 +107,9 @@ CREATE INDEX nodes_user_id_typename_parent_id_idx ON public.nodes USING btree (u
CREATE UNIQUE INDEX ON public.ngrams(terms); CREATE UNIQUE INDEX ON public.ngrams(terms);
--CREATE UNIQUE INDEX ON public.ngrams(terms,n); --CREATE UNIQUE INDEX ON public.ngrams(terms,n);
CREATE INDEX nodes_ngrams_ngram_id_idx ON public.nodes_ngrams USING btree (ngram_id); CREATE INDEX nodes_ngrams_ngrams_id_idx ON public.nodes_ngrams USING btree (ngrams_id);
CREATE INDEX nodes_ngrams_ngrams_node_id_idx ON public.nodes_ngrams_ngrams USING btree (node_id); CREATE INDEX nodes_ngrams_ngrams_node_id_idx ON public.nodes_ngrams_ngrams USING btree (node_id);
CREATE UNIQUE INDEX ON public.nodes_ngrams USING btree (node_id,ngram_id); CREATE UNIQUE INDEX ON public.nodes_ngrams USING btree (node_id,ngrams_id,ngrams_type);
CREATE INDEX nodes_nodes_delete ON public.nodes_nodes USING btree (node1_id, node2_id, delete); CREATE INDEX nodes_nodes_delete ON public.nodes_nodes USING btree (node1_id, node2_id, delete);
CREATE UNIQUE INDEX nodes_nodes_node1_id_node2_id_idx ON public.nodes_nodes USING btree (node1_id, node2_id); CREATE UNIQUE INDEX nodes_nodes_node1_id_node2_id_idx ON public.nodes_nodes USING btree (node1_id, node2_id);
......
...@@ -104,7 +104,7 @@ queryInCorpusWithContacts cId q _ _ _ = proc () -> do ...@@ -104,7 +104,7 @@ queryInCorpusWithContacts cId q _ _ _ = proc () -> do
restrict -< (_ns_search docs) @@ (pgTSQuery $ unpack q ) restrict -< (_ns_search docs) @@ (pgTSQuery $ unpack q )
restrict -< (_ns_typename docs) .== (pgInt4 $ nodeTypeId NodeDocument) restrict -< (_ns_typename docs) .== (pgInt4 $ nodeTypeId NodeDocument)
restrict -< (nodeNode_node1_id corpusDoc) .== (toNullable $ pgNodeId cId) restrict -< (nodeNode_node1_id corpusDoc) .== (toNullable $ pgNodeId cId)
restrict -< (nodeNgram_type docNgrams) .== (toNullable $ pgInt4 $ ngramsTypeId Authors) restrict -< (_nn_listType docNgrams) .== (toNullable $ pgInt4 $ ngramsTypeId Authors)
restrict -< (_node_typename contacts) .== (toNullable $ pgInt4 $ nodeTypeId NodeContact) restrict -< (_node_typename contacts) .== (toNullable $ pgInt4 $ nodeTypeId NodeContact)
-- let contact_id = ifThenElse (isNull $ _node_id contacts) (toNullable $ pgInt4 0) (_node_id contacts) -- let contact_id = ifThenElse (isNull $ _node_id contacts) (toNullable $ pgInt4 0) (_node_id contacts)
returnA -< FacetPaired (_ns_id docs) (_ns_date docs) (_ns_hyperdata docs) (pgInt4 0) (Pair (_node_id contacts) (ngrams_terms ngrams')) returnA -< FacetPaired (_ns_id docs) (_ns_date docs) (_ns_hyperdata docs) (pgInt4 0) (Pair (_node_id contacts) (ngrams_terms ngrams'))
...@@ -113,16 +113,16 @@ joinInCorpusWithContacts :: O.Query (NodeSearchRead, (NodeNodeReadNull, (NodeNgr ...@@ -113,16 +113,16 @@ joinInCorpusWithContacts :: O.Query (NodeSearchRead, (NodeNodeReadNull, (NodeNgr
joinInCorpusWithContacts = leftJoin6 queryNodeTable queryNodeNgramTable queryNgramsTable queryNodeNgramTable queryNodeNodeTable queryNodeSearchTable cond12 cond23 cond34 cond45 cond56 joinInCorpusWithContacts = leftJoin6 queryNodeTable queryNodeNgramTable queryNgramsTable queryNodeNgramTable queryNodeNodeTable queryNodeSearchTable cond12 cond23 cond34 cond45 cond56
where where
cond12 :: (NodeNgramRead, NodeRead) -> Column PGBool cond12 :: (NodeNgramRead, NodeRead) -> Column PGBool
cond12 (ng3, n2) = _node_id n2 .== nodeNgram_node_id ng3 cond12 (ng3, n2) = _node_id n2 .== _nn_node_id ng3
--------- ---------
cond23 :: (NgramsRead, (NodeNgramRead, NodeReadNull)) -> Column PGBool cond23 :: (NgramsRead, (NodeNgramRead, NodeReadNull)) -> Column PGBool
cond23 (ng2, (nng2, _)) = nodeNgram_ngrams_id nng2 .== ngrams_id ng2 cond23 (ng2, (nng2, _)) = _nn_ngrams_id nng2 .== ngrams_id ng2
cond34 :: (NodeNgramRead, (NgramsRead, (NodeNgramReadNull, NodeReadNull))) -> Column PGBool cond34 :: (NodeNgramRead, (NgramsRead, (NodeNgramReadNull, NodeReadNull))) -> Column PGBool
cond34 (nng, (ng, (_,_))) = ngrams_id ng .== nodeNgram_ngrams_id nng cond34 (nng, (ng, (_,_))) = ngrams_id ng .== _nn_ngrams_id nng
cond45 :: (NodeNodeRead, (NodeNgramRead, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull)))) -> Column PGBool cond45 :: (NodeNodeRead, (NodeNgramRead, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull)))) -> Column PGBool
cond45 (nn, (nng, (_,(_,_)))) = nodeNgram_node_id nng .== nodeNode_node2_id nn cond45 (nn, (nng, (_,(_,_)))) = _nn_node_id nng .== nodeNode_node2_id nn
cond56 :: (NodeSearchRead, (NodeNodeRead, (NodeNgramReadNull, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull))))) -> Column PGBool cond56 :: (NodeSearchRead, (NodeNodeRead, (NodeNgramReadNull, (NgramsReadNull, (NodeNgramReadNull, NodeReadNull))))) -> Column PGBool
cond56 (n, (nn, (_,(_,(_,_))))) = _ns_id n .== nodeNode_node2_id nn cond56 (n, (nn, (_,(_,(_,_))))) = _ns_id n .== nodeNode_node2_id nn
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment