Commit 2487bd75 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[CONFIG] Database specific functions from Core folder to Database Folder.

parent 0655415e
......@@ -34,10 +34,12 @@ library:
- Gargantext.Core
- Gargantext.Core.Types
- Gargantext.Core.Types.Main
- Gargantext.Core.Types.Node
- Gargantext.Core.Utils.Prefix
- Gargantext.Database
- Gargantext.Database.Bashql
- Gargantext.Database.Node.Document.Import
- Gargantext.Database.Types.Node
- Gargantext.Database.User
- Gargantext.Prelude
- Gargantext.Text
- Gargantext.Text.Context
......
......@@ -38,7 +38,7 @@ import Servant
-- import Servant.Multipart
import Gargantext.Prelude
import Gargantext.Core.Types.Node
import Gargantext.Database.Types.Node
import Gargantext.Database.Node ( getNodesWithParentId
, getNode, getNodesWith
, deleteNode, deleteNodes)
......
......@@ -16,7 +16,7 @@ commentary with @some markup@.
{-# LANGUAGE DeriveGeneric #-}
module Gargantext.Core.Types ( module Gargantext.Core.Types.Main
, module Gargantext.Core.Types.Node
, module Gargantext.Database.Types.Node
, Term, Terms(..)
, TokenTag(..), POS(..), NER(..)
, Label, Stems
......@@ -31,7 +31,7 @@ import Data.Set (Set, empty)
import Data.Text (Text, unpack)
import Gargantext.Core.Types.Main
import Gargantext.Core.Types.Node
import Gargantext.Database.Types.Node
import Gargantext.Prelude
------------------------------------------------------------------------
......
......@@ -7,8 +7,6 @@ Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
Here is a longer description of this module, containing some
commentary with @some markup@.
-}
{-# OPTIONS_GHC -fno-warn-name-shadowing #-}
......@@ -22,13 +20,11 @@ commentary with @some markup@.
module Gargantext.Core.Types.Main where
------------------------------------------------------------------------
import Data.Maybe (fromMaybe)
import Data.Eq (Eq())
import Data.Monoid ((<>))
import Data.Text (Text, pack)
import Data.List (lookup)
import Data.Text (Text)
import Gargantext.Core.Types.Node
import Gargantext.Database.Types.Node
import Gargantext.Prelude
------------------------------------------------------------------------
......@@ -118,43 +114,6 @@ type Phylo = Node HyperdataPhylo
type Notebook = Node HyperdataNotebook
nodeTypes :: [(NodeType, NodeTypeId)]
nodeTypes = [ (NodeUser , 1)
, (Folder , 2)
, (NodeCorpus , 30)
, (Annuaire , 31)
, (Document , 4)
, (UserPage , 41)
--, (NodeSwap , 19)
------ Lists
-- , (StopList , 5)
-- , (GroupList , 6)
-- , (MainList , 7)
-- , (MapList ,  8)
---- Scores
, (Occurrences , 10)
-- , (Cooccurrences , 9)
--
-- , (Specclusion , 11)
-- , (Genclusion , 18)
-- , (Cvalue , 12)
--
-- , (TfidfCorpus , 13)
-- , (TfidfGlobal , 14)
--
-- , (TirankLocal , 16)
-- , (TirankGlobal , 17)
--
---- Node management
, (Favorites , 15)
--
]
--
nodeTypeId :: NodeType -> NodeTypeId
nodeTypeId tn = fromMaybe (panic $ pack $ "Typename " <> show tn <> " does not exist")
(lookup tn nodeTypes)
-- Temporary types to be removed
type ErrorMessage = Text
......
......@@ -49,6 +49,7 @@ data Phylo = Phylo { _phylo_Duration :: (Start, End)
type Start = POSIXTime
type End = POSIXTime
-- | Indexed Ngram
type Ngram = (NgramId, Text)
type NgramId = Int
......
{-|
Module : Gargantext.Database
Description : BASHQL to deal with Gargantext Database.
Description : Tools for Database
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
* BASHQL is a Domain Specific Language to deal with the Database
All Database related stuff here.
* BASHQL = functional (Bash * SQL)
* Which language to chose when working with a database ? To make it
simple, instead of all common Object Relational Mapping (ORM) [1]
strategy used nowadays inspired more by object logic than functional
logic, the semantics of BASHQL with focus on the function first.
* BASHQL focus on the function, i.e. use bash language function name,
and make it with SQL behind the scene. Then BASHQL is inspired more
by Bash language [2] than SQL and then follows its main commands as
specification and documentation.
* Main arguments:
1. Theoritical: database and FileSystems are each thought as a single
category, assumption based on theoretical work on databases by David Spivak [0].
2. Practical argument: basic bash commands are a daily practice among
developper community.
* How to help ?
1. Choose a command you like in Bash
2. Implement it in Haskell-SQL according to Gargantext Shema (Tree like
filesystem)
3. Translate it in BASHQL (follow previous implementations)
4. Make a pull request (enjoy the community)
* Implementation strategy: Functional adapations are made to the
gargantext languages options and SQL optimization are done continuously
during the project. For the Haskellish part, you may be inspired by
Turtle implementation written by Gabriel Gonzales [3] which shows how to
write Haskell bash translations.
* Semantics
- FileSystem is now a NodeSystem where each File is a Node in a Directed Graph (DG).
* References
[0] MIT Press has published "Category theory for the sciences". The book
can also be purchased on Amazon. Here are reviews by the MAA, by the
AMS, and by SIAM.
[1] https://en.wikipedia.org/wiki/Object-relational_mapping
[2] https://en.wikipedia.org/wiki/Bash_(Unix_shell)
[3] https://github.com/Gabriel439/Haskell-Turtle-Library
Target: just import this module and nothing else to work with
Gargantext's database.
-}
{-# LANGUAGE NoImplicitPrelude #-}
module Gargantext.Database ( module Gargantext.Database.Utils
, get
, ls , ls'
, home, home'
, post, post'
, del , del'
, tree, tree'
, postCorpus, postAnnuaire
, module Gargantext.Database.Bashql
, Connection
)
where
import Gargantext.Core.Types
import Gargantext.Database.Utils (connectGargandb)
import Gargantext.Database.Node
import Gargantext.Prelude
import Gargantext.Database.Bashql
import Database.PostgreSQL.Simple (Connection)
import Data.Text (Text, pack)
import Opaleye hiding (FromField)
import Data.Aeson
import Data.List (last, concat)
--type UserId = Int
--type NodeId = Int
-- List of NodeId
-- type PWD a = PWD UserId [a]
type PWD = [NodeId]
--data PWD' a = a | PWD' [a]
-- | TODO get Children or Node
get :: Connection -> PWD -> IO [Node Value]
get _ [] = pure []
get conn pwd = runQuery conn $ selectNodesWithParentID (last pwd)
-- | Home, need to filter with UserId
home :: Connection -> IO PWD
home c = map node_id <$> getNodesWithParentId c 0 Nothing
-- | ls == get Children
ls :: Connection -> PWD -> IO [Node Value]
ls = get
tree :: Connection -> PWD -> IO [Node Value]
tree c p = do
ns <- get c p
children <- mapM (\p' -> get c [p']) $ map node_id ns
pure $ ns <> (concat children)
-- | TODO
post :: Connection -> PWD -> [NodeWrite'] -> IO Int64
post _ [] _ = pure 0
post _ _ [] = pure 0
post c pth ns = mkNode c (last pth) ns
--postR :: Connection -> PWD -> [NodeWrite'] -> IO [Int]
--postR _ [] _ = pure [0]
--postR _ _ [] = pure [0]
--postR c pth ns = mkNodeR c (last pth) ns
--
--rm :: Connection -> PWD -> [NodeId] -> IO Int
--rm = del
del :: Connection -> [NodeId] -> IO Int
del _ [] = pure 0
del c ns = deleteNodes c ns
-- | TODO
--put :: Connection -> PWD -> [a] -> IO Int64
--put = undefined
-- | TODO
-- cd (Home UserId) | (Node NodeId)
-- cd Path
-- jump NodeId
-- touch Dir
--------------------------------------------------------------
-- Tests
--------------------------------------------------------------
home' :: IO PWD
home' = do
c <- connectGargandb "gargantext.ini"
home c
ls' :: IO [Node Value]
ls' = do
c <- connectGargandb "gargantext.ini"
h <- home c
ls c h
tree' :: IO [Node Value]
tree' = do
c <- connectGargandb "gargantext.ini"
h <- home c
tree c h
post' :: IO [Int]
post' = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' NodeCorpus (pack "Premier corpus") (toJSON (pack "{}"::Text)) [ Node' Document (pack "Doc1") (toJSON (pack "{}" :: Text)) []
, Node' Document (pack "Doc2") (toJSON (pack "{}" :: Text)) []
, Node' Document (pack "Doc3") (toJSON (pack "{}" :: Text)) []
]
)
type CorpusName = Text
-- |
-- myCorpus <- Prelude.map doc2hyperdataDocument <$> toDocs <$> snd <$> readCsv "doc/corpus_imt/Gargantext_Corpus_small.csv"
-- There is an error in the CSV parsing...
-- let myCorpus' = Prelude.filter (\n -> T.length (maybe "" identity (hyperdataDocument_title n)) > 30) myCorpus
postCorpus :: ToJSON a => CorpusName -> (a -> Text) -> [a] -> IO [Int]
postCorpus corpusName title ns = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' NodeCorpus corpusName (toJSON (pack "{}"::Text))
(map (\n -> Node' Document (title n) (toJSON n) []) ns)
)
-- |
-- import IMTClient as C
-- postAnnuaire "Annuaire IMT" (\n -> (maybe "" identity (C.prenom n)) <> " " <> (maybe "" identity (C.nom n))) (take 30 annuaire)
postAnnuaire :: ToJSON a => CorpusName -> (a -> Text) -> [a] -> IO [Int]
postAnnuaire corpusName title ns = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' Annuaire corpusName (toJSON (pack "{}"::Text))
(map (\n -> Node' UserPage (title n) (toJSON n) []) ns)
)
del' :: [NodeId] -> IO Int
del' ns = do
c <- connectGargandb "gargantext.ini"
del c ns
{-|
Module : Gargantext.Database.Bashql
Description : BASHQL to deal with Gargantext Database.
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
* BASHQL is a Domain Specific Language to deal with the Database
* BASHQL = functional (Bash * SQL)
* Which language to chose when working with a database ? To make it
simple, instead of all common Object Relational Mapping (ORM) [1]
strategy used nowadays inspired more by object logic than functional
logic, the semantics of BASHQL with focus on the function first.
* BASHQL focus on the function, i.e. use bash language function name,
and make it with SQL behind the scene. Then BASHQL is inspired more
by Bash language [2] than SQL and then follows its main commands as
specification and documentation.
* Main arguments:
1. Theoritical: database and FileSystems are each thought as a single
category, assumption based on theoretical work on databases by David Spivak [0].
2. Practical argument: basic bash commands are a daily practice among
developper community.
* How to help ?
1. Choose a command you like in Bash
2. Implement it in Haskell-SQL according to Gargantext Shema (Tree like
filesystem)
3. Translate it in BASHQL (follow previous implementations)
4. Make a pull request (enjoy the community)
* Implementation strategy: Functional adapations are made to the
gargantext languages options and SQL optimization are done continuously
during the project. For the Haskellish part, you may be inspired by
Turtle implementation written by Gabriel Gonzales [3] which shows how to
write Haskell bash translations.
* Semantics
- FileSystem is now a NodeSystem where each File is a Node in a Directed Graph (DG).
* References
[0] MIT Press has published "Category theory for the sciences". The book
can also be purchased on Amazon. Here are reviews by the MAA, by the
AMS, and by SIAM.
[1] https://en.wikipedia.org/wiki/Object-relational_mapping
[2] https://en.wikipedia.org/wiki/Bash_(Unix_shell)
[3] https://github.com/Gabriel439/Haskell-Turtle-Library
-}
{-# LANGUAGE NoImplicitPrelude #-}
module Gargantext.Database.Bashql ( get
, ls , ls'
, home, home'
, post, post'
, del , del'
, tree, tree'
, postCorpus, postAnnuaire
)
where
import Gargantext.Core.Types
import Gargantext.Database.Utils (connectGargandb)
import Gargantext.Database.Node
import Gargantext.Prelude
import Database.PostgreSQL.Simple (Connection)
import Data.Text (Text, pack)
import Opaleye hiding (FromField)
import Data.Aeson
import Data.List (last, concat)
--type UserId = Int
--type NodeId = Int
-- List of NodeId
-- type PWD a = PWD UserId [a]
type PWD = [NodeId]
--data PWD' a = a | PWD' [a]
-- | TODO get Children or Node
get :: Connection -> PWD -> IO [Node Value]
get _ [] = pure []
get conn pwd = runQuery conn $ selectNodesWithParentID (last pwd)
-- | Home, need to filter with UserId
home :: Connection -> IO PWD
home c = map node_id <$> getNodesWithParentId c 0 Nothing
-- | ls == get Children
ls :: Connection -> PWD -> IO [Node Value]
ls = get
tree :: Connection -> PWD -> IO [Node Value]
tree c p = do
ns <- get c p
children <- mapM (\p' -> get c [p']) $ map node_id ns
pure $ ns <> (concat children)
-- | TODO
post :: Connection -> PWD -> [NodeWrite'] -> IO Int64
post _ [] _ = pure 0
post _ _ [] = pure 0
post c pth ns = mkNode c (last pth) ns
--postR :: Connection -> PWD -> [NodeWrite'] -> IO [Int]
--postR _ [] _ = pure [0]
--postR _ _ [] = pure [0]
--postR c pth ns = mkNodeR c (last pth) ns
--
--rm :: Connection -> PWD -> [NodeId] -> IO Int
--rm = del
del :: Connection -> [NodeId] -> IO Int
del _ [] = pure 0
del c ns = deleteNodes c ns
-- | TODO
--put :: Connection -> PWD -> [a] -> IO Int64
--put = undefined
-- | TODO
-- cd (Home UserId) | (Node NodeId)
-- cd Path
-- jump NodeId
-- touch Dir
--------------------------------------------------------------
-- Tests
--------------------------------------------------------------
home' :: IO PWD
home' = do
c <- connectGargandb "gargantext.ini"
home c
ls' :: IO [Node Value]
ls' = do
c <- connectGargandb "gargantext.ini"
h <- home c
ls c h
tree' :: IO [Node Value]
tree' = do
c <- connectGargandb "gargantext.ini"
h <- home c
tree c h
post' :: IO [Int]
post' = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' NodeCorpus (pack "Premier corpus") (toJSON (pack "{}"::Text)) [ Node' Document (pack "Doc1") (toJSON (pack "{}" :: Text)) []
, Node' Document (pack "Doc2") (toJSON (pack "{}" :: Text)) []
, Node' Document (pack "Doc3") (toJSON (pack "{}" :: Text)) []
]
)
type CorpusName = Text
-- |
-- myCorpus <- Prelude.map doc2hyperdataDocument <$> toDocs <$> snd <$> readCsv "doc/corpus_imt/Gargantext_Corpus_small.csv"
-- There is an error in the CSV parsing...
-- let myCorpus' = Prelude.filter (\n -> T.length (maybe "" identity (hyperdataDocument_title n)) > 30) myCorpus
postCorpus :: ToJSON a => CorpusName -> (a -> Text) -> [a] -> IO [Int]
postCorpus corpusName title ns = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' NodeCorpus corpusName (toJSON (pack "{}"::Text))
(map (\n -> Node' Document (title n) (toJSON n) []) ns)
)
-- |
-- import IMTClient as C
-- postAnnuaire "Annuaire IMT" (\n -> (maybe "" identity (C.prenom n)) <> " " <> (maybe "" identity (C.nom n))) (take 30 annuaire)
postAnnuaire :: ToJSON a => CorpusName -> (a -> Text) -> [a] -> IO [Int]
postAnnuaire corpusName title ns = do
c <- connectGargandb "gargantext.ini"
pid <- last <$> home c
let uid = 1
postNode c uid pid ( Node' Annuaire corpusName (toJSON (pack "{}"::Text))
(map (\n -> Node' UserPage (title n) (toJSON n) []) ns)
)
del' :: [NodeId] -> IO Int
del' ns = do
c <- connectGargandb "gargantext.ini"
del c ns
{-|
Module : Gargantext.Database
Description : Tools for Database
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
All Database related stuff here.
Target: just import this module and nothing else to work with
Gargantext's database.
-}
{-# LANGUAGE NoImplicitPrelude #-}
module Gargantext.Database.Config
where
import Data.Text (pack)
import Data.Maybe (fromMaybe)
import Data.List (lookup)
import Gargantext.Database.Types.Node
import Gargantext.Prelude
-- | Nodes are typed in the database according to a specific ID
--
nodeTypes :: [(NodeType, NodeTypeId)]
nodeTypes = [ (NodeUser , 1)
, (Folder , 2)
, (NodeCorpus , 30)
, (Annuaire , 31)
, (Document , 4)
, (UserPage , 41)
--, (NodeSwap , 19)
------ Lists
-- , (StopList , 5)
-- , (GroupList , 6)
-- , (MainList , 7)
-- , (MapList ,  8)
---- Scores
, (Occurrences , 10)
-- , (Cooccurrences , 9)
--
-- , (Specclusion , 11)
-- , (Genclusion , 18)
-- , (Cvalue , 12)
--
-- , (TfidfCorpus , 13)
-- , (TfidfGlobal , 14)
--
-- , (TirankLocal , 16)
-- , (TirankGlobal , 17)
--
---- Node management
, (Favorites , 15)
--
]
--
nodeTypeId :: NodeType -> NodeTypeId
nodeTypeId tn = fromMaybe (panic $ pack $ "Typename " <> show tn <> " does not exist")
(lookup tn nodeTypes)
......@@ -50,12 +50,13 @@ import Test.QuickCheck.Arbitrary
import Test.QuickCheck (elements)
import Gargantext.Core.Types
import Gargantext.Core.Types.Node (NodeType)
import Gargantext.Database.Types.Node (NodeType)
import Gargantext.Core.Utils.Prefix (unPrefix)
import Gargantext.Database.NodeNode
import Gargantext.Database.NodeNodeNgram
import Gargantext.Database.Node
import Gargantext.Database.Queries
import Gargantext.Database.Config (nodeTypeId)
-- import Gargantext.Database.NodeNgram
------------------------------------------------------------------------
......
......@@ -34,8 +34,9 @@ import Database.PostgreSQL.Simple.FromField ( Conversion
import Prelude hiding (null, id, map, sum)
import Gargantext.Core.Types
import Gargantext.Core.Types.Node (NodeType)
import Gargantext.Database.Types.Node (NodeType)
import Gargantext.Database.Queries
import Gargantext.Database.Config (nodeTypeId)
import Gargantext.Prelude hiding (sum)
......
......@@ -72,11 +72,11 @@ import qualified Data.Digest.Pure.SHA as SHA (sha256, showDigest)
import qualified Data.ByteString.Lazy.Char8 as DC (pack)
import Gargantext (connectGargandb)
import Gargantext.Core.Types.Main (nodeTypeId)
import Gargantext.Core.Types.Node
import Gargantext.Database.Config (nodeTypeId)
import Gargantext.Database.Types.Node
-- FIXME : the import of Document constructor below does not work
-- import Gargantext.Core.Types.Node (Document)
--import Gargantext.Core.Types.Node (docExample, hyperdataDocument, HyperdataDocument(..)
-- import Gargantext.Database.Types.Node (Document)
--import Gargantext.Database.Types.Node (docExample, hyperdataDocument, HyperdataDocument(..)
-- , hyperdataDocument_uniqId
-- , hyperdataDocument_title
-- , hyperdataDocument_abstract
......
{-|
Module : Gargantext.Core.Types.Nodes
Description : Main Types of Nodes
Module : Gargantext.Database.Types.Nodes
Description : Main Types of Nodes in Database
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
......@@ -18,7 +18,7 @@ Portability : POSIX
{-# LANGUAGE TemplateHaskell #-}
-- {-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.Core.Types.Node where
module Gargantext.Database.Types.Node where
import Prelude (Enum, Bounded, minBound, maxBound)
......
{-|
Module : Gargantext.Database.user
Description :
Description : User Database management tools
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
Here is a longer description of this module, containing some
commentary with @some markup@.
Functions to deal with users, database side.
-}
{-# OPTIONS_GHC -fno-warn-name-shadowing #-}
......@@ -102,28 +101,36 @@ userTable = Table "auth_user" (pUser User { user_id = optional "id"
}
)
------------------------------------------------------------------
queryUserTable :: Query UserRead
queryUserTable = queryTable userTable
selectUsersLight :: Query UserRead
selectUsersLight = proc () -> do
row@(User i _p _ll _is _un _fn _ln _m _iff _ive _dj) <- queryUserTable -< ()
restrict -< i .== 1
--returnA -< User i p ll is un fn ln m iff ive dj
returnA -< row
------------------------------------------------------------------
-- | Select User with some parameters
-- Not optimized version
userWith :: (Eq a1, Foldable t) => (a -> a1) -> a1 -> t a -> Maybe a
userWith f t xs = find (\x -> f x == t) xs
-- | Select User with Username
userWithUsername :: Text -> [User] -> Maybe User
userWithUsername t xs = userWith user_username t xs
userWithId :: Int -> [User] -> Maybe User
userWithId t xs = userWith user_id t xs
userLightWithUsername :: Text -> [UserLight] -> Maybe UserLight
userLightWithUsername t xs = userWith userLight_username t xs
userLightWithId :: Int -> [UserLight] -> Maybe UserLight
userLightWithId t xs = userWith userLight_id t xs
instance QueryRunnerColumnDefault PGTimestamptz (Maybe UTCTime) where
queryRunnerColumnDefault = fieldQueryRunnerColumn
......@@ -133,3 +140,9 @@ users conn = runQuery conn queryUserTable
usersLight :: PGS.Connection -> IO [UserLight]
usersLight conn = map toUserLight <$> runQuery conn queryUserTable
type Username = Text
user :: PGS.Connection -> Username -> IO (Maybe UserLight)
user c u = userLightWithUsername u <$> usersLight c
......@@ -32,7 +32,7 @@ import Data.Vector (Vector)
import qualified Data.Vector as V
import Safe (tailMay)
import Gargantext.Core.Types.Node (HyperdataDocument(..))
import Gargantext.Database.Types.Node (HyperdataDocument(..))
import Gargantext.Text
import Gargantext.Text.Context
import Gargantext.Prelude hiding (length)
......
......@@ -30,7 +30,7 @@ import qualified Data.Map.Strict as M
import Gargantext.Database (Connection)
import Gargantext.Database.Node
import Gargantext.Core.Types.Node
import Gargantext.Database.Types.Node
import Gargantext.Core (Lang)
import Gargantext.Prelude
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment