[toml] implement full config, refactoring of types

Also, tests pass now, locally

gargantext.cabal

@@ -175,11 +175,7 @@ library
       Gargantext.Core.AsyncUpdates.Dispatcher.WebSocket
       Gargantext.Core.AsyncUpdates.Nanomsg
       Gargantext.Core.Config
-      Gargantext.Core.Config.CORS
-      Gargantext.Core.Config.Database
-      Gargantext.Core.Config.Frontend
       Gargantext.Core.Config.Mail
-      Gargantext.Core.Config.MicroServices
       Gargantext.Core.Config.NLP
       Gargantext.Core.Config.Types
       Gargantext.Core.Config.Utils

src/Gargantext/API.hs

@@ -52,9 +52,7 @@ import Gargantext.API.Routes.Named (API)
 import Gargantext.API.Routes.Named.EKG
 import Gargantext.API.Server.Named (server)
 import Gargantext.Core.AsyncUpdates.Constants qualified as AUConstants
-import Gargantext.Core.Config.CORS
-import Gargantext.Core.Config.MicroServices
-import Gargantext.Core.Config.Types (SettingsFile(..))
+import Gargantext.Core.Config.Types (CORSOrigin(..), CORSSettings, SettingsFile(..), corsAllowedOrigins, msProxyPort)
 import Gargantext.Database.Prelude qualified as DB
 import Gargantext.MicroServices.ReverseProxy (microServicesProxyApp)
 import Gargantext.Prelude hiding (putStrLn)

src/Gargantext/API/Admin/Settings.hs

@@ -33,8 +33,7 @@ import Gargantext.API.Prelude
 import Gargantext.Core.AsyncUpdates.CentralExchange qualified as CE
 import Gargantext.Core.AsyncUpdates.Dispatcher qualified as D
 import Gargantext.Core.Config (GargConfig(..), gc_js_job_timeout, gc_js_id_timeout)
-import Gargantext.Core.Config.Frontend qualified as Frontend
-import Gargantext.Core.Config.Types (SettingsFile(..))
+import Gargantext.Core.Config.Types (SettingsFile(..), _fc_cors, _fc_microservices)
 import Gargantext.Core.Config.Utils (readConfig)
 import Gargantext.Core.NLP (nlpServerMap)
 import Gargantext.Core.NodeStory
@@ -69,9 +68,9 @@ devSettings (JwkFile jwkFile) settingsFile = do
   gc@(GargConfig {}) <- readConfig settingsFile
   pure $ Settings
     { -- _corsSettings = _gargCorsSettings
-      _corsSettings = Frontend._fc_cors $ _gc_frontend_config gc
+      _corsSettings = _fc_cors $ _gc_frontend_config gc
     -- , _microservicesSettings = _gargMicroServicesSettings
-    , _microservicesSettings = Frontend._fc_microservices $ _gc_frontend_config gc
+    , _microservicesSettings = _fc_microservices $ _gc_frontend_config gc
     , _appPort = 3000
     , _logLevelLimit = LevelDebug
 --    , _dbServer = "localhost"

src/Gargantext/API/Admin/Types.hs

@@ -5,8 +5,7 @@ module Gargantext.API.Admin.Types where
 import Control.Lens
 import Control.Monad.Logger (LogLevel)
 import GHC.Enum
-import Gargantext.Core.Config.CORS
-import Gargantext.Core.Config.MicroServices
+import Gargantext.Core.Config.Types
 import Gargantext.Prelude
 import Servant.Auth.Server (JWTSettings, CookieSettings(..))
 import Servant.Client (BaseUrl)

src/Gargantext/Core/Config.hs

@@ -45,11 +45,9 @@ module Gargantext.Core.Config (
 
 import Data.Text as T
 import Database.PostgreSQL.Simple qualified as PSQL
-import Gargantext.Core.Config.Database (TOMLConnectInfo(..))
-import Gargantext.Core.Config.Frontend (FrontendConfig(..))
 import Gargantext.Core.Config.Mail (MailConfig)
-import Gargantext.Core.Config.MicroServices (MicroServicesSettings(..))
 import Gargantext.Core.Config.NLP (NLPConfig)
+import Gargantext.Core.Config.Types
 import Gargantext.Prelude
 import Servant.Client (BaseUrl(..), Scheme(Http), parseBaseUrl)
 import Toml.Schema
@@ -101,23 +99,28 @@ instance FromValue GargConfig where
     _gc_mail_config <- reqKey "mail"
     db_config <- reqKey "database"
     _gc_nlp_config <- reqKey "nlp"
+    secrets <- reqKey "secrets"
+    _gc_datafilepath <- reqKeyOf "paths" $ parseTableFromValue $ reqKey "data_filepath"
+    frames <- reqKeyOf "external" $ parseTableFromValue $ reqKey "frames"
+    jobs <- reqKey "jobs"
+    apis <- reqKey "apis"
     return $ GargConfig { _gc_backend_name = _fc_backend_name
                         , _gc_url          = _fc_url
                         , _gc_url_backend_api = _fc_url_backend_api
-                        , _gc_masteruser = ""
-                        , _gc_secretkey = ""
-                        , _gc_datafilepath = ""
-                        , _gc_frame_write_url = ""
-                        , _gc_frame_calc_url = ""
-                        , _gc_frame_visio_url = ""
-                        , _gc_frame_searx_url = ""
-                        , _gc_frame_istex_url = ""
-                        , _gc_max_docs_parsers = 0
-                        , _gc_max_docs_scrapers = 0
-                        , _gc_js_job_timeout = 0
-                        , _gc_js_id_timeout = 0
-                        , _gc_pubmed_api_key = ""
-                        , _gc_epo_api_url = ""
+                        , _gc_masteruser = _s_master_user secrets
+                        , _gc_secretkey = _s_secret_key secrets
+                        , _gc_datafilepath
+                        , _gc_frame_write_url = _f_write_url frames
+                        , _gc_frame_calc_url = _f_calc_url frames
+                        , _gc_frame_visio_url = _f_visio_url frames
+                        , _gc_frame_searx_url = _f_searx_url frames
+                        , _gc_frame_istex_url = _f_istex_url frames
+                        , _gc_max_docs_parsers = _jc_max_docs_parsers jobs
+                        , _gc_max_docs_scrapers = _jc_max_docs_scrapers jobs
+                        , _gc_js_job_timeout = _jc_js_job_timeout jobs
+                        , _gc_js_id_timeout = _jc_js_id_timeout jobs
+                        , _gc_pubmed_api_key = _ac_pubmed_api_key apis
+                        , _gc_epo_api_url = _ac_epo_api_url apis
                         , _gc_frontend_config
                         , _gc_mail_config
                         , _gc_database_config = unTOMLConnectInfo db_config

src/Gargantext/Core/Config/CORS.hs

-{--| Support in Gargantext for CORS (Cross-origin resource sharing) --}
-
-{-# LANGUAGE TemplateHaskell   #-}
-{-# LANGUAGE ViewPatterns      #-}
-
-module Gargantext.Core.Config.CORS where
-
-import Control.Monad.Fail (fail)
-import Data.Text qualified as T
-import Gargantext.Prelude
-import Servant.Client.Core (BaseUrl, parseBaseUrl)
-import Toml
-import Toml.Schema
-
-newtype CORSOrigin = CORSOrigin { _CORSOrigin :: BaseUrl }
-  deriving (Show, Eq)
-
-instance FromValue CORSOrigin where
-  fromValue (Toml.Text' _ t) =
-    case parseBaseUrl (T.unpack t) of
-      Nothing -> fail $ "Cannot parse base url for: " <> T.unpack t
-      Just b  -> return $ CORSOrigin b
-  fromValue _ = fail "Incorrect key type, expected Text"
-
-data CORSSettings =
-  CORSSettings {
-    _corsAllowedOrigins :: [CORSOrigin]
-  , _corsAllowedHosts   :: [CORSOrigin]
-  -- | If 'True', we will reuse the origin whitelist
-  -- as the allowed hosts as well. This allows, for example,
-  -- to connect from \"demo.gargantext.org\" to \"dev.sub.gargantext.org\"
-  -- and vice-versa.
-  , _corsUseOriginsForHosts :: !Bool
-  } deriving (Show, Eq)
-
-instance FromValue CORSSettings where
-  fromValue = parseTableFromValue $ do
-    _corsAllowedOrigins <- reqKey "allowed-origins"
-    let _corsAllowedHosts = mempty
-    _corsUseOriginsForHosts <- reqKey "use-origins-for-hosts"
-    return $ CORSSettings { .. }
-
--- corsOriginCodec :: TomlBiMap CORSOrigin AnyValue
--- corsOriginCodec = _Orig >>> _Text
---   where
---    _Orig :: BiMap e CORSOrigin T.Text
---    _Orig = iso (T.pack . showBaseUrl . _CORSOrigin)
---                (\(T.unpack -> u) -> CORSOrigin . fromMaybe (error $ "invalid origin: " <> u) . parseBaseUrl $ u)
-
--- corsSettingsCodec :: TomlCodec CORSSettings
--- corsSettingsCodec = CORSSettings
---      <$> Toml.arrayOf corsOriginCodec "allowed-origins" .= _corsAllowedOrigins
---      <*> pure mempty -- FIXME(adn) Currently we don't need to support this field.
---      <*> Toml.bool "use-origins-for-hosts" .= _corsUseOriginsForHosts
-
-makeLenses ''CORSSettings

src/Gargantext/Core/Config/Database.hs

-{-|
-Module      : Gargantext.Core.Config.Database
-Description :
-Copyright   : (c) CNRS, 2024-Present
-License     : AGPL + CECILL v3
-Maintainer  : team@gargantext.org
-Stability   : experimental
-Portability : POSIX
-
--}
-
-module Gargantext.Core.Config.Database (
-    TOMLConnectInfo(..)
-  )
-  where
-
-import Database.PostgreSQL.Simple qualified as PGS
-import Gargantext.Prelude
-import Toml.Schema
-
-newtype TOMLConnectInfo = TOMLConnectInfo { unTOMLConnectInfo :: PGS.ConnectInfo }
-instance FromValue TOMLConnectInfo where
-  fromValue = parseTableFromValue $ do
-    host <- reqKey "host"
-    port <- reqKey "port"
-    user <- reqKey "user"
-    password <- reqKey "pass"
-    db <- reqKey "name"
-    return $ TOMLConnectInfo $ PGS.ConnectInfo { PGS.connectHost = host
-                                               , PGS.connectPort = port
-                                               , PGS.connectUser = user
-                                               , PGS.connectPassword = password
-                                               , PGS.connectDatabase = db }
-
--- pgsCodec :: Toml.TomlCodec PGS.ConnectInfo
--- pgsCodec = PGS.ConnectInfo
---   <$> Toml.string "database.host" .= PGS.connectHost
---   <*> word16Toml "database.port" .= PGS.connectPort
---   <*> Toml.string "database.user" .= PGS.connectUser
---   <*> Toml.string "database.password" .= PGS.connectPassword
---   <*> Toml.string "database.name" .= PGS.connectDatabase
-  
-  -- ini <- readIniFile' fp
-  -- let val' key = unpack $ val ini "database" key
-  -- let dbPortRaw = val' "DB_PORT"
-  -- let dbPort = case (readMaybe dbPortRaw :: Maybe Word16) of
-  --       Nothing -> panicTrace $ "DB_PORT incorrect: " <> (pack dbPortRaw)
-  --       Just d  -> d
-
-  -- pure $ PGS.ConnectInfo { PGS.connectHost     = val' "DB_HOST"
-  --                        , PGS.connectPort     = dbPort
-  --                        , PGS.connectUser     = val' "DB_USER"
-  --                        , PGS.connectPassword = val' "DB_PASS"
-  --                        , PGS.connectDatabase = val' "DB_NAME"
-  --                        }

src/Gargantext/Core/Config/Frontend.hs

-{-|
-Module      : Gargantext.Core.Config.Frontend
-Description :
-Copyright   : (c) CNRS, 2024-Present
-License     : AGPL + CECILL v3
-Maintainer  : team@gargantext.org
-Stability   : experimental
-Portability : POSIX
-
--}
-
-{-# LANGUAGE TemplateHaskell #-}
-
-module Gargantext.Core.Config.Frontend (
-  -- * Types
-    FrontendConfig(..)
-  )
-  where
-
-import Gargantext.Core.Config.CORS (CORSSettings)
-import Gargantext.Core.Config.MicroServices (MicroServicesSettings)
-import Gargantext.Prelude
-import Toml.Schema
-
-
-data FrontendConfig =
-  FrontendConfig { _fc_url           :: !Text
-                 , _fc_backend_name  :: !Text
-                 , _fc_url_backend_api :: !Text
-                 , _fc_jwt_settings  :: !Text
-                 , _fc_cors          :: !CORSSettings
-                 , _fc_microservices :: !MicroServicesSettings
-               }
-  deriving (Generic, Show)
-
-instance FromValue FrontendConfig where
-  fromValue = parseTableFromValue $ do
-    _fc_url <- reqKey "url"
-    _fc_backend_name <- reqKey "backend_name"
-    _fc_url_backend_api <- reqKey "url_backend_api"
-    _fc_jwt_settings <- reqKey "jwt_settings"
-    _fc_cors <- reqKey "cors"
-    _fc_microservices <- reqKey "microservices"
-    return $ FrontendConfig { .. }

src/Gargantext/Core/Config/Mail.hs

@@ -70,7 +70,7 @@ data MailConfig = MailConfig { _mc_mail_host         :: !T.Text
 
 instance FromValue MailConfig where
   fromValue = parseTableFromValue $ do
-    _mc_mail_host <- reqKey "m-host"
+    _mc_mail_host <- reqKey "host"
     port <- reqKey "port" :: ParseTable l Int
     _mc_mail_user <- reqKey "user"
     _mc_mail_password <- reqKey "password"

src/Gargantext/Core/Config/MicroServices.hs

-{-|
-Module      : Gargantext.Core.Config.MicroServices
-Description :
-Copyright   : (c) CNRS, 2024-Present
-License     : AGPL + CECILL v3
-Maintainer  : team@gargantext.org
-Stability   : experimental
-Portability : POSIX
-
--}
-
-{-# LANGUAGE TemplateHaskell   #-}
-
-module Gargantext.Core.Config.MicroServices where
-
-import Control.Lens.TH
-import Gargantext.Prelude
-import Toml.Schema
-
-
-data MicroServicesSettings =
-  MicroServicesSettings {
-    -- | The port where the microservices proxy will be listening on.
-    _msProxyPort    :: !Int
-  , _msProxyEnabled :: !Bool
-  } deriving (Show, Eq)
-
-instance FromValue MicroServicesSettings where
-  fromValue = parseTableFromValue $ reqKeyOf "proxy" $ parseTableFromValue $ do
-    _msProxyPort <- reqKey "port"
-    _msProxyEnabled <- reqKey "enabled"
-    return $ MicroServicesSettings { .. }
-
-
-makeLenses ''MicroServicesSettings

src/Gargantext/Core/Config/Types.hs

@@ -10,9 +10,179 @@ Portability : POSIX
 TODO-SECURITY: Critical
 -}
 
-module Gargantext.Core.Config.Types where
+{-# LANGUAGE TemplateHaskell   #-}
+{-# LANGUAGE ViewPatterns      #-}
 
+module Gargantext.Core.Config.Types
+  ( SettingsFile(..)
+  , CORSOrigin(..)
+  , CORSSettings(..)
+  , MicroServicesSettings(..)
+  , TOMLConnectInfo(..)
+  , FramesConfig(..)
+  , FrontendConfig(..)
+  , JobsConfig(..)
+  , APIsConfig(..)
+  , SecretsConfig(..)
+
+  , corsUseOriginsForHosts
+  , corsAllowedOrigins
+  , corsAllowedHosts
+  , msProxyPort
+  , msProxyEnabled )
+where
+
+import Control.Monad.Fail (fail)
+import Data.Text qualified as T
+import Database.PostgreSQL.Simple qualified as PGS
 import Gargantext.Prelude
+import Servant.Client.Core (BaseUrl, parseBaseUrl)
+import Toml
+import Toml.Schema
+
+
+
 
 newtype SettingsFile = SettingsFile { _SettingsFile :: FilePath }
   deriving (Show, Eq, IsString)
+
+
+
+newtype CORSOrigin = CORSOrigin { _CORSOrigin :: BaseUrl }
+  deriving (Show, Eq)
+instance FromValue CORSOrigin where
+  fromValue (Toml.Text' _ t) =
+    case parseBaseUrl (T.unpack t) of
+      Nothing -> fail $ "Cannot parse base url for: " <> T.unpack t
+      Just b  -> return $ CORSOrigin b
+  fromValue _ = fail "Incorrect key type, expected Text"
+
+
+data CORSSettings =
+  CORSSettings {
+    _corsAllowedOrigins :: [CORSOrigin]
+  , _corsAllowedHosts   :: [CORSOrigin]
+  -- | If 'True', we will reuse the origin whitelist
+  -- as the allowed hosts as well. This allows, for example,
+  -- to connect from \"demo.gargantext.org\" to \"dev.sub.gargantext.org\"
+  -- and vice-versa.
+  , _corsUseOriginsForHosts :: !Bool
+  } deriving (Show, Eq)
+instance FromValue CORSSettings where
+  fromValue = parseTableFromValue $ do
+    _corsAllowedOrigins <- reqKey "allowed-origins"
+    let _corsAllowedHosts = mempty
+    _corsUseOriginsForHosts <- reqKey "use-origins-for-hosts"
+    return $ CORSSettings { .. }
+
+makeLenses ''CORSSettings
+
+
+data MicroServicesSettings =
+  MicroServicesSettings {
+    -- | The port where the microservices proxy will be listening on.
+    _msProxyPort    :: !Int
+  , _msProxyEnabled :: !Bool
+  } deriving (Show, Eq)
+
+instance FromValue MicroServicesSettings where
+  fromValue = parseTableFromValue $ reqKeyOf "proxy" $ parseTableFromValue $ do
+    _msProxyPort <- reqKey "port"
+    _msProxyEnabled <- reqKey "enabled"
+    return $ MicroServicesSettings { .. }
+
+
+makeLenses ''MicroServicesSettings
+
+
+
+newtype TOMLConnectInfo = TOMLConnectInfo { unTOMLConnectInfo :: PGS.ConnectInfo }
+instance FromValue TOMLConnectInfo where
+  fromValue = parseTableFromValue $ do
+    host <- reqKey "host"
+    port <- reqKey "port"
+    user <- reqKey "user"
+    password <- reqKey "pass"
+    db <- reqKey "name"
+    return $ TOMLConnectInfo $ PGS.ConnectInfo { PGS.connectHost = host
+                                               , PGS.connectPort = port
+                                               , PGS.connectUser = user
+                                               , PGS.connectPassword = password
+                                               , PGS.connectDatabase = db }
+
+
+data FramesConfig =
+  FramesConfig { _f_write_url :: Text
+               , _f_calc_url  :: Text
+               , _f_visio_url :: Text
+               , _f_searx_url :: Text
+               , _f_istex_url :: Text
+               }
+  deriving (Generic, Show)
+
+instance FromValue FramesConfig where
+  fromValue = parseTableFromValue $ do
+    _f_write_url <- reqKey "write_url"
+    _f_calc_url <- reqKey "calc_url"
+    _f_visio_url <- reqKey "visio_url"
+    _f_searx_url <- reqKey "searx_url"
+    _f_istex_url <- reqKey "istex_url"
+    return $ FramesConfig { .. }
+
+
+data FrontendConfig =
+  FrontendConfig { _fc_url           :: !Text
+                 , _fc_backend_name  :: !Text
+                 , _fc_url_backend_api :: !Text
+                 , _fc_jwt_settings  :: !Text
+                 , _fc_cors          :: !CORSSettings
+                 , _fc_microservices :: !MicroServicesSettings
+               }
+  deriving (Generic, Show)
+
+instance FromValue FrontendConfig where
+  fromValue = parseTableFromValue $ do
+    _fc_url <- reqKey "url"
+    _fc_backend_name <- reqKey "backend_name"
+    _fc_url_backend_api <- reqKey "url_backend_api"
+    _fc_jwt_settings <- reqKey "jwt_settings"
+    _fc_cors <- reqKey "cors"
+    _fc_microservices <- reqKey "microservices"
+    return $ FrontendConfig { .. }
+
+
+data SecretsConfig =
+  SecretsConfig { _s_master_user :: !Text
+                , _s_secret_key  :: !Text
+               }
+  deriving (Generic, Show)
+
+instance FromValue SecretsConfig where
+  fromValue = parseTableFromValue $ do
+    _s_master_user <- reqKey "master_user"
+    _s_secret_key  <- reqKey "secret_key"
+    return $ SecretsConfig { .. }
+
+
+data JobsConfig =
+  JobsConfig { _jc_max_docs_parsers  :: !Integer
+             , _jc_max_docs_scrapers :: !Integer
+             , _jc_js_job_timeout    :: !Integer
+             , _jc_js_id_timeout     :: !Integer }
+instance FromValue JobsConfig where
+  fromValue = parseTableFromValue $ do
+    _jc_max_docs_parsers <- reqKey "max_docs_parsers"
+    _jc_max_docs_scrapers <- reqKey "max_docs_scrapers"
+    _jc_js_job_timeout <- reqKey "js_job_timeout"
+    _jc_js_id_timeout <- reqKey "js_id_timeout"
+    return $ JobsConfig { .. }
+
+
+data APIsConfig =
+  APIsConfig { _ac_pubmed_api_key :: !Text
+             , _ac_epo_api_url    :: !Text }
+instance FromValue APIsConfig where
+  fromValue = parseTableFromValue $ do
+    _ac_pubmed_api_key <- reqKeyOf "pubmed" $ parseTableFromValue $ reqKey "api_key"
+    _ac_epo_api_url <- reqKeyOf "epo" $ parseTableFromValue $ reqKey "api_url"
+    return $ APIsConfig { .. }

src/Gargantext/Database/Action/Node.hs

@@ -25,7 +25,7 @@ import Data.Text qualified as T
 import Gargantext.API.Admin.Types (settings, _microservicesSettings, HasSettings)
 import Gargantext.Core
 import Gargantext.Core.Config (GargConfig(..), mkProxyUrl)
-import Gargantext.Core.Config.MicroServices (MicroServicesSettings(..))
+import Gargantext.Core.Config.Types (MicroServicesSettings(..))
 import Gargantext.Core.Types (Name)
 import Gargantext.Database.Admin.Types.Hyperdata
 import Gargantext.Database.Admin.Types.Hyperdata.Default