Commit 63f1a17b authored by Alfredo Di Napoli's avatar Alfredo Di Napoli

Initial work on Toml

parent 9c80d56c
...@@ -305,7 +305,6 @@ constraints: any.Cabal ==3.8.1.0, ...@@ -305,7 +305,6 @@ constraints: any.Cabal ==3.8.1.0,
any.indexed-profunctors ==0.1.1.1, any.indexed-profunctors ==0.1.1.1,
any.indexed-traversable ==0.1.3, any.indexed-traversable ==0.1.3,
any.indexed-traversable-instances ==0.1.1.2, any.indexed-traversable-instances ==0.1.1.2,
any.ini ==0.4.2,
any.insert-ordered-containers ==0.2.5.3, any.insert-ordered-containers ==0.2.5.3,
any.integer-gmp ==1.1, any.integer-gmp ==1.1,
any.integer-logarithms ==1.0.3.1, any.integer-logarithms ==1.0.3.1,
......
...@@ -49,8 +49,10 @@ data-files: ...@@ -49,8 +49,10 @@ data-files:
test-data/phylo/phylo2dot2json.golden.json test-data/phylo/phylo2dot2json.golden.json
test-data/stemming/lancaster.txt test-data/stemming/lancaster.txt
test-data/test_config.ini test-data/test_config.ini
test-data/test_config.toml
gargantext-cors-settings.toml gargantext-cors-settings.toml
.clippy.dhall .clippy.dhall
gargantext.toml_toModify
-- common options -- common options
-- https://vrom911.github.io/blog/common-stanzas -- https://vrom911.github.io/blog/common-stanzas
...@@ -138,6 +140,7 @@ library ...@@ -138,6 +140,7 @@ library
Gargantext.API.Prelude Gargantext.API.Prelude
Gargantext.API.Routes Gargantext.API.Routes
Gargantext.Core Gargantext.Core
Gargantext.Core.Config
Gargantext.Core.Mail.Types Gargantext.Core.Mail.Types
Gargantext.Core.Methods.Similarities Gargantext.Core.Methods.Similarities
Gargantext.Core.Methods.Similarities.Conditional Gargantext.Core.Methods.Similarities.Conditional
...@@ -530,7 +533,6 @@ library ...@@ -530,7 +533,6 @@ library
, ihaskell >= 0.11.0.0 , ihaskell >= 0.11.0.0
-- necessary for ihaskell to build -- necessary for ihaskell to build
, ipython-kernel >= 0.11.0.0 , ipython-kernel >= 0.11.0.0
, ini ^>= 0.4.1
, insert-ordered-containers ^>= 0.2.5.1 , insert-ordered-containers ^>= 0.2.5.1
, iso639 , iso639
, jose ^>= 0.8.4 , jose ^>= 0.8.4
...@@ -683,7 +685,6 @@ executable gargantext-cli ...@@ -683,7 +685,6 @@ executable gargantext-cli
, full-text-search ^>= 0.2.1.4 , full-text-search ^>= 0.2.1.4
, gargantext , gargantext
, gargantext-prelude , gargantext-prelude
, ini ^>= 0.4.1
, optparse-generic ^>= 1.4.7 , optparse-generic ^>= 1.4.7
, protolude ^>= 0.3.3 , protolude ^>= 0.3.3
, split ^>= 0.2.3.4 , split ^>= 0.2.3.4
...@@ -797,7 +798,6 @@ executable gargantext-phylo ...@@ -797,7 +798,6 @@ executable gargantext-phylo
, extra ^>= 1.7.9 , extra ^>= 1.7.9
, gargantext , gargantext
, gargantext-prelude , gargantext-prelude
, ini ^>= 0.4.1
, optparse-generic ^>= 1.4.7 , optparse-generic ^>= 1.4.7
, parallel ^>= 3.2.2.0 , parallel ^>= 3.2.2.0
, split ^>= 0.2.3.4 , split ^>= 0.2.3.4
...@@ -822,7 +822,6 @@ executable gargantext-server ...@@ -822,7 +822,6 @@ executable gargantext-server
, full-text-search ^>= 0.2.1.4 , full-text-search ^>= 0.2.1.4
, gargantext , gargantext
, gargantext-prelude , gargantext-prelude
, ini ^>= 0.4.1
, optparse-generic ^>= 1.4.7 , optparse-generic ^>= 1.4.7
, postgresql-simple ^>= 0.6.4 , postgresql-simple ^>= 0.6.4
, text ^>= 1.2.4.1 , text ^>= 1.2.4.1
...@@ -881,6 +880,7 @@ test-suite garg-test-tasty ...@@ -881,6 +880,7 @@ test-suite garg-test-tasty
Test.Offline.JSON Test.Offline.JSON
Test.Offline.Phylo Test.Offline.Phylo
Test.Offline.Stemming.Lancaster Test.Offline.Stemming.Lancaster
Test.Offline.TomlConfig
Test.Parsers.Date Test.Parsers.Date
Test.Parsers.Types Test.Parsers.Types
Test.Parsers.WOS Test.Parsers.WOS
......
title = "Gargantext Toml configuration file"
[gargantext]
# Main url serving the FrontEnd
URL = "http://localhost"
# The instance name
BACKEND_NAME = "localhost"
# Main API url serving the BackEnd
URL_BACKEND_API = "http://localhost:8008/api/v1.0"
# Needed to instantiate the first users and first data
MASTER_USER = "gargantua"
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "PASSWORD_TO_CHANGE"
# Data path to local files
DATA_FILEPATH = "FILEPATH_TO_CHANGE"
# Data path to local files (do not use quotes)
REPO_FILEPATH = "FILEPATH_TO_CHANGE"
PUBMED_API_KEY = "ENTER_PUBMED_API_KEY"
# [external]
# FRAMES (i.e. iframe sources used in various places on the frontend)
#FRAME_WRITE_URL = http://write.frame.gargantext.org/
FRAME_WRITE_URL = "URL_TO_CHANGE"
#FRAME_CALC_URL = http://calc.frame.gargantext.org/
FRAME_CALC_URL = "URL_TO_CHANGE"
FRAME_VISIO_URL = "URL_TO_CHANGE"
FRAME_SEARX_URL = "URL_TO_CHANGE"
FRAME_ISTEX_URL = "URL_TO_CHANGE"
MAX_DOCS_PARSERS = 1000000
MAX_DOCS_SCRAPERS = 10000
# Specific Services
EPO_API_URL = ""
# in seconds
JS_JOB_TIMEOUT = 1800
JS_ID_TIMEOUT = 1800
[server]
# Server config (TODO connect in ReaderMonad)
ALLOWED_ORIGIN = "http://localhost"
ALLOWED_ORIGIN_PORT = 8008
ALLOWED_HOST = "localhost"
ALLOWED_HOST_PORT = 3000
JWT_SETTINGS = "TODO"
[network]
# Emails From address (sent by smtp)
MAIL = "username@gargantext.org"
HOST = "localhost"
# if remote smtp host
# HOST_USER = user
# HOST_password = password
[database]
# PostgreSQL access
host = "127.0.0.1"
port = 5432
name = "gargandbV5"
user = "gargantua"
password = "PASSWORD_TO_CHANGE"
[logs]
LOG_FILE = "/var/log/gargantext/backend.log"
LOG_LEVEL = "LevelDebug"
LOG_FORMATTER = "verbose"
[mail]
MAIL_PORT = 25
MAIL_HOST = "localhost"
MAIL_USER = "gargantext"
MAIL_PASSWORD = ""
MAIL_FROM = ""
# NoAuth | Normal | SSL | TLS | STARTTLS
MAIL_LOGIN_TYPE = "Normal"
[nlp]
# Possible choices (see Gargantext.Core.NLP):
# - spacy:// (for http:// Spacy)
# - spacys:// (for https:// Spacy)
# - corenlp:// (for http:// CoreNLP)
# - corenlps:// (for https:// CoreNLP)
# - johnsnow:// (for http:// JohnSnow)
# - johnsnows:// (for https:// JohnSnow)
EN = "corenlp://localhost:9000"
FR = "spacy://localhost:8001"
All = "corenlp://localhost:9000"
module Gargantext.Core.Config (
readTomlConfigFile
, readTomlConfigFileFrom
) where
import Paths_gargantext
import Prelude
import Toml
readTomlConfigFile :: TomlCodec a -> IO (Either [TomlDecodeError] a)
readTomlConfigFile codec = getDataFileName "gargantext.toml" >>= Toml.decodeFileEither codec
readTomlConfigFileFrom :: TomlCodec a -> FilePath -> IO (Either [TomlDecodeError] a)
readTomlConfigFileFrom codec fp = getDataFileName fp >>= Toml.decodeFileEither codec
...@@ -14,6 +14,7 @@ Portability : POSIX ...@@ -14,6 +14,7 @@ Portability : POSIX
module Gargantext.Database.Prelude where module Gargantext.Database.Prelude where
import Control.Arrow ((>>>))
import Control.Exception (throw) import Control.Exception (throw)
import Control.Lens (Getter, view) import Control.Lens (Getter, view)
import Control.Monad.Random ( MonadRandom ) import Control.Monad.Random ( MonadRandom )
...@@ -23,20 +24,21 @@ import Data.ByteString qualified as DB ...@@ -23,20 +24,21 @@ import Data.ByteString qualified as DB
import Data.List qualified as DL import Data.List qualified as DL
import Data.Pool (Pool, withResource) import Data.Pool (Pool, withResource)
import Data.Profunctor.Product.Default (Default) import Data.Profunctor.Product.Default (Default)
import Data.Text (pack, unpack)
import Database.PostgreSQL.Simple (Connection, connect) import Database.PostgreSQL.Simple (Connection, connect)
import Database.PostgreSQL.Simple qualified as PGS import Database.PostgreSQL.Simple qualified as PGS
import Database.PostgreSQL.Simple.FromField ( Conversion, ResultError(ConversionFailed), fromField, returnError) import Database.PostgreSQL.Simple.FromField ( Conversion, ResultError(ConversionFailed), fromField, returnError)
import Database.PostgreSQL.Simple.Internal (Field) import Database.PostgreSQL.Simple.Internal (Field)
import Database.PostgreSQL.Simple.Types (Query(..)) import Database.PostgreSQL.Simple.Types (Query(..))
import Gargantext.Core.Config (readTomlConfigFileFrom)
import Gargantext.Core.Mail.Types (HasMail) import Gargantext.Core.Mail.Types (HasMail)
import Gargantext.Core.NLP (HasNLPServer) import Gargantext.Core.NLP (HasNLPServer)
import Gargantext.Prelude import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(), readIniFile', val) import Gargantext.Prelude.Config (GargConfig())
import Opaleye (Unpackspec, showSql, FromFields, Select, runSelect, SqlJsonb, DefaultFromField, toFields, matchMaybe, MaybeFields) import Opaleye (Unpackspec, showSql, FromFields, Select, runSelect, SqlJsonb, DefaultFromField, toFields, matchMaybe, MaybeFields)
import Opaleye.Aggregate (countRows) import Opaleye.Aggregate (countRows)
import Opaleye.Internal.Constant qualified import Opaleye.Internal.Constant qualified
import Opaleye.Internal.Operators qualified import Opaleye.Internal.Operators qualified
import Toml
------------------------------------------------------- -------------------------------------------------------
class HasConnectionPool env where class HasConnectionPool env where
...@@ -179,22 +181,30 @@ runPGSQuery_ q = mkCmd $ \conn -> catch (PGS.query_ conn q) printError ...@@ -179,22 +181,30 @@ runPGSQuery_ q = mkCmd $ \conn -> catch (PGS.query_ conn q) printError
execPGSQuery :: PGS.ToRow a => PGS.Query -> a -> DBCmd err Int64 execPGSQuery :: PGS.ToRow a => PGS.Query -> a -> DBCmd err Int64
execPGSQuery q a = mkCmd $ \conn -> PGS.execute conn q a execPGSQuery q a = mkCmd $ \conn -> PGS.execute conn q a
_Word16 :: TomlBiMap Word16 AnyValue
_Word16 = _BoundedInteger >>> _Integer
word16 :: Key -> TomlCodec Word16
word16 = match _Word16
{-# INLINE word16 #-}
dbParamsCodec :: TomlCodec PGS.ConnectInfo
dbParamsCodec = flip Toml.table "database" $ PGS.ConnectInfo
<$> Toml.string "host" .= PGS.connectHost
<*> word16 "port" .= PGS.connectPort
<*> Toml.string "user" .= PGS.connectUser
<*> Toml.string "password" .= PGS.connectPassword
<*> Toml.string "name" .= PGS.connectDatabase
------------------------------------------------------------------------ ------------------------------------------------------------------------
databaseParameters :: FilePath -> IO PGS.ConnectInfo databaseParameters :: FilePath -> IO PGS.ConnectInfo
databaseParameters fp = do databaseParameters fp = do
ini <- readIniFile' fp res <- readTomlConfigFileFrom dbParamsCodec fp
let val' key = unpack $ val ini "database" key case res of
let dbPortRaw = val' "DB_PORT" Left errs
let dbPort = case (readMaybe dbPortRaw :: Maybe Word16) of -> panicTrace $ "Error, gargantext.toml parsing failed: " <> Toml.prettyTomlDecodeErrors errs
Nothing -> panicTrace $ "DB_PORT incorrect: " <> (pack dbPortRaw) Right dbParams
Just d -> d -> pure dbParams
pure $ PGS.ConnectInfo { PGS.connectHost = val' "DB_HOST"
, PGS.connectPort = dbPort
, PGS.connectUser = val' "DB_USER"
, PGS.connectPassword = val' "DB_PASS"
, PGS.connectDatabase = val' "DB_NAME"
}
connectGargandb :: FilePath -> IO Connection connectGargandb :: FilePath -> IO Connection
connectGargandb fp = databaseParameters fp >>= \params -> connect params connectGargandb fp = databaseParameters fp >>= \params -> connect params
......
[gargantext]
URL = "http://localhost"
BACKEND_NAME = "localhost"
URL_BACKEND_API = "http://localhost:8008/api/v1.0"
EPO_API_URL = ""
MASTER_USER = "gargantua"
SECRET_KEY = "test_key"
DATA_FILEPATH = "~/.garg"
REPO_FILEPATH = "~/.garg"
FRAME_WRITE_URL = "URL_TO_CHANGE"
FRAME_CALC_URL = "URL_TO_CHANGE"
FRAME_VISIO_URL = "URL_TO_CHANGE"
FRAME_SEARX_URL = "URL_TO_CHANGE"
FRAME_ISTEX_URL = "URL_TO_CHANGE"
MAX_DOCS_PARSERS = "1000000"
MAX_DOCS_SCRAPERS = "10000"
JS_JOB_TIMEOUT = "1800"
JS_ID_TIMEOUT = "1800"
PUBMED_API_KEY = "no_key"
[nlp]
EN = "corenlp://localhost:9000"
FR = "spacy://localhost:8001"
All = "corenlp://localhost:9000"
[database]
host = "127.0.0.1"
port = 5432
name = "db_name"
user = "db_user"
password = "db_pass"
[mail]
MAIL_PORT = "25"
MAIL_HOST = "localhost"
MAIL_USER = "gargantext"
MAIL_PASSWORD = ""
MAIL_FROM = ""
MAIL_LOGIN_TYPE = "Normal"
{-# LANGUAGE OverloadedStrings #-}
module Test.Offline.TomlConfig (tests) where
import Database.PostgreSQL.Simple qualified as PGS
import Gargantext.Database.Prelude
import Prelude
import Test.Tasty
import Test.Tasty.HUnit
import Test.Utils
tests :: TestTree
tests = testGroup "TomlConfig" [
testGroup "Parsing" [
testCase "databaseParameters can be parsed from Toml" test_databaseParameters
]
]
test_databaseParameters :: Assertion
test_databaseParameters = do
params <- databaseParameters "test-data/test_config.toml"
PGS.connectPort params @??= 5432
...@@ -21,6 +21,7 @@ import qualified Test.Offline.JSON as JSON ...@@ -21,6 +21,7 @@ import qualified Test.Offline.JSON as JSON
import qualified Test.Offline.Errors as Errors import qualified Test.Offline.Errors as Errors
import qualified Test.Offline.Phylo as Phylo import qualified Test.Offline.Phylo as Phylo
import qualified Test.Offline.Stemming.Lancaster as Lancaster import qualified Test.Offline.Stemming.Lancaster as Lancaster
import qualified Test.Offline.TomlConfig as Config
import qualified Test.Parsers.Date as PD import qualified Test.Parsers.Date as PD
import qualified Test.Utils.Crypto as Crypto import qualified Test.Utils.Crypto as Crypto
import qualified Test.Utils.Jobs as Jobs import qualified Test.Utils.Jobs as Jobs
...@@ -55,4 +56,5 @@ main = do ...@@ -55,4 +56,5 @@ main = do
, similaritySpec , similaritySpec
, Phylo.tests , Phylo.tests
, testGroup "Stemming" [ Lancaster.tests ] , testGroup "Stemming" [ Lancaster.tests ]
, Config.tests
] ]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment