Simplify ExternalAPIs type

This commit simplifies the `ExternalAPIs` type, which is now a pure enumeration, moving the lookup of the Pubmed key at the usage sites. This simplified the code quite a bit, and made some otherwise-effectful functions pure.

Simplify ExternalAPIs type
This commit simplifies the `ExternalAPIs` type, which is now a pure enumeration, moving the lookup of the Pubmed key at the usage sites. This simplified the code quite a bit, and made some otherwise-effectful functions pure.
b3fb1a16 · Alfredo Di Napoli · b0ca0a6a · b3fb1a16 · b3fb1a16 · b3fb1a16
Commit b3fb1a16 authored May 22, 2023 by Alfredo Di Napoli
7 changed files
--- a/src/Gargantext/API/Admin/Orchestrator/Types.hs
+++ b/src/Gargantext/API/Admin/Orchestrator/Types.hs
@@ -6,7 +6,6 @@ module Gargantext.API.Admin.Orchestrator.Types
  where
 import Control.Lens hiding (elements)
-import Control.Monad.Reader (MonadReader)
 import Data.Aeson
 import Data.Morpheus.Types
  ( GQLType
@@ -24,9 +23,7 @@ import Test.QuickCheck.Arbitrary
 import qualified Gargantext.API.GraphQL.Utils as GQLU
 import Gargantext.Core.Types (TODO(..))
-import Gargantext.Database.Prelude (HasConfig(..))
 import Gargantext.Prelude
-import Gargantext.Prelude.Config (gc_pubmed_api_key)
 ------------------------------------------------------------------------
 instance Arbitrary a => Arbitrary (JobStatus 'Safe a) where
@@ -38,38 +35,24 @@ instance Arbitrary a => Arbitrary (JobOutput a) where
 -- | Main Types
 -- TODO IsidoreAuth
 data ExternalAPIs = All
-                  | PubMed { mAPIKey :: Maybe Text }
+                  | PubMed
                  | Arxiv
                  | HAL
                  | IsTex
                  | Isidore
-  deriving (Show, Eq, Generic)
+  deriving (Show, Eq, Generic, Enum, Bounded)
 -- | Main Instances
 instance FromJSON ExternalAPIs
 instance ToJSON ExternalAPIs
-externalAPIs :: ( MonadReader env m
+externalAPIs :: [ExternalAPIs]
-                , HasConfig env) => m [ExternalAPIs]
+externalAPIs = [minBound .. maxBound]
-externalAPIs = do
-  pubmed_api_key <- view $ hasConfig . gc_pubmed_api_key
-  pure [ All
-       , PubMed { mAPIKey = Just pubmed_api_key }
-       , Arxiv
-       , HAL
-       , IsTex
-       , Isidore ]
 instance Arbitrary ExternalAPIs
  where
-    arbitrary = elements [ All
+    arbitrary = arbitraryBoundedEnum
-                         , PubMed { mAPIKey = Nothing }
-                         , Arxiv
-                         , HAL
-                         , IsTex
-                         , Isidore ]
 instance ToSchema ExternalAPIs where
  declareNamedSchema = genericDeclareNamedSchemaUnrestricted defaultSchemaOptions

--- a/src/Gargantext/API/Node/Corpus/New.hs
+++ b/src/Gargantext/API/Node/Corpus/New.hs
@@ -54,13 +54,13 @@ import Gargantext.Database.Action.Mail (sendMail)
 import Gargantext.Database.Action.Node (mkNodeWithParent)
 import Gargantext.Database.Action.User (getUserId)
 import Gargantext.Database.Admin.Types.Hyperdata
-import Gargantext.Database.Admin.Types.Node (CorpusId, NodeType(..), UserId)
+import Gargantext.Database.Admin.Types.Node (CorpusId, NodeType(..))
 import Gargantext.Database.Prelude (hasConfig)
 import Gargantext.Database.Query.Table.Node (getNodeWith, updateCorpusPubmedAPIKey)
 import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
 import Gargantext.Database.Schema.Node (node_hyperdata)
 import Gargantext.Prelude
-import Gargantext.Prelude.Config (gc_max_docs_parsers)
+import Gargantext.Prelude.Config (gc_max_docs_parsers, gc_pubmed_api_key)
 import Gargantext.Utils.Jobs (JobHandle, MonadJobStatus(..))
 import qualified Gargantext.Core.Text.Corpus.API as API
 import qualified Gargantext.Core.Text.Corpus.Parsers as Parser (FileType(..), parseFormatC)
@@ -130,11 +130,8 @@ deriveJSON (unPrefix "") 'ApiInfo
 instance ToSchema ApiInfo
-info :: FlowCmdM env err m => UserId -> m ApiInfo
+info :: ApiInfo
-info _u = do
+info = ApiInfo API.externalAPIs
-  ext <- API.externalAPIs
-  pure $ ApiInfo ext
 ------------------------------------------------------------------------
 ------------------------------------------------------------------------
@@ -210,7 +207,8 @@ addToCorpusWithQuery user cid (WithQuery { _wq_query = q
    _ -> do
      case datafield of
-         Just (External (PubMed { _api_key })) -> do
+         Just (External PubMed) -> do
+           _api_key <- view $ hasConfig . gc_pubmed_api_key
           printDebug "[addToCorpusWithQuery] pubmed api key" _api_key
           _ <- updateCorpusPubmedAPIKey cid _api_key
           pure ()
@@ -222,7 +220,7 @@ addToCorpusWithQuery user cid (WithQuery { _wq_query = q
      --      if cid is corpus -> add to corpus
      --      if cid is root   -> create corpus in Private
      -- printDebug "[G.A.N.C.New] getDataText with query" q
-      db   <- database2origin dbs
+      let db = database2origin dbs
      eTxt <- getDataText db (Multi l) q maybeLimit
      -- printDebug "[G.A.N.C.New] lTxts" lTxts

--- a/src/Gargantext/API/Node/Corpus/Types.hs
+++ b/src/Gargantext/API/Node/Corpus/Types.hs
@@ -3,44 +3,37 @@
 module Gargantext.API.Node.Corpus.Types where
 import Control.Lens hiding (elements, Empty)
-import Control.Monad.Reader (MonadReader)
 import Data.Aeson
 import Data.Aeson.TH (deriveJSON)
 import Data.Monoid (mempty)
 import Data.Swagger
 import GHC.Generics (Generic)
-import qualified PUBMED.Types as PUBMED
 import Gargantext.Prelude
 import qualified Gargantext.API.Admin.Orchestrator.Types as T
 import Gargantext.Core.Utils.Prefix (unPrefix)
 import Gargantext.Database.Action.Flow (DataOrigin(..))
-import Gargantext.Database.Prelude (HasConfig(..))
 data Database = Empty
-              | PubMed { _api_key :: Maybe PUBMED.APIKey }
+              | PubMed
              | Arxiv
              | HAL
              | IsTex
              | Isidore
-  deriving (Eq, Show, Generic)
+  deriving (Eq, Show, Generic, Enum, Bounded)
 deriveJSON (unPrefix "") ''Database
 instance ToSchema Database where
  declareNamedSchema = genericDeclareNamedSchemaUnrestricted defaultSchemaOptions
-database2origin :: ( MonadReader env m
+database2origin :: Database -> DataOrigin
-                   , HasConfig env ) => Database -> m DataOrigin
+database2origin Empty   = InternalOrigin T.IsTex
-database2origin Empty   = pure $ InternalOrigin T.IsTex
+database2origin PubMed  = ExternalOrigin T.PubMed
-database2origin (PubMed { _api_key }) = do
+database2origin Arxiv   = ExternalOrigin T.Arxiv
-  -- pubmed_api_key <- view $ hasConfig . gc_pubmed_api_key
+database2origin HAL     = ExternalOrigin T.HAL
+database2origin IsTex   = ExternalOrigin T.IsTex
-  pure $ ExternalOrigin $ T.PubMed { mAPIKey = _api_key }
+database2origin Isidore = ExternalOrigin T.Isidore
-database2origin Arxiv   = pure $ ExternalOrigin T.Arxiv
-database2origin HAL     = pure $ ExternalOrigin T.HAL
-database2origin IsTex   = pure $ ExternalOrigin T.IsTex
-database2origin Isidore = pure $ ExternalOrigin T.Isidore
 ------------------------------------------------------------------------
 data Datafield = Gargantext

--- a/src/Gargantext/Core/Text/Corpus/API.hs
+++ b/src/Gargantext/Core/Text/Corpus/API.hs
@@ -18,12 +18,14 @@ module Gargantext.Core.Text.Corpus.API
  ) where
 import Conduit
+import Control.Lens ((^.))
 import Data.Either (Either(..))
 import Data.Maybe
 import Gargantext.API.Admin.Orchestrator.Types (ExternalAPIs(..), externalAPIs)
 import Gargantext.Core (Lang(..))
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
 import Gargantext.Prelude
+import Gargantext.Prelude.Config (GargConfig, gc_pubmed_api_key)
 import qualified Gargantext.Core.Text.Corpus.API.Arxiv   as Arxiv
 import qualified Gargantext.Core.Text.Corpus.API.Hal     as HAL
 import qualified Gargantext.Core.Text.Corpus.API.Isidore as ISIDORE
@@ -32,24 +34,25 @@ import qualified Gargantext.Core.Text.Corpus.API.Pubmed  as PUBMED
 import Servant.Client (ClientError)
 -- | Get External API metadata main function
-get :: ExternalAPIs
+get :: GargConfig
+    -> ExternalAPIs
    -> Lang
    -> Query
    -> Maybe Limit
    -- -> IO [HyperdataDocument]
    -> IO (Either ClientError (Maybe Integer, ConduitT () HyperdataDocument IO ()))
-get PubMed { mAPIKey = mAPIKey }  _la q limit = PUBMED.get mAPIKey q limit
+get cfg PubMed _la q limit = PUBMED.get (cfg ^. gc_pubmed_api_key) q limit
  --docs <- PUBMED.get   q default_limit -- EN only by default
  --pure (Just $ fromIntegral $ length docs, yieldMany docs)
-get Arxiv    la q limit = Arxiv.get la q (fromIntegral <$> limit)
+get _ Arxiv    la q limit = Arxiv.get la q (fromIntegral <$> limit)
-get HAL      la q limit = HAL.getC  la q limit
+get _ HAL      la q limit = HAL.getC  la q limit
-get IsTex    la q limit = do
+get _ IsTex    la q limit = do
  docs <- ISTEX.get la q limit
  pure $ Right (Just $ fromIntegral $ length docs, yieldMany docs)
-get Isidore  la q limit = do
+get _ Isidore  la q limit = do
  docs <- ISIDORE.get la (fromIntegral <$> limit) (Just q) Nothing
  pure $ Right (Just $ fromIntegral $ length docs, yieldMany docs)
-get externalApi  _  _ _ = panic $ "[G.C.T.Corpus.API] This options are note taken into account: " <>  (cs $ show externalApi)
+get _ externalApi  _  _ _ = panic $ "[G.C.T.Corpus.API] This options are note taken into account: " <>  (cs $ show externalApi)
 -- | Some Sugar for the documentation
 type Query = PUBMED.Query

--- a/src/Gargantext/Core/Text/Corpus/API/Pubmed.hs
+++ b/src/Gargantext/Core/Text/Corpus/API/Pubmed.hs
@@ -36,12 +36,12 @@ type Limit = Integer
 -- | TODO put default pubmed query in gargantext.ini
 -- by default: 10K docs
-get :: Maybe Text
+get :: Text
    -> Query
    -> Maybe Limit
    -> IO (Either ClientError (Maybe Integer, ConduitT () HyperdataDocument IO ()))
-get mAPIKey q l = do
+get apiKey q l = do
-  eRes <- runReaderT PubMed.getMetadataWithC (Config { apiKey  = mAPIKey
+  eRes <- runReaderT PubMed.getMetadataWithC (Config { apiKey  = Just apiKey
                                                     , query   = q
                                                     , perPage = Just 200
                                                     , mWebEnv = Nothing })

--- a/src/Gargantext/Database/Action/Flow.hs
+++ b/src/Gargantext/Database/Action/Flow.hs
@@ -50,7 +50,6 @@ module Gargantext.Database.Action.Flow -- (flowDatabase, ngrams2list)
 import Conduit
 import Control.Lens ((^.), view, _Just, makeLenses, over, traverse)
-import Control.Monad.Reader (MonadReader)
 import Data.Aeson.TH (deriveJSON)
 import Data.Conduit.Internal (zipSources)
 import qualified Data.Conduit.List as CList
@@ -133,13 +132,8 @@ deriveJSON (unPrefix "_do_") ''DataOrigin
 instance ToSchema DataOrigin where
  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_do_")
-allDataOrigins :: ( MonadReader env m
+allDataOrigins :: [DataOrigin]
-                  , HasConfig env) => m [DataOrigin]
+allDataOrigins = map InternalOrigin API.externalAPIs <> map ExternalOrigin API.externalAPIs
-allDataOrigins = do
-  ext <- API.externalAPIs
-  pure $ map InternalOrigin ext
-      <> map ExternalOrigin ext
 ---------------
 data DataText = DataOld ![NodeId]
@@ -160,8 +154,9 @@ getDataText :: FlowCmdM env err m
            -> API.Query
            -> Maybe API.Limit
            -> m (Either ClientError DataText)
-getDataText (ExternalOrigin api) la q li = liftBase $ do
+getDataText (ExternalOrigin api) la q li = do
-  eRes <- API.get api (_tt_lang la) q li
+  cfg  <- view $ hasConfig
+  eRes <- liftBase $ API.get cfg api (_tt_lang la) q li
  pure $ DataNew <$> eRes
 getDataText (InternalOrigin _) _la q _li = do

--- a/src/Gargantext/Database/Query/Table/Node.hs
+++ b/src/Gargantext/Database/Query/Table/Node.hs
@@ -342,8 +342,8 @@ getCorpusPubmedAPIKey cId = do
            |]
    params = PGS.Only cId
-updateCorpusPubmedAPIKey :: NodeId -> Maybe PUBMED.APIKey -> Cmd err Int64
+updateCorpusPubmedAPIKey :: NodeId -> PUBMED.APIKey -> Cmd err Int64
-updateCorpusPubmedAPIKey cId mAPIKey =
+updateCorpusPubmedAPIKey cId apiKey =
  execPGSQuery query params
  where
    query :: PGS.Query
@@ -352,7 +352,7 @@ updateCorpusPubmedAPIKey cId mAPIKey =
                SET hyperdata = hyperdata || ?
                WHERE id = ?
            |]
-    params = (encode $ object [ "pubmed_api_key" .= mAPIKey ], cId)
+    params = (encode $ object [ "pubmed_api_key" .= apiKey ], cId)
 ------------------------------------------------------------------------
 -- TODO
 -- currently this function removes the child relation