[ngrams] simplify WithQuery json structure

There is now only a 'datafield' field, no need for duplicated
'database'.

Related to #441

gargantext.cabal

@@ -788,6 +788,7 @@ test-suite garg-test-tasty
       Test.API.Prelude
       Test.API.UpdateList
       Test.Core.Notifications
+      Test.Core.Orchestrator
       Test.Core.Similarity
       Test.Core.Text
       Test.Core.Text.Corpus.Query

src/Gargantext/API/Admin/Orchestrator/Types.hs

@@ -17,7 +17,8 @@ Portability : POSIX
 module Gargantext.API.Admin.Orchestrator.Types
   where
 
-import Data.Aeson (genericParseJSON, genericToJSON)
+import Data.Aeson (genericParseJSON, genericToJSON, object, withObject, (.=), (.:), (.:?), Value(String))
+import Data.Aeson.Types (unexpected)
 import Data.Morpheus.Types ( GQLType(..), DropNamespace(..), typeDirective )
 import Data.Swagger (ToSchema, URL, declareNamedSchema, defaultSchemaOptions, genericDeclareNamedSchemaUnrestricted)
 -- import Gargantext.API.GraphQL.UnPrefix qualified as GQLU
@@ -42,8 +43,24 @@ data ExternalAPIs = OpenAlex
 
 
 -- | Main Instances
-instance FromJSON ExternalAPIs
-instance ToJSON ExternalAPIs
+instance FromJSON ExternalAPIs where
+  parseJSON = withObject "ExternalAPIs" $ \o -> do
+    db <- o .: "db"
+    case db of
+      "OpenAlex" -> pure OpenAlex
+      "PubMed" -> do
+        mAPIKey <- o .:? "api_key"
+        pure $ PubMed mAPIKey
+      "Arxiv" -> pure Arxiv
+      "HAL" -> pure HAL
+      "IsTex" -> pure IsTex
+      "Isidore" -> pure Isidore
+      "EPO" -> pure EPO
+      s -> unexpected (String s)
+instance ToJSON ExternalAPIs where
+  toJSON (PubMed mAPIKey) = object [ "db" .= toJSON ("PubMed" :: Text)
+                                   , "api_key" .= toJSON mAPIKey ]
+  toJSON t = object [ "db" .= toJSON (show t :: Text) ]
 
 externalAPIs :: [ExternalAPIs]
 externalAPIs =
@@ -55,10 +72,6 @@ externalAPIs =
   , Isidore
   , EPO ]
 
-instance Arbitrary ExternalAPIs
-  where
-    arbitrary = elements externalAPIs
-
 instance ToSchema ExternalAPIs where
   declareNamedSchema = genericDeclareNamedSchemaUnrestricted defaultSchemaOptions
 

src/Gargantext/API/Node/Corpus/New.hs

@@ -32,7 +32,7 @@ import Gargantext.API.Admin.Orchestrator.Types qualified as API
 import Gargantext.API.Ngrams (commitStatePatch, Versioned(..))
 import Gargantext.API.Node.Corpus.New.Types ( FileFormat(..), FileType(..) )
 import Gargantext.API.Node.Corpus.Searx ( triggerSearxSearch )
-import Gargantext.API.Node.Corpus.Types ( Datafield(Web), database2origin )
+import Gargantext.API.Node.Corpus.Types ( Datafield(Web), datafield2origin )
 import Gargantext.API.Node.Corpus.Update (addLanguageToCorpus)
 import Gargantext.API.Node.Types
 import Gargantext.Core (withDefaultLanguage, defaultLanguage)
@@ -155,13 +155,12 @@ addToCorpusWithQuery :: ( FlowCmdM env err m
                        -> JobHandle m
                        -> m ()
 addToCorpusWithQuery user cid (WithQuery { _wq_query = q
-                                         , _wq_databases = dbs
                                          , _wq_datafield = datafield
                                          , _wq_lang = l
                                          , _wq_flowListWith = flw
                                          , .. }) maybeLimit jobHandle = do
   -- TODO ...
-  $(logLocM) DEBUG $ "[addToCorpusWithQuery] (cid, dbs) " <> show (cid, dbs)
+  $(logLocM) DEBUG $ "[addToCorpusWithQuery] cid " <> show cid
   $(logLocM) DEBUG $ "[addToCorpusWithQuery] datafield  " <> show datafield
   $(logLocM) DEBUG $ "[addToCorpusWithQuery] flowListWith " <> show flw
 
@@ -173,7 +172,7 @@ addToCorpusWithQuery user cid (WithQuery { _wq_query = q
   $(logLocM) DEBUG "[addToCorpusWithQuery] after addLanguageToCorpus"
   
   case datafield of
-    Just Web -> do
+    Web -> do
       $(logLocM) DEBUG $ "[addToCorpusWithQuery] processing web request " <> show datafield
 
       markStarted 1 jobHandle
@@ -190,7 +189,7 @@ addToCorpusWithQuery user cid (WithQuery { _wq_query = q
       --      if cid is corpus -> add to corpus
       --      if cid is root   -> create corpus in Private
       $(logLocM) DEBUG $ "[addToCorpusWithQuery] getDataText with query: " <> show q
-      let db = database2origin dbs
+      let db = datafield2origin datafield
       -- mPubmedAPIKey <- getUserPubmedAPIKey user
       -- printDebug "[addToCorpusWithQuery] mPubmedAPIKey" mPubmedAPIKey
       eTxt <- getDataText db (Multi l) q mEPOAuthKey maybeLimit

src/Gargantext/API/Node/Corpus/Types.hs

@@ -9,17 +9,15 @@ Portability : POSIX
 -}
 
 
-{-# LANGUAGE TemplateHaskell    #-}
-
 module Gargantext.API.Node.Corpus.Types where
 
 import Control.Lens ( (?~) )
 import Control.Monad.Fail (fail)
-import Data.Aeson ( Value(..), (.:), withText, object )
+import Data.Aeson ( Value(..), (.:), (.:?), (.=), withText, object, withObject )
+import Data.Aeson.Types (unexpected)
 import Data.Swagger
 import Data.Text qualified as T
 import Gargantext.API.Admin.Orchestrator.Types qualified as Types
-import Gargantext.Core.Utils.Prefix (unPrefix)
 import Gargantext.Database.Action.Flow.Types (DataOrigin(..))
 import Gargantext.Prelude
 import PUBMED.Types qualified as PUBMED
@@ -34,19 +32,39 @@ data Database = Empty
               | EPO
   deriving (Eq, Show, Generic)
 
-deriveJSON (unPrefix "") ''Database
+instance FromJSON Database where
+  parseJSON = withObject "Database" $ \o -> do
+    db <- o .: "db"
+    case db of
+      "Empty" -> pure Empty
+      "OpenAlex" -> pure OpenAlex
+      "PubMed" -> do
+        mAPIKey <- o .:? "api_key"
+        pure $ PubMed mAPIKey
+      "Arxiv" -> pure Arxiv
+      "HAL" -> pure HAL
+      "IsTex" -> pure IsTex
+      "Isidore" -> pure Isidore
+      "EPO" -> pure EPO
+      s -> unexpected (String s)
+instance ToJSON Database where
+  toJSON (PubMed mAPIKey) = object [ "db" .= toJSON ("PubMed" :: Text)
+                                   , "api_key" .= toJSON mAPIKey ]
+  toJSON t = object [ "db" .= toJSON (show t :: Text) ]
 instance ToSchema Database where
   declareNamedSchema = genericDeclareNamedSchemaUnrestricted defaultSchemaOptions
 
-database2origin :: Database -> DataOrigin
-database2origin Empty      = InternalOrigin Types.IsTex
-database2origin OpenAlex   = ExternalOrigin Types.OpenAlex
-database2origin (PubMed k) = ExternalOrigin (Types.PubMed k)
-database2origin Arxiv      = ExternalOrigin Types.Arxiv
-database2origin HAL        = ExternalOrigin Types.HAL
-database2origin IsTex      = ExternalOrigin Types.IsTex
-database2origin Isidore    = ExternalOrigin Types.Isidore
-database2origin EPO        = ExternalOrigin Types.EPO
+datafield2origin :: Datafield -> DataOrigin
+datafield2origin (External Empty)            = InternalOrigin Types.IsTex
+datafield2origin (External OpenAlex)         = ExternalOrigin Types.OpenAlex
+datafield2origin (External (PubMed mAPIKey)) = ExternalOrigin (Types.PubMed mAPIKey)
+datafield2origin (External Arxiv)            = ExternalOrigin Types.Arxiv
+datafield2origin (External HAL)              = ExternalOrigin Types.HAL
+datafield2origin (External IsTex)            = ExternalOrigin Types.IsTex
+datafield2origin (External Isidore)          = ExternalOrigin Types.Isidore
+datafield2origin (External EPO)              = ExternalOrigin Types.EPO
+-- | This isn't really used
+datafield2origin _                           = InternalOrigin Types.IsTex
 
 ------------------------------------------------------------------------
 data Datafield = Gargantext

src/Gargantext/API/Node/Types.hs

@@ -83,8 +83,7 @@ instance GargDB.SaveFile NewWithFile where
 
 data WithQuery = WithQuery
   { _wq_query        :: !API.RawQuery
-  , _wq_databases    :: !Database
-  , _wq_datafield    :: !(Maybe Datafield)
+  , _wq_datafield    :: !Datafield
   , _wq_lang         :: !Lang
   , _wq_node_id      :: !Int
   , _wq_flowListWith :: !FlowSocialListWith

test-data/test_config.toml

@@ -22,8 +22,6 @@ data_filepath = "~/.garg"
 #repo_filepath = "~/.garg"
 
 [apis]
-[apis.pubmed]
-api_key = "no_key"
 
 [apis.epo]
 api_url = ""

test/Test/Core/Orchestrator.hs

+{-|
+Module      : Core.Orchestrator
+Description :
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+module Test.Core.Orchestrator
+  ( qcTests )
+where
+
+import Data.Aeson qualified as A
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.Prelude
+import Test.Instances ()
+import Test.Tasty
+import Test.Tasty.QuickCheck qualified as QC
+
+
+qcTests :: TestTree
+qcTests =
+  testGroup "Orchestrator QuickCheck tests" $ do
+    [ QC.testProperty "ExternalAPIs aeson encoding" $ \m -> A.decode (A.encode (m :: ExternalAPIs)) == Just m ]

test/Test/Instances.hs

@@ -23,13 +23,15 @@ import Data.Text qualified as T
 import Data.Validity (Validation(..), ValidationChain (..), prettyValidation)
 import EPO.API.Client.Types qualified as EPO
 import Gargantext.API.Admin.Auth.Types (AuthenticatedUser, ForgotPasswordAsyncParams)
+import Gargantext.API.Admin.Orchestrator.Types qualified as Orch
 import Gargantext.API.Errors.Types qualified as Errors
 import Gargantext.API.Ngrams.Types qualified as Ngrams
 import Gargantext.API.Node.Contact.Types (AddContactParams)
 import Gargantext.API.Node.Corpus.Annuaire (AnnuaireWithForm)
 import Gargantext.API.Node.Corpus.New (ApiInfo)
 import Gargantext.API.Node.Corpus.New.Types (FileFormat, FileType)
-import Gargantext.API.Node.Corpus.Types (Datafield, Database)
+import Gargantext.API.Node.Corpus.Types (Datafield)
+import Gargantext.API.Node.Corpus.Types qualified as CT
 import Gargantext.API.Node.DocumentsFromWriteNodes.Types qualified as DFWN
 import Gargantext.API.Node.DocumentUpload.Types (DocumentUpload)
 import Gargantext.API.Node.FrameCalcUpload.Types qualified as FCU
@@ -127,7 +129,15 @@ instance Arbitrary ApiInfo where arbitrary = genericArbitrary
 instance Arbitrary FileFormat where arbitrary = genericArbitrary
 instance Arbitrary FileType where arbitrary = genericArbitrary
 
-instance Arbitrary Database where arbitrary = arbitraryBoundedEnum
+instance Arbitrary CT.Database where
+  arbitrary = oneof [ pure CT.Empty
+                    , pure CT.OpenAlex
+                    , CT.PubMed <$> arbitrary
+                    , pure CT.Arxiv
+                    , pure CT.HAL
+                    , pure CT.IsTex
+                    , pure CT.Isidore
+                    , pure CT.EPO ]
 instance Arbitrary Datafield where arbitrary = genericArbitrary
 
 instance Arbitrary WithQuery where arbitrary = genericArbitrary
@@ -291,6 +301,17 @@ instance Arbitrary Hyperdata.HyperdataPublic where
   arbitrary = pure Hyperdata.defaultHyperdataPublic
 
 
+
+instance Arbitrary Orch.ExternalAPIs where
+  arbitrary = oneof [ pure Orch.OpenAlex
+                    , Orch.PubMed <$> arbitrary
+                    , pure Orch.Arxiv
+                    , pure Orch.HAL
+                    , pure Orch.IsTex
+                    , pure Orch.Isidore
+                    , pure Orch.EPO ]
+
+
 -- instance Arbitrary NewWithFile where
 --   arbitrary = NewWithFile <$> arbitrary  -- _wfi_b64_data
 --                           <*> arbitrary  -- _wf_lang

test/drivers/tasty/Main.hs

@@ -13,6 +13,7 @@ module Main where
 import Gargantext.Prelude
 
 import qualified Test.Core.Notifications         as Notifications
+import qualified Test.Core.Orchestrator          as Orchestrator
 import qualified Test.Core.Similarity            as Similarity
 import qualified Test.Core.Text.Corpus.Query     as CorpusQuery
 import qualified Test.Core.Text.Corpus.TSV       as TSVParser
@@ -75,5 +76,6 @@ main = do
     , Worker.tests
     , asyncUpdatesSpec
     , Notifications.qcTests
+    , Orchestrator.qcTests
     , NgramsTerms.tests
     ]