[API|Query] WIP need to fit query with frontend

src/Gargantext/API/Admin/Orchestrator/Types.hs

@@ -36,15 +36,9 @@ instance Arbitrary a => Arbitrary (JobOutput a) where
 -- | Main Types
 data ExternalAPIs = All
                   | PubMed
-
-                  | HAL_EN
-                  | HAL_FR
-
-                  | IsTex_EN
-                  | IsTex_FR
-
-                  | Isidore_EN
-                  | Isidore_FR
+                  | HAL
+                  | IsTex
+                  | Isidore
                   -- | IsidoreAuth
   deriving (Show, Eq, Enum, Bounded, Generic)
 

src/Gargantext/API/Corpus/New.hs

@@ -38,10 +38,9 @@ import Gargantext.API.Corpus.New.File
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Types.Individu (UserId, User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, flowCorpusSearchInDatabase)
+import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, getDataText, flowDataText, TermType(..), DataOrigin(..))
 import Gargantext.Database.Admin.Types.Node (CorpusId, ToHyperdataDocument(..))
 import Gargantext.Prelude
-import Gargantext.Text.Terms (TermType(..))
 import Servant
 import Servant.API.Flatten (Flat)
 import Servant.Job.Core
@@ -65,8 +64,9 @@ deriveJSON (unPrefix "query_") 'Query
 
 instance Arbitrary Query where
     arbitrary = elements [ Query q n fs
-                         | q <- ["honeybee* AND collopase"
-                                ,"covid 19"]
+                         | q <- ["honeybee* AND collapse"
+                                ,"covid 19"
+                                ]
                          , n <- [0..10]
                          , fs <- take 3 $ repeat API.externalAPIs
                          ]
@@ -119,8 +119,8 @@ info _u = pure $ ApiInfo API.externalAPIs
 ------------------------------------------------------------------------
 data WithQuery = WithQuery
   { _wq_query     :: !Text
-  , _wq_databases :: ![ExternalAPIs]
-  , _wq_lang      :: !(Maybe Lang)
+  , _wq_databases :: ![DataOrigin]
+  , _wq_lang      :: !(Maybe (TermType Lang))
   }
   deriving Generic
 
@@ -192,7 +192,7 @@ addToCorpusWithQuery :: FlowCmdM env err m
                        -> WithQuery
                        -> (ScraperStatus -> m ())
                        -> m ScraperStatus
-addToCorpusWithQuery u cid (WithQuery q _dbs l) logStatus = do
+addToCorpusWithQuery u cid (WithQuery q dbs l) logStatus = do
   -- TODO ...
   logStatus ScraperStatus { _scst_succeeded = Just 10
                           , _scst_failed    = Just 2
@@ -204,7 +204,8 @@ addToCorpusWithQuery u cid (WithQuery q _dbs l) logStatus = do
   -- TODO if cid is folder -> create Corpus
   --      if cid is corpus -> add to corpus
   --      if cid is root   -> create corpus in Private
-  cids <- flowCorpusSearchInDatabase u (maybe EN identity l) q
+  txts <- mapM (\db  -> getDataText db (fromMaybe (Multi EN) l) q (Just 10000)) dbs
+  cids <- mapM (\txt -> flowDataText u txt (fromMaybe (Multi EN) l) cid) txts
   printDebug "corpus id" cids
   -- TODO ...
   pure      ScraperStatus { _scst_succeeded = Just 137

src/Gargantext/Text/Corpus/API.hs

@@ -34,20 +34,17 @@ import qualified Gargantext.Text.Corpus.API.Istex   as ISTEX
 import qualified Gargantext.Text.Corpus.API.Pubmed  as PUBMED
 
 -- | Get External API metadata main function
-get :: ExternalAPIs -> Query -> Maybe Limit -> IO [HyperdataDocument]
+get :: ExternalAPIs
+    -> Lang
+    -> Query
+    -> Maybe Limit
+    -> IO [HyperdataDocument]
+get PubMed  _la q l = PUBMED.get q l -- EN only by default
+get HAL      la q l = HAL.get   la q l
+get IsTex    la q l = ISTEX.get la q l
+get Isidore  la q l = ISIDORE.get la (fromIntegral <$> l) (Just q) Nothing
+get _        _  _ _ = undefined
 
-get PubMed     q l = PUBMED.get q l
-
-get HAL_EN     q l = HAL.get EN q l
-get HAL_FR     q l = HAL.get FR q l
-
-get IsTex_EN   q l = ISTEX.get EN q l
-get IsTex_FR   q l = ISTEX.get FR q l
-
-get Isidore_EN q l = ISIDORE.get EN (fromIntegral <$> l) (Just q) Nothing
-get Isidore_FR q l = ISIDORE.get FR (fromIntegral <$> l) (Just q) Nothing
-
-get _  _ _ = undefined
 -- | Some Sugar for the documentation
 type Query = PUBMED.Query
 type Limit = PUBMED.Limit

src/Gargantext/Text/Terms.hs

@@ -30,41 +30,45 @@ compute graph
 
 {-# LANGUAGE NoImplicitPrelude #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE DeriveGeneric     #-}
 {-# LANGUAGE TemplateHaskell   #-}
 
 module Gargantext.Text.Terms
   where
 
 import Control.Lens
+import Data.Aeson.TH (deriveJSON)
+import Data.Swagger
 import Data.Text (Text)
 import Data.Traversable
 import GHC.Base (String)
-
-import Gargantext.Prelude
+import GHC.Generics (Generic)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Core
 import Gargantext.Core.Types
-import Gargantext.Text.Terms.Multi (multiterms)
+import Gargantext.Prelude
+import Gargantext.Text (sentences)
+import Gargantext.Text.Terms.Eleve (mainEleveWith, Tries, Token, buildTries, toToken)
 import Gargantext.Text.Terms.Mono  (monoTerms)
 import Gargantext.Text.Terms.Mono.Stem (stem)
-
-import qualified Data.Set  as Set
+import Gargantext.Text.Terms.Mono.Token.En (tokenize)
+import Gargantext.Text.Terms.Multi (multiterms)
 import qualified Data.List as List
+import qualified Data.Set  as Set
 import qualified Data.Text as Text
-import Gargantext.Text (sentences)
-import Gargantext.Text.Terms.Mono.Token.En (tokenize)
-import Gargantext.Text.Terms.Eleve (mainEleveWith, Tries, Token, buildTries, toToken)
 
 data TermType lang
   = Mono      { _tt_lang :: lang }
   | Multi     { _tt_lang :: lang }
   | MonoMulti { _tt_lang :: lang }
   | Unsupervised { _tt_lang  :: lang
-                 , _tt_windoSize  :: Int
+                 , _tt_windowSize :: Int
                  , _tt_ngramsSize :: Int
-                 , _tt_model :: Maybe (Tries Token ())
-  }
-makeLenses ''TermType
+                 , _tt_model      :: Maybe (Tries Token ())
+                 }
+  deriving Generic
 
+makeLenses ''TermType
 --group :: [Text] -> [Text]
 --group = undefined