[API Query] connection with front ok

src/Gargantext/API.hs

@@ -97,7 +97,7 @@ import qualified Data.Text.IO               as T
 import qualified Gargantext.API.Corpus.Annuaire  as Annuaire
 import qualified Gargantext.API.Corpus.Export    as Export
 import qualified Gargantext.API.Corpus.New       as New
-import qualified Gargantext.API.Ngrams.List      as List
+-- import qualified Gargantext.API.Ngrams.List      as List
 import qualified Paths_gargantext                as PG -- cabal magic build module
 
 showAsServantErr :: GargError -> ServerError
@@ -300,22 +300,25 @@ type GargPrivateAPI' =
                           :> TreeAPI
 
            -- :<|> New.Upload
-           :<|> New.AddWithForm
+           -- :<|> New.AddWithForm
            :<|> New.AddWithQuery
 
-           :<|> "annuaire" :> Annuaire.AddWithForm
+           -- :<|> "annuaire" :> Annuaire.AddWithForm
            -- :<|> New.AddWithFile
        --  :<|> "scraper" :> WithCallbacks ScraperAPI
        --  :<|> "new"  :> New.Api
 
-           :<|> "lists"  :> Summary "List export API"
-                         :> Capture "listId" ListId
-                         :> List.API
-
            :<|> "wait"   :> Summary "Wait test"
                          :> Capture "x" Int
                          :> WaitAPI -- Get '[JSON] Int
 
+          -- TODO "list"
+          {-
+           :<|> "lists"  :> Summary "List export API"
+                         :> Capture "listId" ListId
+                         :> List.API
+-}
+
 -- /mv/<id>/<id>
 -- /merge/<id>/<id>
 -- /rename/<id>
@@ -406,25 +409,33 @@ serverPrivateGargAPI' (AuthenticatedUser (NodeId uid))
      -- TODO access
      -- :<|> addUpload
      -- :<|> (\corpus -> addWithQuery corpus :<|> addWithFile corpus)
-     :<|> addCorpusWithForm  (UserDBId uid)
+     -- :<|> addCorpusWithForm  (UserDBId uid)
      :<|> addCorpusWithQuery (RootId   (NodeId uid))
 
-     :<|> addAnnuaireWithForm
+     -- :<|> addAnnuaireWithForm
      -- :<|> New.api  uid -- TODO-SECURITY
      -- :<|> New.info uid -- TODO-SECURITY
-     :<|> List.api
      :<|> waitAPI
+     -- :<|> List.api
 
 
 addCorpusWithQuery :: User -> GargServer New.AddWithQuery
 addCorpusWithQuery user cid =
   serveJobsAPI $
-    JobFunction (\i log -> New.addToCorpusWithQuery user cid i (liftBase . log))
+    JobFunction (\q log ->
+      let
+        log' x = do
+          printDebug "addToCorpusWithQuery" x
+          liftBase $ log x
+      in New.addToCorpusWithQuery user cid q log'
+      )
 
+{-
 addWithFile :: GargServer New.AddWithFile
 addWithFile cid i f =
   serveJobsAPI $
     JobFunction (\_i log -> New.addToCorpusWithFile cid i f (liftBase . log))
+-}
 
 addCorpusWithForm :: User -> GargServer New.AddWithForm
 addCorpusWithForm user cid =
@@ -432,7 +443,7 @@ addCorpusWithForm user cid =
     JobFunction (\i log ->
       let
         log' x = do
-          printDebug "addCorpusWithForm" x
+          printDebug "addToCorpusWithForm" x
           liftBase $ log x
       in New.addToCorpusWithForm user cid i log')
 

src/Gargantext/API/Corpus/New.hs

@@ -25,7 +25,7 @@ New corpus means either:
 module Gargantext.API.Corpus.New
       where
 
-import Control.Lens hiding (elements)
+import Control.Lens hiding (elements, Empty)
 import Data.Aeson
 import Data.Aeson.TH (deriveJSON)
 import Data.Either
@@ -33,12 +33,13 @@ import Data.Maybe (fromMaybe)
 import Data.Swagger
 import Data.Text (Text)
 import GHC.Generics (Generic)
-import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Admin.Orchestrator.Types (ScraperStatus(..))
+import qualified Gargantext.API.Admin.Orchestrator.Types as T
 import Gargantext.API.Corpus.New.File
-import Gargantext.Core (Lang(..), allLangs)
+import Gargantext.Core (Lang(..){-, allLangs-})
 import Gargantext.Core.Types.Individu (UserId, User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, getDataText, flowDataText, TermType(..), DataOrigin(..), allDataOrigins)
+import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, getDataText, flowDataText, TermType(..), DataOrigin(..){-, allDataOrigins-})
 import Gargantext.Database.Admin.Types.Node (CorpusId, ToHyperdataDocument(..))
 import Gargantext.Prelude
 import Servant
@@ -46,14 +47,15 @@ import Servant.API.Flatten (Flat)
 import Servant.Job.Core
 import Servant.Job.Types
 import Servant.Job.Utils (jsonOptions)
-import Servant.Multipart
-import Test.QuickCheck (elements)
+-- import Servant.Multipart
+-- import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary
 import Web.FormUrlEncoded          (FromForm)
 import qualified Gargantext.Text.Corpus.API as API
 import qualified Gargantext.Text.Corpus.Parsers as Parser (FileFormat(..), parseFormat)
 
 ------------------------------------------------------------------------
+{-
 data Query = Query { query_query      :: Text
                    , query_node_id    :: Int
                    , query_lang       :: Lang
@@ -75,9 +77,11 @@ instance Arbitrary Query where
 
 instance ToSchema Query where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "query_")
+-}
 
 ------------------------------------------------------------------------
 
+{-
 type Api = PostApi
         :<|> GetApi
 
@@ -85,6 +89,7 @@ type PostApi = Summary "New Corpus endpoint"
              :> ReqBody '[JSON] Query
              :> Post    '[JSON] CorpusId
 type GetApi = Get '[JSON] ApiInfo
+-}
 
 -- | TODO manage several apis
 -- TODO-ACCESS
@@ -118,11 +123,30 @@ info :: FlowCmdM env err m => UserId -> m ApiInfo
 info _u = pure $ ApiInfo API.externalAPIs
 
 ------------------------------------------------------------------------
+
+data Database = Empty
+              | PubMed
+              | HAL
+              | IsTex
+              | Isidore
+  deriving (Eq, Show, Generic)
+
+deriveJSON (unPrefix "") ''Database
+instance ToSchema Database
+
+database2origin :: Database -> DataOrigin
+database2origin Empty   = InternalOrigin T.IsTex
+database2origin PubMed  = ExternalOrigin T.PubMed
+database2origin HAL     = ExternalOrigin T.HAL
+database2origin IsTex   = ExternalOrigin T.IsTex
+database2origin Isidore = ExternalOrigin T.Isidore
+
 ------------------------------------------------------------------------
 data WithQuery = WithQuery
   { _wq_query     :: !Text
-  , _wq_databases :: ![DataOrigin]
-  , _wq_lang      :: !(Maybe (TermType Lang))
+  , _wq_databases :: !Database
+  , _wq_lang      :: !Lang
+  , _wq_node_id   :: !Int
   }
   deriving Generic
 
@@ -152,22 +176,13 @@ type AsyncJobs event ctI input output =
   Flat (AsyncJobsAPI' 'Unsafe 'Safe ctI '[JSON] Maybe event input output)
 ------------------------------------------------------------------------
 
-type Upload = Summary "Corpus Upload endpoint"
-   :> "corpus"
-     :> Capture "corpus_id" CorpusId
-   :<|> "addWithquery"
-     :> AsyncJobsAPI ScraperStatus                   WithQuery ScraperStatus
-   :<|> "addWithfile"
-     :> AsyncJobs    ScraperStatus '[FormUrlEncoded] WithForm  ScraperStatus
-
 type AddWithQuery = Summary "Add with Query to corpus endpoint"
    :> "corpus"
      :> Capture "corpus_id" CorpusId
-   :> "add"
    :> "query"
-   :> "async"
-     :> AsyncJobsAPI ScraperStatus WithQuery ScraperStatus
+     :> AsyncJobs ScraperStatus '[JSON] WithQuery ScraperStatus
 
+{-
 type AddWithFile = Summary "Add with MultipartData to corpus endpoint"
    :> "corpus"
      :> Capture "corpus_id" CorpusId
@@ -177,6 +192,7 @@ type AddWithFile = Summary "Add with MultipartData to corpus endpoint"
      :> QueryParam "fileType"  FileType
    :> "async"
      :> AsyncJobs ScraperStatus '[JSON] () ScraperStatus
+-}
 
 type AddWithForm = Summary "Add with FormUrlEncoded to corpus endpoint"
    :> "corpus"
@@ -194,7 +210,7 @@ addToCorpusWithQuery :: FlowCmdM env err m
                        -> WithQuery
                        -> (ScraperStatus -> m ())
                        -> m ScraperStatus
-addToCorpusWithQuery u cid (WithQuery q dbs l) logStatus = do
+addToCorpusWithQuery u cid (WithQuery q dbs l _nid) logStatus = do
   -- TODO ...
   logStatus ScraperStatus { _scst_succeeded = Just 10
                           , _scst_failed    = Just 2
@@ -206,8 +222,8 @@ addToCorpusWithQuery u cid (WithQuery q dbs l) logStatus = do
   -- TODO if cid is folder -> create Corpus
   --      if cid is corpus -> add to corpus
   --      if cid is root   -> create corpus in Private
-  txts <- mapM (\db  -> getDataText db     (fromMaybe (Multi EN) l) q (Just 10000)) dbs
-  cids <- mapM (\txt -> flowDataText u txt (fromMaybe (Multi EN) l) cid) txts
+  txts <- mapM (\db  -> getDataText db     (Multi l) q (Just 10000)) [database2origin dbs]
+  cids <- mapM (\txt -> flowDataText u txt (Multi l) cid) txts
   printDebug "corpus id" cids
   -- TODO ...
   pure      ScraperStatus { _scst_succeeded = Just 137
@@ -216,27 +232,6 @@ addToCorpusWithQuery u cid (WithQuery q dbs l) logStatus = do
                           , _scst_events    = Just []
                           }
 
-addToCorpusWithFile :: FlowCmdM env err m
-                    => CorpusId
-                    -> MultipartData Mem
-                    -> Maybe FileType
-                    -> (ScraperStatus -> m ())
-                    -> m ScraperStatus
-addToCorpusWithFile cid input filetype logStatus = do
-  logStatus ScraperStatus { _scst_succeeded = Just 10
-                          , _scst_failed    = Just 2
-                          , _scst_remaining = Just 138
-                          , _scst_events    = Just []
-                          }
-  printDebug "addToCorpusWithFile" cid
-  _h <- postUpload cid filetype input
-
-  pure      ScraperStatus { _scst_succeeded = Just 137
-                          , _scst_failed    = Just 13
-                          , _scst_remaining = Just 0
-                          , _scst_events    = Just []
-                          }
-
 addToCorpusWithForm :: FlowCmdM env err m
                     => User
                     -> CorpusId
@@ -282,3 +277,27 @@ addToCorpusWithForm user cid (WithForm ft d l _n) logStatus = do
                           , _scst_events    = Just []
                           }
 
+{-
+addToCorpusWithFile :: FlowCmdM env err m
+                    => CorpusId
+                    -> MultipartData Mem
+                    -> Maybe FileType
+                    -> (ScraperStatus -> m ())
+                    -> m ScraperStatus
+addToCorpusWithFile cid input filetype logStatus = do
+  logStatus ScraperStatus { _scst_succeeded = Just 10
+                          , _scst_failed    = Just 2
+                          , _scst_remaining = Just 138
+                          , _scst_events    = Just []
+                          }
+  printDebug "addToCorpusWithFile" cid
+  _h <- postUpload cid filetype input
+
+  pure      ScraperStatus { _scst_succeeded = Just 137
+                          , _scst_failed    = Just 13
+                          , _scst_remaining = Just 0
+                          , _scst_events    = Just []
+                          }
+-}
+
+

src/Gargantext/API/Ngrams/List.hs

@@ -54,11 +54,7 @@ type API =  Get '[JSON, HTML] (Headers '[Header "Content-Disposition" Text] Ngra
        :<|> PostAPI
 
 api :: ListId -> GargServer API
-api l =
-        get l
-    :<|>
-        -- post l
-        postAsync l
+api l = get l :<|> postAsync l
 
 data HTML
 instance Accept HTML where

src/Gargantext/Database/Action/Flow.hs

@@ -103,8 +103,8 @@ import qualified Gargantext.Text.Corpus.API as API
 
 ------------------------------------------------------------------------
 -- TODO use internal with API name (could be old data)
-data DataOrigin = Internal { _do_api :: API.ExternalAPIs }
-                | External { _do_api :: API.ExternalAPIs }
+data DataOrigin = InternalOrigin { _do_api :: API.ExternalAPIs }
+                | ExternalOrigin { _do_api :: API.ExternalAPIs }
                -- TODO Web
   deriving (Generic, Eq)
 
@@ -114,7 +114,8 @@ instance ToSchema DataOrigin where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_do_")
 
 allDataOrigins :: [DataOrigin]
-allDataOrigins = map Internal API.externalAPIs <> map External API.externalAPIs
+allDataOrigins = map InternalOrigin API.externalAPIs
+              <> map ExternalOrigin API.externalAPIs
 
 ---------------
 
@@ -129,10 +130,10 @@ getDataText :: FlowCmdM env err m
             -> API.Query
             -> Maybe API.Limit
             -> m DataText
-getDataText (External api) la q li = liftBase $ DataNew
+getDataText (ExternalOrigin api) la q li = liftBase $ DataNew
                                   <$> splitEvery 500
                                   <$> API.get api (_tt_lang la) q li
-getDataText (Internal _) _la q _li = do
+getDataText (InternalOrigin _) _la q _li = do
   (_masterUserId, _masterRootId, cId) <- getOrMk_RootWithCorpus
                                            (UserName userMaster)
                                            (Left "")