[CONFIG] limit to crawlers

src/Gargantext/API.hs

@@ -181,9 +181,8 @@ makeMockApp env = do
 
 makeDevMiddleware :: Mode -> IO Middleware
 makeDevMiddleware mode = do
-
-    -- logWare <- mkRequestLogger def { destination = RequestLogger.Logger $ env^.logger }
-    --logWare <- mkRequestLogger def { destination = RequestLogger.Logger "/tmp/logs.txt" }
+-- logWare <- mkRequestLogger def { destination = RequestLogger.Logger $ env^.logger }
+-- logWare <- mkRequestLogger def { destination = RequestLogger.Logger "/tmp/logs.txt" }
 --    let checkOriginAndHost app req resp = do
 --            blocking <- fireWall req (env ^. menv_firewall)
 --            case blocking  of

src/Gargantext/API/Node/Corpus/New.hs

@@ -184,9 +184,10 @@ addToCorpusWithQuery :: FlowCmdM env err m
                        => User
                        -> CorpusId
                        -> WithQuery
+                       -> Maybe Integer
                        -> (JobLog -> m ())
                        -> m JobLog
-addToCorpusWithQuery u cid (WithQuery q dbs l _nid) logStatus = do
+addToCorpusWithQuery u cid (WithQuery q dbs l _nid) maybeLimit logStatus = do
   -- TODO ...
   logStatus JobLog { _scst_succeeded = Just 0
                    , _scst_failed    = Just 0
@@ -198,7 +199,7 @@ addToCorpusWithQuery u cid (WithQuery q dbs l _nid) logStatus = do
   -- TODO if cid is folder -> create Corpus
   --      if cid is corpus -> add to corpus
   --      if cid is root   -> create corpus in Private
-  txts <- mapM (\db  -> getDataText db     (Multi l) q Nothing) [database2origin dbs]
+  txts <- mapM (\db  -> getDataText db (Multi l) q maybeLimit) [database2origin dbs]
 
   logStatus JobLog { _scst_succeeded = Just 2
                    , _scst_failed    = Just 0

src/Gargantext/API/Prelude.hs

@@ -81,8 +81,8 @@ type GargServerC env err m =
     , HasJoseError     err
     , ToJSON           err -- TODO this is arguable
     , Exception        err
-    , HasRepo      env
-    , HasSettings  env
+    , HasRepo      env  -- TODO rename HasNgramsRepo
+    , HasSettings  env  -- TODO rename HasDbSettings
     , HasJobEnv    env JobLog JobLog
     , HasConfig    env
     )

src/Gargantext/API/Routes.hs

@@ -26,6 +26,7 @@ module Gargantext.API.Routes
 
 -- import qualified Gargantext.API.Search as Search
 import Control.Concurrent (threadDelay)
+-- import Control.Lens (view)
 import Data.Text (Text)
 import Data.Validity
 import Gargantext.API.Admin.Auth (AuthRequest, AuthResponse, AuthenticatedUser(..), withAccess, PathId(..))
@@ -36,6 +37,7 @@ import Gargantext.API.Node
 import Gargantext.API.Prelude
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Viz.Graph.API
+-- import Gargantext.Database.Prelude (HasConfig(..))
 import Gargantext.Database.Admin.Types.Hyperdata
 import Gargantext.Database.Admin.Types.Node
 import Gargantext.Database.Admin.Types.Node (NodeId, CorpusId, AnnuaireId)
@@ -245,14 +247,17 @@ waitAPI n = do
 ----------------------------------------
 
 addCorpusWithQuery :: User -> GargServer New.AddWithQuery
-addCorpusWithQuery user cid =
+addCorpusWithQuery user cid = do
+  -- TODO gargantext.ini
+  -- _env <- view hasConfig
+  let limit = Just 10000
   serveJobsAPI $
     JobFunction (\q log ->
       let
         log' x = do
           printDebug "addToCorpusWithQuery" x
           liftBase $ log x
-      in New.addToCorpusWithQuery user cid q log'
+      in New.addToCorpusWithQuery user cid q limit log'
       )
 
 {-

src/Gargantext/Core/Text/Corpus/API.hs

@@ -36,7 +36,7 @@ get :: ExternalAPIs
     -> Query
     -> Maybe Limit
     -> IO [HyperdataDocument]
-get PubMed  _la q l = PUBMED.get q l -- EN only by default
+get PubMed  _la q l = PUBMED.get   q l -- EN only by default
 get HAL      la q l = HAL.get   la q l
 get IsTex    la q l = ISTEX.get la q l
 get Isidore  la q l = ISIDORE.get la (fromIntegral <$> l) (Just q) Nothing