[config] move worker timeouts to the .toml file

gargantext-settings.toml_toModify

@@ -159,6 +159,11 @@ default_visibility_timeout = 1
 # default delay before job is visible to the worker
 default_delay = 0
 
+# default timeout (in seconds)
+default_job_timeout = 60
+# default timeout for "long" jobs (in seconds)
+long_job_timeout = 3000
+
 # if you leave the same credentials as in [database] section above,
 #  workers will try to set up the `gargantext_pgmq` database
 #  automatically

src/Gargantext/Core/Config/Worker.hs

@@ -38,8 +38,13 @@ type WorkerName = Text
 data WorkerSettings =
   WorkerSettings {
       _wsDatabase     :: !PGS.ConnectInfo
-      -- After this number of seconds, the job will be available again.
 
+    -- | default job timeout, in seconds
+    , _wsDefaultJobTimeout :: ~Int
+    -- | default "long" job timeout, in seconds
+    , _wsLongJobTimeout :: ~Int
+      
+      -- After this number of seconds, the job will be available again.
       -- You can set timeout for each job individually and this is the
       -- preferred method over using defaultVt.
     , _wsDefaultVisibilityTimeout :: PGMQ.VisibilityTimeout
@@ -53,8 +58,12 @@ instance FromValue WorkerSettings where
     dbConfig <- reqKey "database"
     _wsDefinitions <- reqKey "definitions"
     _wsDefaultVisibilityTimeout <- reqKey "default_visibility_timeout"
+    _wsDefaultJobTimeout <- reqKey "default_job_timeout"
+    _wsLongJobTimeout <- reqKey "long_job_timeout"
     defaultDelay <- reqKey "default_delay"
     return $ WorkerSettings { _wsDatabase = unTOMLConnectInfo dbConfig
+                            , _wsDefaultJobTimeout
+                            , _wsLongJobTimeout
                             , _wsDefinitions
                             , _wsDefaultVisibilityTimeout
                             , _wsDefaultDelay = B.TimeoutS defaultDelay }
@@ -63,6 +72,8 @@ instance ToValue WorkerSettings where
 instance ToTable WorkerSettings where
   toTable (WorkerSettings { .. }) =
     table [ "database" .= TOMLConnectInfo _wsDatabase
+          , "default_job_timeout" .= _wsDefaultJobTimeout
+          , "long_job_timeout" .= _wsLongJobTimeout
           , "default_visibility_timeout" .= _wsDefaultVisibilityTimeout
           , "default_delay" .= B._TimeoutS _wsDefaultDelay
           , "definitions" .= _wsDefinitions ]

src/Gargantext/Core/Worker/Jobs.hs

@@ -45,40 +45,34 @@ sendJobWithCfg gcConfig job = do
     Just wd -> do
       b <- initBrokerWithDBCreate (gcConfig ^. gc_database_config) ws
       let queueName = _wdQueue wd
-      let job' = (updateJobData job $ W.mkDefaultSendJob' b queueName job) { W.delay = _wsDefaultDelay }
+      let job' = (updateJobData ws job $ W.mkDefaultSendJob' b queueName job) { W.delay = _wsDefaultDelay }
       withLogger (gcConfig ^. gc_logging) $ \ioL ->
         $(logLoc) ioL DEBUG $ "[sendJob] sending job " <> show job <> " (delay " <> show (W.delay job') <> ")"
       W.sendJob' job'
 
--- | In seconds
-longJobTimeout :: Int
-longJobTimeout = 3000
-
--- | In seconds
-defaultJobTimeout :: Int
-defaultJobTimeout = 60
-
 -- | We want to fine-tune job metadata parameters, for each job type
-updateJobData :: Job -> SendJob -> SendJob
-updateJobData (AddCorpusTempFileAsync {}) sj = sj { W.timeout = longJobTimeout
-                                                  , W.toStrat = WT.TSDelete
+updateJobData :: WorkerSettings -> Job -> SendJob -> SendJob
+updateJobData ws (AddCorpusTempFileAsync {}) sj = withLongTimeout ws $ sj { W.toStrat = WT.TSDelete
                                                                           , W.resendOnKill = False }
-updateJobData (AddCorpusWithQuery {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (AddToAnnuaireWithForm {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (AddWithFile {}) sj = sj { W.timeout = longJobTimeout
-                                       , W.toStrat = WT.TSDelete
+updateJobData ws (AddCorpusWithQuery {}) sj = withLongTimeout ws sj
+updateJobData ws (AddToAnnuaireWithForm {}) sj = withLongTimeout ws sj
+updateJobData ws (AddWithFile {}) sj = withLongTimeout ws $ sj { W.toStrat = WT.TSDelete
                                                                , W.resendOnKill = False }
-updateJobData (DocumentsFromWriteNodes {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (FrameCalcUpload {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (JSONPost {}) sj = sj { W.timeout = longJobTimeout
-                                    , W.toStrat = WT.TSDelete
+updateJobData ws (DocumentsFromWriteNodes {}) sj = withLongTimeout ws sj
+updateJobData ws (FrameCalcUpload {}) sj = withLongTimeout ws sj
+updateJobData ws (JSONPost {}) sj = withLongTimeout ws $ sj { W.toStrat = WT.TSDelete
                                                             , W.resendOnKill = False }
-updateJobData (NgramsPostCharts {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (RecomputeGraph {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (UpdateNode {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (UploadDocument {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (ImportRemoteDocuments {}) sj = sj { W.timeout = longJobTimeout }
-updateJobData (ImportRemoteTerms {}) sj = sj { W.timeout = longJobTimeout }
+updateJobData ws (NgramsPostCharts {}) sj = withLongTimeout ws sj
+updateJobData ws (RecomputeGraph {}) sj = withLongTimeout ws sj
+updateJobData ws (UpdateNode {}) sj = withLongTimeout ws sj
+updateJobData ws (UploadDocument {}) sj = withLongTimeout ws sj
+updateJobData ws (ImportRemoteDocuments {}) sj = withLongTimeout ws sj
+updateJobData ws (ImportRemoteTerms {}) sj = withLongTimeout ws sj
 -- | ForgotPasswordAsync, PostNodeAsync
-updateJobData _ sj = sj { W.resendOnKill = False
-                        , W.timeout = defaultJobTimeout }
+updateJobData ws _ sj = withDefaultTimeout ws $ sj { W.resendOnKill = False }
+
+withDefaultTimeout :: WorkerSettings -> SendJob -> SendJob
+withDefaultTimeout (WorkerSettings { _wsDefaultJobTimeout }) sj = sj { W.timeout = _wsDefaultJobTimeout }
+
+withLongTimeout :: WorkerSettings -> SendJob -> SendJob
+withLongTimeout (WorkerSettings { _wsLongJobTimeout }) sj = sj { W.timeout = _wsLongJobTimeout }

test-data/test_config.toml

@@ -92,6 +92,11 @@ default_visibility_timeout = 1
 # default delay before job is visible to the worker
 default_delay = 1
 
+# default timeout (in seconds)
+default_job_timeout = 60
+# default timeout for "long" jobs (in seconds)
+long_job_timeout = 3000
+
 # NOTE This is overridden by Test.Database.Setup
 [worker.database]
 host = "127.0.0.1"