Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
191
Issues
191
List
Board
Labels
Milestones
Merge Requests
8
Merge Requests
8
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
b74070c9
Verified
Commit
b74070c9
authored
Jun 30, 2025
by
Przemyslaw Kaminski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[config] move worker timeouts to the .toml file
parent
62e0d9cd
Pipeline
#7712
failed with stages
in 17 minutes and 42 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
47 additions
and
32 deletions
+47
-32
gargantext-settings.toml_toModify
gargantext-settings.toml_toModify
+5
-0
Worker.hs
src/Gargantext/Core/Config/Worker.hs
+12
-1
Jobs.hs
src/Gargantext/Core/Worker/Jobs.hs
+25
-31
test_config.toml
test-data/test_config.toml
+5
-0
No files found.
gargantext-settings.toml_toModify
View file @
b74070c9
...
@@ -159,6 +159,11 @@ default_visibility_timeout = 1
...
@@ -159,6 +159,11 @@ default_visibility_timeout = 1
# default delay before job is visible to the worker
# default delay before job is visible to the worker
default_delay = 0
default_delay = 0
# default timeout (in seconds)
default_job_timeout = 60
# default timeout for "long" jobs (in seconds)
long_job_timeout = 3000
# if you leave the same credentials as in [database] section above,
# if you leave the same credentials as in [database] section above,
# workers will try to set up the `gargantext_pgmq` database
# workers will try to set up the `gargantext_pgmq` database
# automatically
# automatically
...
...
src/Gargantext/Core/Config/Worker.hs
View file @
b74070c9
...
@@ -38,8 +38,13 @@ type WorkerName = Text
...
@@ -38,8 +38,13 @@ type WorkerName = Text
data
WorkerSettings
=
data
WorkerSettings
=
WorkerSettings
{
WorkerSettings
{
_wsDatabase
::
!
PGS
.
ConnectInfo
_wsDatabase
::
!
PGS
.
ConnectInfo
-- After this number of seconds, the job will be available again.
-- | default job timeout, in seconds
,
_wsDefaultJobTimeout
::
~
Int
-- | default "long" job timeout, in seconds
,
_wsLongJobTimeout
::
~
Int
-- After this number of seconds, the job will be available again.
-- You can set timeout for each job individually and this is the
-- You can set timeout for each job individually and this is the
-- preferred method over using defaultVt.
-- preferred method over using defaultVt.
,
_wsDefaultVisibilityTimeout
::
PGMQ
.
VisibilityTimeout
,
_wsDefaultVisibilityTimeout
::
PGMQ
.
VisibilityTimeout
...
@@ -53,8 +58,12 @@ instance FromValue WorkerSettings where
...
@@ -53,8 +58,12 @@ instance FromValue WorkerSettings where
dbConfig
<-
reqKey
"database"
dbConfig
<-
reqKey
"database"
_wsDefinitions
<-
reqKey
"definitions"
_wsDefinitions
<-
reqKey
"definitions"
_wsDefaultVisibilityTimeout
<-
reqKey
"default_visibility_timeout"
_wsDefaultVisibilityTimeout
<-
reqKey
"default_visibility_timeout"
_wsDefaultJobTimeout
<-
reqKey
"default_job_timeout"
_wsLongJobTimeout
<-
reqKey
"long_job_timeout"
defaultDelay
<-
reqKey
"default_delay"
defaultDelay
<-
reqKey
"default_delay"
return
$
WorkerSettings
{
_wsDatabase
=
unTOMLConnectInfo
dbConfig
return
$
WorkerSettings
{
_wsDatabase
=
unTOMLConnectInfo
dbConfig
,
_wsDefaultJobTimeout
,
_wsLongJobTimeout
,
_wsDefinitions
,
_wsDefinitions
,
_wsDefaultVisibilityTimeout
,
_wsDefaultVisibilityTimeout
,
_wsDefaultDelay
=
B
.
TimeoutS
defaultDelay
}
,
_wsDefaultDelay
=
B
.
TimeoutS
defaultDelay
}
...
@@ -63,6 +72,8 @@ instance ToValue WorkerSettings where
...
@@ -63,6 +72,8 @@ instance ToValue WorkerSettings where
instance
ToTable
WorkerSettings
where
instance
ToTable
WorkerSettings
where
toTable
(
WorkerSettings
{
..
})
=
toTable
(
WorkerSettings
{
..
})
=
table
[
"database"
.=
TOMLConnectInfo
_wsDatabase
table
[
"database"
.=
TOMLConnectInfo
_wsDatabase
,
"default_job_timeout"
.=
_wsDefaultJobTimeout
,
"long_job_timeout"
.=
_wsLongJobTimeout
,
"default_visibility_timeout"
.=
_wsDefaultVisibilityTimeout
,
"default_visibility_timeout"
.=
_wsDefaultVisibilityTimeout
,
"default_delay"
.=
B
.
_TimeoutS
_wsDefaultDelay
,
"default_delay"
.=
B
.
_TimeoutS
_wsDefaultDelay
,
"definitions"
.=
_wsDefinitions
]
,
"definitions"
.=
_wsDefinitions
]
...
...
src/Gargantext/Core/Worker/Jobs.hs
View file @
b74070c9
...
@@ -45,40 +45,34 @@ sendJobWithCfg gcConfig job = do
...
@@ -45,40 +45,34 @@ sendJobWithCfg gcConfig job = do
Just
wd
->
do
Just
wd
->
do
b
<-
initBrokerWithDBCreate
(
gcConfig
^.
gc_database_config
)
ws
b
<-
initBrokerWithDBCreate
(
gcConfig
^.
gc_database_config
)
ws
let
queueName
=
_wdQueue
wd
let
queueName
=
_wdQueue
wd
let
job'
=
(
updateJobData
job
$
W
.
mkDefaultSendJob'
b
queueName
job
)
{
W
.
delay
=
_wsDefaultDelay
}
let
job'
=
(
updateJobData
ws
job
$
W
.
mkDefaultSendJob'
b
queueName
job
)
{
W
.
delay
=
_wsDefaultDelay
}
withLogger
(
gcConfig
^.
gc_logging
)
$
\
ioL
->
withLogger
(
gcConfig
^.
gc_logging
)
$
\
ioL
->
$
(
logLoc
)
ioL
DEBUG
$
"[sendJob] sending job "
<>
show
job
<>
" (delay "
<>
show
(
W
.
delay
job'
)
<>
")"
$
(
logLoc
)
ioL
DEBUG
$
"[sendJob] sending job "
<>
show
job
<>
" (delay "
<>
show
(
W
.
delay
job'
)
<>
")"
W
.
sendJob'
job'
W
.
sendJob'
job'
-- | In seconds
longJobTimeout
::
Int
longJobTimeout
=
3000
-- | In seconds
defaultJobTimeout
::
Int
defaultJobTimeout
=
60
-- | We want to fine-tune job metadata parameters, for each job type
-- | We want to fine-tune job metadata parameters, for each job type
updateJobData
::
Job
->
SendJob
->
SendJob
updateJobData
::
WorkerSettings
->
Job
->
SendJob
->
SendJob
updateJobData
(
AddCorpusTempFileAsync
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
updateJobData
ws
(
AddCorpusTempFileAsync
{})
sj
=
withLongTimeout
ws
$
sj
{
W
.
toStrat
=
WT
.
TSDelete
,
W
.
toStrat
=
WT
.
TSDelete
,
W
.
resendOnKill
=
False
}
,
W
.
resendOnKill
=
False
}
updateJobData
ws
(
AddCorpusWithQuery
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
AddCorpusWithQuery
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
AddToAnnuaireWithForm
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
AddToAnnuaireWithForm
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
AddWithFile
{})
sj
=
withLongTimeout
ws
$
sj
{
W
.
toStrat
=
WT
.
TSDelete
updateJobData
(
AddWithFile
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
,
W
.
resendOnKill
=
False
}
,
W
.
toStrat
=
WT
.
TSDelete
updateJobData
ws
(
DocumentsFromWriteNodes
{})
sj
=
withLongTimeout
ws
sj
,
W
.
resendOnKill
=
False
}
updateJobData
ws
(
FrameCalcUpload
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
DocumentsFromWriteNodes
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
JSONPost
{})
sj
=
withLongTimeout
ws
$
sj
{
W
.
toStrat
=
WT
.
TSDelete
updateJobData
(
FrameCalcUpload
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
,
W
.
resendOnKill
=
False
}
updateJobData
(
JSONPost
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
updateJobData
ws
(
NgramsPostCharts
{})
sj
=
withLongTimeout
ws
sj
,
W
.
toStrat
=
WT
.
TSDelete
updateJobData
ws
(
RecomputeGraph
{})
sj
=
withLongTimeout
ws
sj
,
W
.
resendOnKill
=
False
}
updateJobData
ws
(
UpdateNode
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
NgramsPostCharts
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
UploadDocument
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
RecomputeGraph
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
ImportRemoteDocuments
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
UpdateNode
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
ws
(
ImportRemoteTerms
{})
sj
=
withLongTimeout
ws
sj
updateJobData
(
UploadDocument
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
(
ImportRemoteDocuments
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
updateJobData
(
ImportRemoteTerms
{})
sj
=
sj
{
W
.
timeout
=
longJobTimeout
}
-- | ForgotPasswordAsync, PostNodeAsync
-- | ForgotPasswordAsync, PostNodeAsync
updateJobData
_
sj
=
sj
{
W
.
resendOnKill
=
False
updateJobData
ws
_
sj
=
withDefaultTimeout
ws
$
sj
{
W
.
resendOnKill
=
False
}
,
W
.
timeout
=
defaultJobTimeout
}
withDefaultTimeout
::
WorkerSettings
->
SendJob
->
SendJob
withDefaultTimeout
(
WorkerSettings
{
_wsDefaultJobTimeout
})
sj
=
sj
{
W
.
timeout
=
_wsDefaultJobTimeout
}
withLongTimeout
::
WorkerSettings
->
SendJob
->
SendJob
withLongTimeout
(
WorkerSettings
{
_wsLongJobTimeout
})
sj
=
sj
{
W
.
timeout
=
_wsLongJobTimeout
}
test-data/test_config.toml
View file @
b74070c9
...
@@ -92,6 +92,11 @@ default_visibility_timeout = 1
...
@@ -92,6 +92,11 @@ default_visibility_timeout = 1
# default delay before job is visible to the worker
# default delay before job is visible to the worker
default_delay
=
1
default_delay
=
1
# default timeout (in seconds)
default_job_timeout
=
60
# default timeout for "long" jobs (in seconds)
long_job_timeout
=
3000
# NOTE This is overridden by Test.Database.Setup
# NOTE This is overridden by Test.Database.Setup
[worker.database]
[worker.database]
host
=
"127.0.0.1"
host
=
"127.0.0.1"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment