Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
195
Issues
195
List
Board
Labels
Milestones
Merge Requests
12
Merge Requests
12
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
8ddb544b
Commit
8ddb544b
authored
Mar 03, 2025
by
Fabien Maniere
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch '428-dev-worker-fixes' into 'dev'
[worker] various fixes See merge request
!389
parents
3192b0f5
250efc5b
Pipeline
#7395
passed with stages
in 64 minutes and 28 seconds
Changes
7
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
33 additions
and
44 deletions
+33
-44
update-project-dependencies
bin/update-project-dependencies
+2
-2
cabal.project
cabal.project
+1
-1
cabal.project.freeze
cabal.project.freeze
+1
-1
API.hs
src/Gargantext/API.hs
+1
-1
Update.hs
src/Gargantext/API/Node/Update.hs
+25
-9
Worker.hs
src/Gargantext/Core/Worker.hs
+1
-1
stack.yaml
stack.yaml
+2
-29
No files found.
bin/update-project-dependencies
View file @
8ddb544b
...
@@ -18,8 +18,8 @@ fi
...
@@ -18,8 +18,8 @@ fi
# with the `sha256sum` result calculated on the `cabal.project` and
# with the `sha256sum` result calculated on the `cabal.project` and
# `cabal.project.freeze`. This ensures the files stay deterministic so that CI
# `cabal.project.freeze`. This ensures the files stay deterministic so that CI
# cache can kick in.
# cache can kick in.
expected_cabal_project_hash
=
"
ae8a153144dbf82c8676cd494d5afced39122fca8eac4facf20d3aec1c7c5dcc"
expected_cabal_project_hash
=
"
43ef700760f469f504cc78ccb7ca0fce80aba265a1bcac26e0db250b4b8562b6"
expected_cabal_project_freeze_hash
=
"
50f40d4fc0190b8e8645e2de3c3d40ddb3664339e7995f52f883d925685a9a49"
expected_cabal_project_freeze_hash
=
"
bf98c4373747e16acdba3e143ad67c978b53587918ee68b313237434dc21d56d"
cabal
--store-dir
=
$STORE_DIR
v2-build
--dry-run
cabal
--store-dir
=
$STORE_DIR
v2-build
--dry-run
...
...
cabal.project
View file @
8ddb544b
...
@@ -91,7 +91,7 @@ source-repository-package
...
@@ -91,7 +91,7 @@ source-repository-package
source
-
repository
-
package
source
-
repository
-
package
type
:
git
type
:
git
location
:
https
://
gitlab
.
iscpif
.
fr
/
gargantext
/
crawlers
/
openalex
.
git
location
:
https
://
gitlab
.
iscpif
.
fr
/
gargantext
/
crawlers
/
openalex
.
git
tag
:
a80e0ea57379d23f5e18a412606a71471b8ef681
tag
:
c86412b5b8713b2bdd63b2bed2a2259c5d143a88
source
-
repository
-
package
source
-
repository
-
package
type
:
git
type
:
git
...
...
cabal.project.freeze
View file @
8ddb544b
...
@@ -352,7 +352,7 @@ constraints: any.Boolean ==0.2.4,
...
@@ -352,7 +352,7 @@ constraints: any.Boolean ==0.2.4,
any.old-locale ==1.0.0.7,
any.old-locale ==1.0.0.7,
any.old-time ==1.1.0.4,
any.old-time ==1.1.0.4,
any.opaleye-textsearch ==0.2.0.0,
any.opaleye-textsearch ==0.2.0.0,
any.openalex ==0.
1
.0.0,
any.openalex ==0.
2
.0.0,
any.optics-core ==0.4.1.1,
any.optics-core ==0.4.1.1,
optics-core -explicit-generic-labels,
optics-core -explicit-generic-labels,
any.optics-extra ==0.4.2.1,
any.optics-extra ==0.4.2.1,
...
...
src/Gargantext/API.hs
View file @
8ddb544b
...
@@ -101,7 +101,7 @@ startGargantext mode port sf@(SettingsFile settingsFile) = withLoggerHoisted mod
...
@@ -101,7 +101,7 @@ startGargantext mode port sf@(SettingsFile settingsFile) = withLoggerHoisted mod
case
r
of
case
r
of
Right
True
->
pure
()
Right
True
->
pure
()
Right
False
->
panicTrace
$
Right
False
->
panicTrace
$
"You must run 'gargantext init "
<>
pack
settingsFile
<>
"You must run 'gargantext init
-c
"
<>
pack
settingsFile
<>
"' before running gargantext-server (only the first time)."
"' before running gargantext-server (only the first time)."
Left
err
->
panicTrace
$
"Unexpected exception:"
<>
show
err
Left
err
->
panicTrace
$
"Unexpected exception:"
<>
show
err
oneHour
=
Clock
.
fromNanoSecs
3600
_000_000_000
oneHour
=
Clock
.
fromNanoSecs
3600
_000_000_000
...
...
src/Gargantext/API/Node/Update.hs
View file @
8ddb544b
...
@@ -10,7 +10,6 @@ Portability : POSIX
...
@@ -10,7 +10,6 @@ Portability : POSIX
-}
-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeOperators #-}
module
Gargantext.API.Node.Update
module
Gargantext.API.Node.Update
where
where
...
@@ -67,6 +66,7 @@ updateNode nId (UpdateNodeParamsGraph
...
@@ -67,6 +66,7 @@ updateNode nId (UpdateNodeParamsGraph
(
UpdateNodeConfigGraph
metric
partitionMethod
bridgeMethod
strength
nt1
nt2
))
jobHandle
=
do
(
UpdateNodeConfigGraph
metric
partitionMethod
bridgeMethod
strength
nt1
nt2
))
jobHandle
=
do
markStarted
2
jobHandle
markStarted
2
jobHandle
markProgress
1
jobHandle
-- printDebug "Computing graph: " method
-- printDebug "Computing graph: " method
_
<-
recomputeGraph
nId
partitionMethod
bridgeMethod
(
Just
metric
)
(
Just
strength
)
nt1
nt2
True
_
<-
recomputeGraph
nId
partitionMethod
bridgeMethod
(
Just
metric
)
(
Just
strength
)
nt1
nt2
True
-- printDebug "Graph computed: " method
-- printDebug "Graph computed: " method
...
@@ -74,6 +74,7 @@ updateNode nId (UpdateNodeParamsGraph
...
@@ -74,6 +74,7 @@ updateNode nId (UpdateNodeParamsGraph
updateNode
nid1
(
LinkNodeReq
nt
nid2
)
jobHandle
=
do
updateNode
nid1
(
LinkNodeReq
nt
nid2
)
jobHandle
=
do
markStarted
2
jobHandle
markStarted
2
jobHandle
markProgress
1
jobHandle
_
<-
case
nt
of
_
<-
case
nt
of
NodeAnnuaire
->
pairing
nid2
nid1
Nothing
-- defaultList
NodeAnnuaire
->
pairing
nid2
nid1
Nothing
-- defaultList
NodeCorpus
->
pairing
nid1
nid2
Nothing
-- defaultList
NodeCorpus
->
pairing
nid1
nid2
Nothing
-- defaultList
...
@@ -84,7 +85,7 @@ updateNode nid1 (LinkNodeReq nt nid2) jobHandle = do
...
@@ -84,7 +85,7 @@ updateNode nid1 (LinkNodeReq nt nid2) jobHandle = do
-- | `Advanced` to update graphs
-- | `Advanced` to update graphs
updateNode
lId
(
UpdateNodeParamsList
Advanced
)
jobHandle
=
do
updateNode
lId
(
UpdateNodeParamsList
Advanced
)
jobHandle
=
do
markStarted
3
jobHandle
markStarted
4
jobHandle
corpusId
<-
view
node_parent_id
<$>
getNode
lId
corpusId
<-
view
node_parent_id
<$>
getNode
lId
markProgress
1
jobHandle
markProgress
1
jobHandle
...
@@ -92,7 +93,9 @@ updateNode lId (UpdateNodeParamsList Advanced) jobHandle = do
...
@@ -92,7 +93,9 @@ updateNode lId (UpdateNodeParamsList Advanced) jobHandle = do
_
<-
case
corpusId
of
_
<-
case
corpusId
of
Just
cId
->
do
Just
cId
->
do
_
<-
Metrics
.
updatePie
cId
(
Just
lId
)
NgramsTypes
.
Authors
Nothing
_
<-
Metrics
.
updatePie
cId
(
Just
lId
)
NgramsTypes
.
Authors
Nothing
markProgress
1
jobHandle
_
<-
Metrics
.
updateTree
cId
(
Just
lId
)
NgramsTypes
.
Institutes
MapTerm
_
<-
Metrics
.
updateTree
cId
(
Just
lId
)
NgramsTypes
.
Institutes
MapTerm
markProgress
1
jobHandle
_
<-
Metrics
.
updatePie
cId
(
Just
lId
)
NgramsTypes
.
Sources
Nothing
_
<-
Metrics
.
updatePie
cId
(
Just
lId
)
NgramsTypes
.
Sources
Nothing
pure
()
pure
()
Nothing
->
pure
()
Nothing
->
pure
()
...
@@ -108,6 +111,7 @@ updateNode lId (UpdateNodeParamsList _mode) jobHandle = do
...
@@ -108,6 +111,7 @@ updateNode lId (UpdateNodeParamsList _mode) jobHandle = do
_
<-
case
corpusId
of
_
<-
case
corpusId
of
Just
cId
->
do
Just
cId
->
do
_
<-
reIndexWith
cId
lId
NgramsTerms
(
Set
.
singleton
MapTerm
)
_
<-
reIndexWith
cId
lId
NgramsTerms
(
Set
.
singleton
MapTerm
)
markProgress
1
jobHandle
_
<-
updateNgramsOccurrences
cId
lId
_
<-
updateNgramsOccurrences
cId
lId
pure
()
pure
()
Nothing
->
pure
()
Nothing
->
pure
()
...
@@ -124,7 +128,7 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
...
@@ -124,7 +128,7 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
let
corpusId
=
fromMaybe
(
panicTrace
"no corpus id"
)
corpusId'
let
corpusId
=
fromMaybe
(
panicTrace
"no corpus id"
)
corpusId'
phy
<-
timeMeasured
"updateNode.flowPhyloAPI"
$
flowPhyloAPI
(
subConfigAPI2config
config
)
mbComputeHistory
corpusId
phy
<-
timeMeasured
"updateNode.flowPhyloAPI"
$
flowPhyloAPI
(
subConfigAPI2config
config
)
mbComputeHistory
corpusId
markProgress
2
jobHandle
markProgress
1
jobHandle
{-
{-
logStatus JobLog { _scst_succeeded = Just 2
logStatus JobLog { _scst_succeeded = Just 2
...
@@ -140,12 +144,12 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
...
@@ -140,12 +144,12 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
markComplete
jobHandle
markComplete
jobHandle
updateNode
tId
(
UpdateNodeParamsTexts
_mode
)
jobHandle
=
do
updateNode
tId
(
UpdateNodeParamsTexts
_mode
)
jobHandle
=
do
markStarted
3
jobHandle
markStarted
2
jobHandle
corpusId
<-
view
node_parent_id
<$>
getNode
tId
corpusId
<-
view
node_parent_id
<$>
getNode
tId
markProgress
1
jobHandle
markProgress
1
jobHandle
_
<-
case
corpusId
of
_
<-
case
corpusId
of
Just
cId
->
updateDocs
cId
Just
cId
->
updateDocs
cId
jobHandle
Nothing
->
do
Nothing
->
do
_
<-
panicTrace
"[G.A.N.Update] updateNode/UpdateNodeParamsText: no corpus Id given"
_
<-
panicTrace
"[G.A.N.Update] updateNode/UpdateNodeParamsText: no corpus Id given"
pure
()
pure
()
...
@@ -155,7 +159,7 @@ updateNode tId (UpdateNodeParamsTexts _mode) jobHandle = do
...
@@ -155,7 +159,7 @@ updateNode tId (UpdateNodeParamsTexts _mode) jobHandle = do
updateNode
tId
updateNode
tId
(
UpdateNodeParamsCorpus
methodGraph
methodPhylo
methodTexts
methodList
)
(
UpdateNodeParamsCorpus
methodGraph
methodPhylo
methodTexts
methodList
)
jobHandle
=
do
jobHandle
=
do
markStarted
3
jobHandle
markStarted
5
jobHandle
markProgress
1
jobHandle
markProgress
1
jobHandle
_
<-
getNode
tId
_
<-
getNode
tId
...
@@ -165,8 +169,11 @@ updateNode tId
...
@@ -165,8 +169,11 @@ updateNode tId
childNodeLists
<-
getChildrenByType
tId
NodeList
childNodeLists
<-
getChildrenByType
tId
NodeList
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsTexts
methodTexts
)
jobHandle
)
childTexts
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsTexts
methodTexts
)
jobHandle
)
childTexts
markProgress
1
jobHandle
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsGraph
methodGraph
)
jobHandle
)
childGraphs
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsGraph
methodGraph
)
jobHandle
)
childGraphs
markProgress
1
jobHandle
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodePhylo
methodPhylo
)
jobHandle
)
childPhylos
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodePhylo
methodPhylo
)
jobHandle
)
childPhylos
markProgress
1
jobHandle
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsList
methodList
)
jobHandle
)
childNodeLists
mapM_
(
\
cId
->
updateNode
cId
(
UpdateNodeParamsList
methodList
)
jobHandle
)
childNodeLists
markComplete
jobHandle
markComplete
jobHandle
...
@@ -175,14 +182,23 @@ updateNode _nId _p jobHandle = do
...
@@ -175,14 +182,23 @@ updateNode _nId _p jobHandle = do
simuLogs
jobHandle
10
simuLogs
jobHandle
10
------------------------------------------------------------------------
------------------------------------------------------------------------
updateDocs
::
(
HasNodeStory
env
err
m
)
updateDocs
::
(
HasNodeStory
env
err
m
=>
NodeId
->
m
()
,
MonadJobStatus
m
updateDocs
cId
=
do
,
MonadLogger
m
)
=>
NodeId
->
JobHandle
m
->
m
()
updateDocs
cId
jobHandle
=
do
markStarted
4
jobHandle
lId
<-
defaultList
cId
lId
<-
defaultList
cId
_
<-
reIndexWith
cId
lId
NgramsTerms
(
Set
.
singleton
MapTerm
)
_
<-
reIndexWith
cId
lId
NgramsTerms
(
Set
.
singleton
MapTerm
)
markProgress
1
jobHandle
_
<-
updateNgramsOccurrences
cId
lId
_
<-
updateNgramsOccurrences
cId
lId
markProgress
1
jobHandle
_
<-
updateContextScore
cId
lId
_
<-
updateContextScore
cId
lId
markProgress
1
jobHandle
_
<-
Metrics
.
updateChart'
cId
lId
NgramsTypes
.
Docs
Nothing
_
<-
Metrics
.
updateChart'
cId
lId
NgramsTypes
.
Docs
Nothing
markProgress
1
jobHandle
-- printDebug "updateContextsScore" (cId, lId, u)
-- printDebug "updateContextsScore" (cId, lId, u)
pure
()
pure
()
src/Gargantext/Core/Worker.hs
View file @
8ddb544b
...
@@ -137,7 +137,7 @@ notifyJobFailed env (W.State { name }) bm exc = do
...
@@ -137,7 +137,7 @@ notifyJobFailed env (W.State { name }) bm exc = do
let
ji
=
JobInfo
{
_ji_message_id
=
messageId
bm
let
ji
=
JobInfo
{
_ji_message_id
=
messageId
bm
,
_ji_mNode_id
=
getWorkerMNodeId
job
}
,
_ji_mNode_id
=
getWorkerMNodeId
job
}
let
jh
=
WorkerJobHandle
{
_w_job_info
=
ji
}
let
jh
=
WorkerJobHandle
{
_w_job_info
=
ji
}
runWorkerMonad
env
$
markFailed
(
Just
$
UnsafeMkHumanFriendlyErrorText
"Worker job failed"
)
jh
runWorkerMonad
env
$
markFailed
(
Just
$
UnsafeMkHumanFriendlyErrorText
$
"Worker job failed: "
<>
show
exc
)
jh
notifyJobKilled
::
(
HasWorkerBroker
,
HasCallStack
)
notifyJobKilled
::
(
HasWorkerBroker
,
HasCallStack
)
=>
WorkerEnv
=>
WorkerEnv
...
...
stack.yaml
View file @
8ddb544b
...
@@ -126,7 +126,6 @@
...
@@ -126,7 +126,6 @@
-
"
taggy-0.2.1"
-
"
taggy-0.2.1"
-
"
taggy-lens-0.1.2"
-
"
taggy-lens-0.1.2"
-
"
tasty-1.5"
-
"
tasty-1.5"
-
"
tasty-bench-0.4"
-
"
tasty-hspec-1.2.0.4"
-
"
tasty-hspec-1.2.0.4"
-
"
tasty-hunit-0.10.2"
-
"
tasty-hunit-0.10.2"
-
"
tasty-quickcheck-0.11"
-
"
tasty-quickcheck-0.11"
...
@@ -157,14 +156,6 @@
...
@@ -157,14 +156,6 @@
-
"
zip-2.0.1"
-
"
zip-2.0.1"
-
"
zip-archive-0.4.3.2"
-
"
zip-archive-0.4.3.2"
-
"
zlib-0.7.1.0"
-
"
zlib-0.7.1.0"
-
commit
:
2b5d69448557e89002c0179ea1aaf59bb757a6e3
git
:
"
https://github.com/AccelerateHS/accelerate-llvm.git"
subdirs
:
-
"
accelerate-llvm-native/"
-
commit
:
2b5d69448557e89002c0179ea1aaf59bb757a6e3
git
:
"
https://github.com/AccelerateHS/accelerate-llvm.git"
subdirs
:
-
"
accelerate-llvm/"
-
commit
:
334d05519436bb7f20f9926ec76418f5b8afa359
-
commit
:
334d05519436bb7f20f9926ec76418f5b8afa359
git
:
"
https://github.com/AccelerateHS/accelerate.git"
git
:
"
https://github.com/AccelerateHS/accelerate.git"
subdirs
:
subdirs
:
...
@@ -173,14 +164,6 @@
...
@@ -173,14 +164,6 @@
git
:
"
https://github.com/adinapoli/http-reverse-proxy.git"
git
:
"
https://github.com/adinapoli/http-reverse-proxy.git"
subdirs
:
subdirs
:
-
.
-
.
-
commit
:
7533a9ccd3bfe77141745f6b61039a26aaf5c83b
git
:
"
https://github.com/adinapoli/llvm-hs.git"
subdirs
:
-
"
llvm-hs"
-
commit
:
7533a9ccd3bfe77141745f6b61039a26aaf5c83b
git
:
"
https://github.com/adinapoli/llvm-hs.git"
subdirs
:
-
"
llvm-hs-pure"
-
commit
:
a110807651036ca2228a76507ee35bbf7aedf87a
-
commit
:
a110807651036ca2228a76507ee35bbf7aedf87a
git
:
"
https://github.com/alpmestan/accelerate-arithmetic.git"
git
:
"
https://github.com/alpmestan/accelerate-arithmetic.git"
subdirs
:
subdirs
:
...
@@ -245,7 +228,7 @@
...
@@ -245,7 +228,7 @@
git
:
"
https://gitlab.iscpif.fr/gargantext/crawlers/istex.git"
git
:
"
https://gitlab.iscpif.fr/gargantext/crawlers/istex.git"
subdirs
:
subdirs
:
-
.
-
.
-
commit
:
a80e0ea57379d23f5e18a412606a71471b8ef681
-
commit
:
c86412b5b8713b2bdd63b2bed2a2259c5d143a88
git
:
"
https://gitlab.iscpif.fr/gargantext/crawlers/openalex.git"
git
:
"
https://gitlab.iscpif.fr/gargantext/crawlers/openalex.git"
subdirs
:
subdirs
:
-
.
-
.
...
@@ -326,8 +309,6 @@ flags:
...
@@ -326,8 +309,6 @@ flags:
templatehaskell
:
true
templatehaskell
:
true
SHA
:
SHA
:
exe
:
false
exe
:
false
"
abstract-deque"
:
usecas
:
false
accelerate
:
accelerate
:
"
bounds-checks"
:
true
"
bounds-checks"
:
true
debug
:
false
debug
:
false
...
@@ -441,7 +422,7 @@ flags:
...
@@ -441,7 +422,7 @@ flags:
"
no-double-conversion"
:
false
"
no-double-conversion"
:
false
gargantext
:
gargantext
:
"
enable-benchmarks"
:
false
"
enable-benchmarks"
:
false
"
no-phylo-debug-logs"
:
tru
e
"
no-phylo-debug-logs"
:
fals
e
"
test-crypto"
:
false
"
test-crypto"
:
false
graphviz
:
graphviz
:
"
test-parsing"
:
false
"
test-parsing"
:
false
...
@@ -496,18 +477,12 @@ flags:
...
@@ -496,18 +477,12 @@ flags:
"
test-properties"
:
true
"
test-properties"
:
true
"
test-templates"
:
true
"
test-templates"
:
true
trustworthy
:
true
trustworthy
:
true
libffi
:
"
ghc-bundled-libffi"
:
true
libyaml
:
libyaml
:
"
no-unicode"
:
false
"
no-unicode"
:
false
"
system-libyaml"
:
false
"
system-libyaml"
:
false
linear
:
linear
:
herbie
:
false
herbie
:
false
"
template-haskell"
:
true
"
template-haskell"
:
true
"
llvm-hs"
:
debug
:
false
"
llvm-with-rtti"
:
false
"
shared-llvm"
:
true
lzma
:
lzma
:
pkgconfig
:
true
pkgconfig
:
true
massiv
:
massiv
:
...
@@ -620,8 +595,6 @@ flags:
...
@@ -620,8 +595,6 @@ flags:
transformers
:
true
transformers
:
true
tasty
:
tasty
:
unix
:
true
unix
:
true
"
tasty-bench"
:
tasty
:
true
"
tasty-golden"
:
"
tasty-golden"
:
"
build-example"
:
false
"
build-example"
:
false
"
text-format"
:
"
text-format"
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment