Merge branch '428-dev-worker-fixes' into 'dev'

[worker] various fixes

See merge request !389

bin/update-project-dependencies

@@ -18,8 +18,8 @@ fi
 # with the `sha256sum` result calculated on the `cabal.project` and
 # `cabal.project.freeze`. This ensures the files stay deterministic so that CI
 # cache can kick in.
-expected_cabal_project_hash="ae8a153144dbf82c8676cd494d5afced39122fca8eac4facf20d3aec1c7c5dcc"
-expected_cabal_project_freeze_hash="50f40d4fc0190b8e8645e2de3c3d40ddb3664339e7995f52f883d925685a9a49"
+expected_cabal_project_hash="43ef700760f469f504cc78ccb7ca0fce80aba265a1bcac26e0db250b4b8562b6"  
+expected_cabal_project_freeze_hash="bf98c4373747e16acdba3e143ad67c978b53587918ee68b313237434dc21d56d" 
 
 
 cabal --store-dir=$STORE_DIR v2-build --dry-run

cabal.project

@@ -91,7 +91,7 @@ source-repository-package
 source-repository-package
     type: git
     location: https://gitlab.iscpif.fr/gargantext/crawlers/openalex.git
-    tag: a80e0ea57379d23f5e18a412606a71471b8ef681
+    tag: c86412b5b8713b2bdd63b2bed2a2259c5d143a88
 
 source-repository-package
     type: git

cabal.project.freeze

@@ -352,7 +352,7 @@ constraints: any.Boolean ==0.2.4,
              any.old-locale ==1.0.0.7,
              any.old-time ==1.1.0.4,
              any.opaleye-textsearch ==0.2.0.0,
-             any.openalex ==0.1.0.0,
+             any.openalex ==0.2.0.0,
              any.optics-core ==0.4.1.1,
              optics-core -explicit-generic-labels,
              any.optics-extra ==0.4.2.1,

src/Gargantext/API.hs

@@ -101,7 +101,7 @@ startGargantext mode port sf@(SettingsFile settingsFile) = withLoggerHoisted mod
           case r of
             Right True  -> pure ()
             Right False -> panicTrace $
-              "You must run 'gargantext init " <> pack settingsFile <>
+              "You must run 'gargantext init -c " <> pack settingsFile <>
               "' before running gargantext-server (only the first time)."
             Left err -> panicTrace $ "Unexpected exception:" <> show err
         oneHour = Clock.fromNanoSecs 3600_000_000_000

src/Gargantext/API/Node/Update.hs

@@ -10,7 +10,6 @@ Portability : POSIX
 -}
 
 {-# LANGUAGE TypeApplications #-}
-{-# LANGUAGE TypeOperators    #-}
 
 module Gargantext.API.Node.Update
       where
@@ -67,6 +66,7 @@ updateNode nId (UpdateNodeParamsGraph
                     (UpdateNodeConfigGraph metric partitionMethod bridgeMethod strength nt1 nt2)) jobHandle = do
 
   markStarted 2 jobHandle
+  markProgress 1 jobHandle
   -- printDebug "Computing graph: " method
   _ <- recomputeGraph nId partitionMethod bridgeMethod (Just metric) (Just strength) nt1 nt2 True
   -- printDebug "Graph computed: " method
@@ -74,6 +74,7 @@ updateNode nId (UpdateNodeParamsGraph
 
 updateNode nid1 (LinkNodeReq nt nid2) jobHandle = do
   markStarted 2 jobHandle
+  markProgress 1 jobHandle
   _ <- case nt of
     NodeAnnuaire -> pairing nid2 nid1 Nothing -- defaultList
     NodeCorpus   -> pairing nid1 nid2 Nothing -- defaultList
@@ -84,7 +85,7 @@ updateNode nid1 (LinkNodeReq nt nid2) jobHandle = do
 
 -- | `Advanced` to update graphs
 updateNode lId (UpdateNodeParamsList Advanced) jobHandle = do
-  markStarted 3 jobHandle
+  markStarted 4 jobHandle
   corpusId <- view node_parent_id <$> getNode lId
 
   markProgress 1 jobHandle
@@ -92,7 +93,9 @@ updateNode lId (UpdateNodeParamsList Advanced) jobHandle = do
   _ <- case corpusId of
     Just cId -> do
       _ <- Metrics.updatePie cId (Just lId) NgramsTypes.Authors Nothing
+      markProgress 1 jobHandle
       _ <- Metrics.updateTree cId (Just lId) NgramsTypes.Institutes MapTerm
+      markProgress 1 jobHandle
       _ <- Metrics.updatePie cId (Just lId) NgramsTypes.Sources Nothing
       pure ()
     Nothing  -> pure ()
@@ -108,6 +111,7 @@ updateNode lId (UpdateNodeParamsList _mode) jobHandle = do
   _ <- case corpusId of
     Just cId -> do
       _ <- reIndexWith cId lId NgramsTerms (Set.singleton MapTerm)
+      markProgress 1 jobHandle
       _ <- updateNgramsOccurrences cId lId
       pure ()
     Nothing  -> pure ()
@@ -124,7 +128,7 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
   let corpusId = fromMaybe (panicTrace "no corpus id") corpusId'
 
   phy <- timeMeasured "updateNode.flowPhyloAPI" $ flowPhyloAPI (subConfigAPI2config config) mbComputeHistory corpusId
-  markProgress 2 jobHandle
+  markProgress 1 jobHandle
 
 {-
   logStatus JobLog { _scst_succeeded = Just 2
@@ -140,12 +144,12 @@ updateNode phyloId (UpdateNodePhylo config) jobHandle = do
   markComplete jobHandle
 
 updateNode tId (UpdateNodeParamsTexts _mode) jobHandle = do
-  markStarted 3 jobHandle
+  markStarted 2 jobHandle
   corpusId <- view node_parent_id <$> getNode tId
   markProgress 1 jobHandle
 
   _ <- case corpusId of
-    Just cId -> updateDocs cId
+    Just cId -> updateDocs cId jobHandle
     Nothing  -> do
       _ <- panicTrace "[G.A.N.Update] updateNode/UpdateNodeParamsText: no corpus Id given"
       pure ()
@@ -155,7 +159,7 @@ updateNode tId (UpdateNodeParamsTexts _mode) jobHandle = do
 updateNode tId
     (UpdateNodeParamsCorpus methodGraph methodPhylo methodTexts methodList)
     jobHandle = do
-  markStarted 3 jobHandle
+  markStarted 5 jobHandle
   markProgress 1 jobHandle
 
   _ <- getNode tId
@@ -165,8 +169,11 @@ updateNode tId
   childNodeLists <- getChildrenByType tId NodeList
 
   mapM_ (\cId -> updateNode cId (UpdateNodeParamsTexts methodTexts) jobHandle) childTexts
+  markProgress 1 jobHandle
   mapM_ (\cId -> updateNode cId (UpdateNodeParamsGraph methodGraph) jobHandle) childGraphs
+  markProgress 1 jobHandle
   mapM_ (\cId -> updateNode cId (UpdateNodePhylo methodPhylo) jobHandle) childPhylos
+  markProgress 1 jobHandle
   mapM_ (\cId -> updateNode cId (UpdateNodeParamsList methodList) jobHandle) childNodeLists
 
   markComplete jobHandle
@@ -175,14 +182,23 @@ updateNode _nId _p jobHandle = do
   simuLogs jobHandle 10
 ------------------------------------------------------------------------
 
-updateDocs :: (HasNodeStory env err m)
-            => NodeId -> m ()
-updateDocs cId = do
+updateDocs :: ( HasNodeStory env err m
+              , MonadJobStatus m
+              , MonadLogger m )
+           => NodeId
+           -> JobHandle m
+           -> m ()
+updateDocs cId jobHandle = do
+  markStarted 4 jobHandle
   lId <- defaultList cId
   _ <- reIndexWith cId lId NgramsTerms (Set.singleton MapTerm)
+  markProgress 1 jobHandle
   _ <- updateNgramsOccurrences cId lId
+  markProgress 1 jobHandle
   _ <- updateContextScore      cId lId
+  markProgress 1 jobHandle
   _ <- Metrics.updateChart'    cId lId NgramsTypes.Docs Nothing
+  markProgress 1 jobHandle
   -- printDebug "updateContextsScore" (cId, lId, u)
   pure ()
 

src/Gargantext/Core/Worker.hs

@@ -137,7 +137,7 @@ notifyJobFailed env (W.State { name }) bm exc = do
   let ji = JobInfo { _ji_message_id = messageId bm
                    , _ji_mNode_id = getWorkerMNodeId job }
   let jh = WorkerJobHandle { _w_job_info = ji }
-  runWorkerMonad env $ markFailed (Just $ UnsafeMkHumanFriendlyErrorText "Worker job failed") jh
+  runWorkerMonad env $ markFailed (Just $ UnsafeMkHumanFriendlyErrorText $ "Worker job failed: " <> show exc) jh
 
 notifyJobKilled :: (HasWorkerBroker, HasCallStack)
                  => WorkerEnv

stack.yaml

@@ -126,7 +126,6 @@
 - "taggy-0.2.1"
 - "taggy-lens-0.1.2"
 - "tasty-1.5"
-- "tasty-bench-0.4"
 - "tasty-hspec-1.2.0.4"
 - "tasty-hunit-0.10.2"
 - "tasty-quickcheck-0.11"
@@ -157,14 +156,6 @@
 - "zip-2.0.1"
 - "zip-archive-0.4.3.2"
 - "zlib-0.7.1.0"
-- commit: 2b5d69448557e89002c0179ea1aaf59bb757a6e3
-  git: "https://github.com/AccelerateHS/accelerate-llvm.git"
-  subdirs:
-  - "accelerate-llvm-native/"
-- commit: 2b5d69448557e89002c0179ea1aaf59bb757a6e3
-  git: "https://github.com/AccelerateHS/accelerate-llvm.git"
-  subdirs:
-  - "accelerate-llvm/"
 - commit: 334d05519436bb7f20f9926ec76418f5b8afa359
   git: "https://github.com/AccelerateHS/accelerate.git"
   subdirs:
@@ -173,14 +164,6 @@
   git: "https://github.com/adinapoli/http-reverse-proxy.git"
   subdirs:
   - .
-- commit: 7533a9ccd3bfe77141745f6b61039a26aaf5c83b
-  git: "https://github.com/adinapoli/llvm-hs.git"
-  subdirs:
-  - "llvm-hs"
-- commit: 7533a9ccd3bfe77141745f6b61039a26aaf5c83b
-  git: "https://github.com/adinapoli/llvm-hs.git"
-  subdirs:
-  - "llvm-hs-pure"
 - commit: a110807651036ca2228a76507ee35bbf7aedf87a
   git: "https://github.com/alpmestan/accelerate-arithmetic.git"
   subdirs:
@@ -245,7 +228,7 @@
   git: "https://gitlab.iscpif.fr/gargantext/crawlers/istex.git"
   subdirs:
   - .
-- commit: a80e0ea57379d23f5e18a412606a71471b8ef681
+- commit: c86412b5b8713b2bdd63b2bed2a2259c5d143a88
   git: "https://gitlab.iscpif.fr/gargantext/crawlers/openalex.git"
   subdirs:
   - .
@@ -326,8 +309,6 @@ flags:
     templatehaskell: true
   SHA:
     exe: false
-  "abstract-deque":
-    usecas: false
   accelerate:
     "bounds-checks": true
     debug: false
@@ -441,7 +422,7 @@ flags:
     "no-double-conversion": false
   gargantext:
     "enable-benchmarks": false
-    "no-phylo-debug-logs": true
+    "no-phylo-debug-logs": false
     "test-crypto": false
   graphviz:
     "test-parsing": false
@@ -496,18 +477,12 @@ flags:
     "test-properties": true
     "test-templates": true
     trustworthy: true
-  libffi:
-    "ghc-bundled-libffi": true
   libyaml:
     "no-unicode": false
     "system-libyaml": false
   linear:
     herbie: false
     "template-haskell": true
-  "llvm-hs":
-    debug: false
-    "llvm-with-rtti": false
-    "shared-llvm": true
   lzma:
     pkgconfig: true
   massiv:
@@ -620,8 +595,6 @@ flags:
     transformers: true
   tasty:
     unix: true
-  "tasty-bench":
-    tasty: true
   "tasty-golden":
     "build-example": false
   "text-format":