Squashed commit of the following:

commit f775d4a3
Merge: 76b557ea d2f4b89d
Author: Alexandre Delanoë <devel+git@delanoe.org>
Date:   Tue Oct 8 16:27:53 2024 +0200

    Merge remote-tracking branch 'origin/dev-guidelines-update' into dev

commit 76b557ea
Merge: 2925d008 50c77ea2
Author: Alexandre Delanoë <devel+git@delanoe.org>
Date:   Tue Oct 8 16:27:27 2024 +0200

    Merge remote-tracking branch 'origin/304-dev-pubmed-api-not-in-toml' into dev

commit d2f4b89d
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Tue Oct 8 15:37:54 2024 +0200

    DEVELOPER_GUIDELINES: update about git amend

    This is the result of Autumn workshop 2024

commit 50c77ea2
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Tue Oct 8 15:15:28 2024 +0200

    [notifications] fix for send

    sendNonblocking threw an error initially. I just do a compromise and
    timeout the normal send (which blocks infinitely sometimes)

commit 025b80b6
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Tue Oct 8 14:10:56 2024 +0200

    [docker] fix network: host, fix caddyfile

commit 2925d008
Author: Christian Merten <christian@merten.dev>
Date:   Tue Oct 8 10:34:17 2024 +0200

    fix arbitrary instance

commit e8fb3db6
Author: Christian Merten <christian@merten.dev>
Date:   Tue Oct 8 10:13:40 2024 +0200

    fix: re-add lost instances

commit b86d2e61
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Tue Oct 8 10:09:18 2024 +0200

    [toml] remove pubmed api key from config

    It's set up in user settings instead and has been for a long time.

commit c06de5ef
Merge: ab710337 a0ec337b
Author: Christian Merten <christian@merten.dev>
Date:   Tue Oct 8 09:35:55 2024 +0200

    Merge remote-tracking branch 'gitlab/dev' into cm/update-corpus-button

commit ab710337
Author: Christian Merten <christian@merten.dev>
Date:   Fri Apr 26 22:32:33 2024 +0200

    feat: update corpus endpoint

DEVELOPER_GUIDELINES.md

@@ -254,6 +254,19 @@ Ideally, we could have the following process, divided in 4 phases:
   marked with an `approved` label on Gitlab. The old `triage` label should be removed;
 - _Implementation_: Finally, the ticket gets implemented. This concludes the lifecycle.
 
+
+### Try to amend or squash if you rebase with `dev`
+
+Example here: https://gitlab.iscpif.fr/gargantext/haskell-gargantext/issues/322
+
+MR here: https://gitlab.iscpif.fr/gargantext/haskell-gargantext/merge_requests/258
+
+With amending/squashing, gitlab produces:
+- for new users on the issue, a large commit set with changes
+- for users already following the issue, gitlab produces a "compare
+  with previous version" like this one:
+  https://gitlab.iscpif.fr/gargantext/haskell-gargantext/merge_requests/258#note_9863
+
 ## Conclusion
 
 We have presented a comprehensive overview on the set of best practices we should put in place within

bin/gargantext-cli/CLI/Ini.hs

@@ -76,8 +76,7 @@ convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo =
                                                    , _jc_max_docs_scrapers = _gc_max_docs_scrapers
                                                    , _jc_js_job_timeout = _gc_js_job_timeout
                                                    , _jc_js_id_timeout = _gc_js_id_timeout }
-                    , _gc_apis = CTypes.APIsConfig { _ac_pubmed_api_key = _gc_pubmed_api_key
-                                                   , _ac_epo_api_url = _gc_epo_api_url
+                    , _gc_apis = CTypes.APIsConfig { _ac_epo_api_url = _gc_epo_api_url
                                                    , _ac_scrapyd_url }
                     , _gc_log_level = LevelDebug
                     }

devops/docker/Caddyfile

+# http disables automatic https
+http://localhost:8108 {
+	root * /srv/purescript-gargantext/dist
+	file_server
+}
+

devops/docker/docker-compose.yaml

@@ -3,7 +3,6 @@ version: '3'
 services:
   caddy:
     image: caddy:alpine
-    network: host
     ports:
       - 8108:8108
     volumes:

gargantext-settings.toml_toModify

@@ -59,9 +59,6 @@ data_filepath = FILEPATH_TO_CHANGE
 
 [apis]
 
-[apis.pubmed]
-api_key = ENTER_PUBMED_API_KEY
-
 [apis.epo]
 api_url = EPO_API_URL
 

src/Gargantext/API/Node/Update.hs

@@ -34,8 +34,9 @@ import Gargantext.Database.Action.Flow (reIndexWith)
 import Gargantext.Database.Action.Flow.Pairing (pairing)
 import Gargantext.Database.Action.Metrics (updateNgramsOccurrences, updateContextScore)
 import Gargantext.Database.Admin.Types.Hyperdata.Phylo ( HyperdataPhylo(HyperdataPhylo) )
-import Gargantext.Database.Admin.Types.Node ( NodeId, NodeType(NodeCorpus, NodeAnnuaire) )
-import Gargantext.Database.Query.Table.Node (defaultList, getNode)
+import Gargantext.Database.Admin.Types.Node ( NodeId,
+    NodeType(NodeCorpus, NodeAnnuaire, NodeTexts, NodeGraph, NodePhylo, NodeList) )
+import Gargantext.Database.Query.Table.Node (defaultList, getNode, getChildrenByType)
 import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
 import Gargantext.Database.Schema.Node (node_parent_id)
 import Gargantext.Prelude
@@ -58,7 +59,8 @@ updateNode :: (HasNodeStory env err m
            -> UpdateNodeParams
            -> JobHandle m
            -> m ()
-updateNode nId (UpdateNodeParamsGraph metric partitionMethod bridgeMethod strength nt1 nt2) jobHandle = do
+updateNode nId (UpdateNodeParamsGraph
+                    (UpdateNodeConfigGraph metric partitionMethod bridgeMethod strength nt1 nt2)) jobHandle = do
 
   markStarted 2 jobHandle
   -- printDebug "Computing graph: " method
@@ -144,6 +146,24 @@ updateNode tId (UpdateNodeParamsTexts _mode) jobHandle = do
 
   markComplete jobHandle
 
+updateNode tId
+    (UpdateNodeParamsCorpus methodGraph methodPhylo methodTexts methodList)
+    jobHandle = do
+  markStarted 3 jobHandle
+  markProgress 1 jobHandle
+
+  _ <- getNode tId
+  childTexts <- getChildrenByType tId NodeTexts
+  childGraphs <- getChildrenByType tId NodeGraph
+  childPhylos <- getChildrenByType tId NodePhylo
+  childNodeLists <- getChildrenByType tId NodeList
+
+  mapM_ (\cId -> updateNode cId (UpdateNodeParamsTexts methodTexts) jobHandle) childTexts
+  mapM_ (\cId -> updateNode cId (UpdateNodeParamsGraph methodGraph) jobHandle) childGraphs
+  mapM_ (\cId -> updateNode cId (UpdateNodePhylo methodPhylo) jobHandle) childPhylos
+  mapM_ (\cId -> updateNode cId (UpdateNodeParamsList methodList) jobHandle) childNodeLists
+
+  markComplete jobHandle
 
 updateNode _nId _p jobHandle = do
   simuLogs jobHandle 10

src/Gargantext/API/Node/Update/Types.hs

@@ -17,16 +17,15 @@ import Test.QuickCheck.Arbitrary ( Arbitrary(arbitrary) )
 ------------------------------------------------------------------------
 data UpdateNodeParams = UpdateNodeParamsList  { methodList  :: !Method      }
 
-                      | UpdateNodeParamsGraph { methodGraphMetric        :: !GraphMetric
-                                              , methodGraphClustering    :: !PartitionMethod
-                                              , methodGraphBridgeness    :: !BridgenessMethod
-                                              , methodGraphEdgesStrength :: !Strength
-                                              , methodGraphNodeType1     :: !NgramsType
-                                              , methodGraphNodeType2     :: !NgramsType
-                                              }
+                      | UpdateNodeParamsGraph { methodGraph :: !UpdateNodeConfigGraph }
 
                       | UpdateNodeParamsTexts { methodTexts :: !Granularity }
 
+                      | UpdateNodeParamsCorpus { methodGraph :: !UpdateNodeConfigGraph
+                                               , methodPhylo :: !PhyloSubConfigAPI
+                                               , methodTexts :: !Granularity
+                                               , methodList  :: !Method }
+
                       | UpdateNodeParamsBoard { methodBoard :: !Charts      }
 
                       | LinkNodeReq           { nodeType    :: !NodeType
@@ -47,6 +46,16 @@ data Granularity = NewNgrams | NewTexts | Both
 data Charts = Sources | Authors | Institutes | Ngrams | All
     deriving (Generic, Eq, Ord, Enum, Bounded)
 
+------------------------------------------------------------------------
+data UpdateNodeConfigGraph = UpdateNodeConfigGraph { methodGraphMetric        :: !GraphMetric
+                                                   , methodGraphClustering    :: !PartitionMethod
+                                                   , methodGraphBridgeness    :: !BridgenessMethod
+                                                   , methodGraphEdgesStrength :: !Strength
+                                                   , methodGraphNodeType1     :: !NgramsType
+                                                   , methodGraphNodeType2     :: !NgramsType
+                                                   }
+    deriving (Generic)
+
 ------------------------------------------------------------------------
 -- TODO unPrefix "pn_" FromJSON, ToJSON, ToSchema, adapt frontend.
 instance FromJSON  UpdateNodeParams where
@@ -59,7 +68,7 @@ instance ToSchema  UpdateNodeParams
 instance Arbitrary UpdateNodeParams where
   arbitrary = do
     l <- UpdateNodeParamsList  <$> arbitrary
-    g <- UpdateNodeParamsGraph <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
+    g <- UpdateNodeParamsGraph <$> arbitrary
     t <- UpdateNodeParamsTexts <$> arbitrary
     b <- UpdateNodeParamsBoard <$> arbitrary
     elements [l,g,t,b]
@@ -82,4 +91,18 @@ instance ToSchema  Charts
 instance Arbitrary Charts where
   arbitrary = elements [ minBound .. maxBound ]
 
+instance FromJSON UpdateNodeConfigGraph
+instance ToJSON UpdateNodeConfigGraph
+instance ToSchema UpdateNodeConfigGraph
+instance Arbitrary UpdateNodeConfigGraph where
+  arbitrary = do
+    methodGraphMetric        <- arbitrary
+    methodGraphClustering    <- arbitrary
+    methodGraphBridgeness    <- arbitrary
+    methodGraphEdgesStrength <- arbitrary
+    methodGraphNodeType1     <- arbitrary
+    methodGraphNodeType2     <- arbitrary
+    return $ UpdateNodeConfigGraph methodGraphMetric methodGraphClustering methodGraphBridgeness
+        methodGraphEdgesStrength methodGraphNodeType1 methodGraphNodeType2
+
 ------------------------------------------------------------------------

src/Gargantext/Core/AsyncUpdates/CentralExchange.hs

@@ -29,7 +29,8 @@ import Gargantext.Core.AsyncUpdates.CentralExchange.Types
 import Gargantext.Core.Config.Types (NotificationsConfig(..))
 import Gargantext.Prelude
 import Gargantext.System.Logging (LogLevel(..), withLogger, logMsg)
-import Nanomsg (Pull(..), Push(..), bind, connect, recv, sendNonblocking, withSocket)
+import Nanomsg (Pull(..), Push(..), bind, connect, recv, send, withSocket)
+import System.Timeout (timeout)
     
 {-
 
@@ -74,7 +75,8 @@ gServer (NotificationsConfig { .. }) = do
             Just _ujp@(UpdateJobProgress _s) -> do
               -- logMsg ioLogger DEBUG $ "[central_exchange] " <> show ujp
               -- send the same message that we received
-              void $ sendNonblocking s_dispatcher r
+              -- void $ sendNonblocking s_dispatcher r
+              void $ timeout 100_000 $ send s_dispatcher r
             Just (UpdateTreeFirstLevel node_id) -> do
               logMsg ioLogger INFO $ "[central_exchange] update tree: " <> show node_id
               -- putText $ "[central_exchange] sending that to the dispatcher: " <> show node_id
@@ -92,7 +94,8 @@ gServer (NotificationsConfig { .. }) = do
               -- gargantext-server but maybe it can be a separate
               -- process, independent of the server.
               -- send the same message that we received
-              void $ sendNonblocking s_dispatcher r
+              -- void $ sendNonblocking s_dispatcher r
+              void $ timeout 100_000 $ send s_dispatcher r
             _ -> logMsg ioLogger DEBUG $ "[central_exchange] unknown message"
       
 
@@ -104,4 +107,6 @@ notify (NotificationsConfig { _nc_central_exchange_connect }) ceMessage = do
       let str = Aeson.encode ceMessage
       withLogger () $ \ioLogger ->
         logMsg ioLogger DEBUG $ "[central_exchange] sending: " <> (T.unpack $ TE.decodeUtf8 $ BSL.toStrict str)
-      void $ sendNonblocking s $ BSL.toStrict str
+      -- err <- sendNonblocking s $ BSL.toStrict str
+      -- putText $ "[notify] err: " <> show err
+      void $ timeout 100_000 $ send s $ BSL.toStrict str

src/Gargantext/Core/Config/Types.hs

@@ -15,7 +15,6 @@ TODO-SECURITY: Critical
 
 module Gargantext.Core.Config.Types
   ( APIsConfig(..)
-  , ac_pubmed_api_key
   , ac_epo_api_url
   , ac_scrapyd_url
   , CORSOrigin(..)
@@ -299,13 +298,11 @@ makeLenses ''JobsConfig
 
 
 data APIsConfig =
-  APIsConfig { _ac_pubmed_api_key :: !Text
-             , _ac_epo_api_url    :: !Text
+  APIsConfig { _ac_epo_api_url    :: !Text
              , _ac_scrapyd_url    :: !BaseUrl }
   deriving (Generic, Show)
 instance FromValue APIsConfig where
   fromValue = parseTableFromValue $ do
-    _ac_pubmed_api_key <- reqKeyOf "pubmed" $ parseTableFromValue $ reqKey "api_key"
     _ac_epo_api_url <- reqKeyOf "epo" $ parseTableFromValue $ reqKey "api_url"
     scrapyd_url <- reqKeyOf "scrapyd" $ parseTableFromValue $ reqKey "url"
     _ac_scrapyd_url <-
@@ -316,8 +313,7 @@ instance FromValue APIsConfig where
 instance ToValue APIsConfig where
   toValue = defaultTableToValue
 instance ToTable APIsConfig where
-  toTable (APIsConfig { .. }) = table [ "pubmed" .= table [ "api_key" .= _ac_pubmed_api_key ]
-                                      , "epo" .= table [ "api_url" .= _ac_epo_api_url ]
+  toTable (APIsConfig { .. }) = table [ "epo" .= table [ "api_url" .= _ac_epo_api_url ]
                                       , "scrapyd" .= table [ "url" .= showBaseUrl _ac_scrapyd_url ]
                                       ]