Merge branch 'dev' into 141-dev-node-stories-db-optimization

CHANGELOG.md

+## Version 0.0.6.2
+* [BACK][FIX] Ngrams List saved in database on halting
+* [BACK][FIX] Confluence on Graph
+* [BACK][FIX] HAL parser fix
+* [BACK][UPGRADE] ./bin/psql gargantext.ini < devops/postgres/upgrade/0.0.6.2.sql
+* [BACK][ADMIN] upgrade script, test ok
+
 ## Version 0.0.6.1
-* [FEAT] Spacy Server connection for French (and others) languages
-* [FEAT] At startup, check if gargantext.init script has been activated
-* [UPGRADE] Use the devops/postgres/upgrade/0.0.6.1.sql uprade script
-* [FIX] PubMed Parser with threadDelay
+* [BACK][FEAT] Spacy Server connection for French (and others) languages
+* [BACK][FEAT] At startup, check if gargantext.init script has been activated
+* [BACK][UPGRADE] Use the devops/postgres/upgrade/0.0.6.1.sql upgrade script
+* [BACK][FIX] PubMed Parser with threadDelay
 * [BACK][FIX] Hash to remove duplicates on filtered text
 
 ## Version 0.0.6
-* [OPTIM] Ngrams Table optmization. To upgrade:
-  1. `./bin/psql gargantext.ini < devops/postgresql/upgrade/0.0.6.sql`
-  2. in `stack --nix repl` run `runCmdReplEasy $ migrateFromDirToDb`
-* [FIX] Ngrams Table next button: loads only one time instead of twice previously
+* [BACK][OPTIM] Ngrams Table optmization. To upgrade:
+      1. `./bin/psql gargantext.ini < devops/postgresql/upgrade/0.0.6.sql`
+      2. in `stack --nix repl` run `runCmdReplEasy $ migrateFromDirToDb`
+* [FRONT][FIX] Ngrams Table next button: loads only one time instead of twice previously
 * [FRONT][FIX] Resize handler on Write Node
 * [FRONT][FIX] Do not highlight ngrams if maximum abstract length > 4500 characters
 

bin/gargantext-upgrade/Main.hs

@@ -24,6 +24,7 @@ import Gargantext.API.Admin.EnvTypes (DevEnv)
 import Gargantext.API.Dev (withDevEnv, runCmdDev)
 import Gargantext.API.Node () -- instances only
 import Gargantext.API.Prelude (GargError)
+import Gargantext.API.Ngrams.Tools (migrateFromDirToDb)
 import Gargantext.Core (HasDBid(toDBid))
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Database.Action.Flow (getOrMk_RootWithCorpus)
@@ -42,10 +43,13 @@ import qualified Data.List as List (cycle, concat, take, unlines)
 main :: IO ()
 main = do
 
-  let ___ = putStrLn $ List.concat $ List.take 72 $ List.cycle ["_"]
+  let ___ = putStrLn
+          $ List.concat
+          $ List.take 72
+          $ List.cycle ["_"]
 
   ___
-  putStrLn "GarganText upgrade to version 0.0.5"
+  putStrLn "GarganText upgrade to version 0.0.6"
   ___
 
   params@[iniPath] <- getArgs
@@ -57,9 +61,6 @@ main = do
            [ "Your Database defined in gargantext.ini will be upgraded."
            , "We stronlgy recommend you to make a backup using pg_dump."
            , ""
-           , "If you encounter issues, please report your bugs here:"
-           , "https://gitlab.iscpif.fr/gargantext/haskell-gargantext/issues/101"
-           , ""
            , "Press ENTER if you want to continue, CTRL+C if you want to stop."
            ]
 
@@ -68,31 +69,9 @@ main = do
   cfg       <- readConfig         iniPath
   let secret = _gc_secretkey cfg
 
-  let
-    contextsTriggers :: Cmd GargError ()
-    contextsTriggers = do
-      (masterUserId, _masterRootId, masterCorpusId)
-                  <- getOrMk_RootWithCorpus (UserName userMaster)
-                                            (Left corpusMasterName)
-                                            (Nothing :: Maybe HyperdataCorpus)
-      masterListId <- getOrMkList masterCorpusId masterUserId
-      _triggers    <- initLastTriggers masterListId
-      pure ()
-
-
   withDevEnv iniPath $ \env -> do
     -- First upgrade the Database Schema
-    _ <- runCmdDev env sqlSchema
-
-    -- Then upgrade the triggers
-    _ <- runCmdDev env (initFirstTriggers secret :: Cmd GargError [Int64])
-    _ <- runCmdDev env (contextsTriggers         :: Cmd GargError ())
-
-   -- Move nodes to contexts table
-    _ <- runCmdDev env sqlNodes2Contexts
-
-   -- Update the hashes
-    _ <- runCmdDev env sqlUpdateTriggerHash
+    _ <- runCmdDev env (migrateFromDirToDb :: Cmd GargError ())
 
     ___
     putStrLn "Uprade done with success !"
@@ -100,7 +79,7 @@ main = do
     pure ()
 
 
-
+{-
 sqlUpdateTriggerHash :: Cmd'' DevEnv IOException Int64
 sqlUpdateTriggerHash = do
   execPGSQuery query ()
@@ -283,3 +262,5 @@ sqlSchema = do
         CREATE INDEX IF NOT EXISTS context_node_node_id_idx ON public.context_node_ngrams USING btree (node_id);
 
   |]
+
+-}

devops/upgrade/0.0.6.1.sh

+#!/bin/bash
+
+# To be executed at the root of the project
+# To upgrade from 0.0.5.9 to 0.0.6.2
+
+
+sudo apt update
+sudo apt -yy upgrade
+
+sudo sed -i "s/buster/bullseye/g" /etc/apt/sources.list
+sudo apt update
+sudo apt -yy dist-upgrade
+
+git pull origin dev
+./bin/install
+
+# Database upgrade
+echo "0.0.6.0 SQL upgrade"
+./bin/psql gargantext.ini < devops/postgres/upgrade/0.0.6.0.sql
+
+echo "Executing script haskell upgrade"
+~/.local/bin/gargantext-upgrade
+
+echo "0.0.6.1 SQL upgrade"
+./bin/psql gargantext.ini < devops/postgres/upgrade/0.0.6.1.sql
+
+sudo -i -u postgres bash << EOF 
+pg_dumpall > /tmp/backup.dump
+EOF
+
+sudo sed -i "s/bullseye/bookworm/g" /etc/apt/sources.list
+sudo apt update
+sudo apt -yy dist-upgrade
+sudo apt install -y postgresql-14 libpq-dev
+sudo apt remove --purge postgresql-11 postgresql-13
+sudo apt autoremove
+
+
+sudo -i -u postgres bash << EOF 
+psql < /tmp/backup.dump
+EOF
+
+sed -i "s/DB_PORT = 5432/DB_PORT = 5434/" gargantext.ini
+
+# be sure the DB password is the right one
+DBPASS=$(grep "DB_PASS" gargantext.ini | sed "s/^.*= //")
+echo $DBPASS
+
+sudo -i -u postgres psql << EOF 
+ALTER ROLE gargantua password '${DBPASS}';
+EOF
+
+echo "0.0.6.2 SQL upgrade"
+./bin/psql gargantext.ini < devops/postgres/upgrade/0.0.6.2.sql
+
+# Make sure compilation is ok
+./bin/install
+
+echo "Upgrade is over"
+
+
+
+

gargantext.cabal

@@ -5,7 +5,7 @@ cabal-version: 1.12
 -- see: https://github.com/sol/hpack
 
 name:           gargantext
-version:        0.0.6.1
+version: 0.0.6.2
 synopsis:       Search, map, share
 description:    Please see README.md
 category:       Data

package.yaml

@@ -6,7 +6,7 @@ name: gargantext
 #         | | | +-----  Layers * : New versions with API additions
 #         | | | | +---  Layers * : New versions without API breaking changes
 #         | | | | |
-version: '0.0.6.1'
+version: '0.0.6.2'
 synopsis: Search, map, share
 description: Please see README.md
 category: Data

src/Gargantext/API/Ngrams/Tools.hs

@@ -199,7 +199,7 @@ getCoocByNgrams' f (Diagonal diag) m =
 ------------------------------------------
 
 
-migrateFromDirToDb :: (CmdM env err m, HasNodeStory env err m)
+migrateFromDirToDb :: (CmdM env err m) -- , HasNodeStory env err m)
                    => m ()
 migrateFromDirToDb = do
   pool <- view connPool

src/Gargantext/Core/Viz/Graph/Tools.hs

@@ -132,7 +132,10 @@ cooc2graphWith' doPartitions distance threshold strength myCooc = do
         (as, bs) = List.unzip $ Map.keys distanceMap
         n' = Set.size $ Set.fromList $ as <> bs
     bridgeness' = bridgeness (fromIntegral nodesApprox) partitions distanceMap
-    confluence' = Map.empty -- BAC.computeConfluences 3 (Map.keys bridgeness') True
+    -- indices of bridgeness seem to start at 1, while computeConfluences
+    -- expects 0-based indexing.
+    -- ks = map (\(a, b) -> (a-1, b-1)) (Map.keys bridgeness')
+    confluence' = Map.empty -- Map.mapKeys (\(a, b) -> (a+1, b+1)) $ BAC.computeConfluences 3 ks True
                -- confluence (Map.keys bridgeness') 3 True False
   seq bridgeness' $ printDebug "bridgeness OK" ()
   seq confluence' $ printDebug "confluence OK" ()

stack.yaml

@@ -35,7 +35,7 @@ extra-deps:
 - git: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude.git
   commit: 08096a4913572cf22762fa77613340207ec6d9fd
 - git: https://gitlab.iscpif.fr/gargantext/gargantext-graph.git
-  commit: f41ee8b53c3264e5aa5adc06b2e5b293d2a8c474
+  commit: 642b9ec7ffa59a5db7b2ec7b24436e07309dc097
   # Data Mining Libs
 - git: https://github.com/delanoe/data-time-segment.git
   commit: 10a416b9f6c443866b36479c3441ebb3bcdeb7ef
@@ -76,7 +76,7 @@ extra-deps:
 - git: https://gitlab.iscpif.fr/gargantext/crawlers/istex.git
   commit: a34bb341236d82cf3d488210bc1d8448a98f5808
 - git: https://gitlab.iscpif.fr/gargantext/crawlers/hal.git
-  commit: 9a43470241690a19c1c381c42a62c5dd4e28dff2
+  commit: 4aad6c62483de21e8b61845c0c487aaf8677da96
 - git: https://gitlab.iscpif.fr/gargantext/crawlers/isidore.git
   commit: 3db385e767d2100d8abe900833c6e7de3ac55e1b
 - git: https://gitlab.iscpif.fr/gargantext/crawlers/arxiv-api.git