[FIX] Refresh Index: reindex of term list, textflow and schedule job

src/Gargantext/API.hs

@@ -34,7 +34,7 @@ module Gargantext.API
       where
 
 import Control.Concurrent
-import Control.Exception (catch, finally, SomeException, displayException)
+import Control.Exception (catch, finally, SomeException, displayException, IOException)
 import Control.Lens
 import Control.Monad.Except
 import Control.Monad.Reader (runReaderT)
@@ -56,7 +56,8 @@ import Gargantext.API.Ngrams (saveNodeStoryImmediate)
 import Gargantext.API.Routes
 import Gargantext.API.Server (server)
 import Gargantext.Core.NodeStory
-import Gargantext.Database.GargDB (refreshNgramsMaterializedView)
+import Gargantext.Database.Prelude (Cmd)
+import Gargantext.Database.Action.Metrics.NgramsByContext (refreshNgramsMaterialized)
 import Gargantext.Prelude hiding (putStrLn)
 import Network.HTTP.Types hiding (Query)
 import Network.Wai
@@ -130,11 +131,13 @@ schedulePeriodicActions env =
     refreshDBViews :: Cron.Schedule ()
     refreshDBViews = do
       let doRefresh = do
-            res <- DB.runCmd env refreshNgramsMaterializedView
+            res <- DB.runCmd env (refreshNgramsMaterialized :: Cmd IOException ())
             case res of
               Left e   -> liftIO $ putStrLn $ pack ("Refreshing Ngrams materialized view failed: " <> displayException e)
-              Right () -> pure ()
-      Cron.addJob doRefresh "5 * * * *"
+              Right () ->  do
+                _ <- liftIO $ putStrLn $ pack "Refresh Index Database done"
+                pure ()
+      Cron.addJob doRefresh "* 2 * * *"
 
 ----------------------------------------------------------------------
 

src/Gargantext/API/Ngrams/List.hs

@@ -38,6 +38,7 @@ import Gargantext.Core.Text.Terms.WithList (buildPatterns, termsInText)
 import Gargantext.Core.Types.Main (ListType(..))
 import Gargantext.Database.Action.Flow (saveDocNgramsWith)
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
+import Gargantext.Database.Action.Metrics.NgramsByContext (refreshNgramsMaterialized)
 import Gargantext.Database.Admin.Types.Hyperdata.Document
 import Gargantext.Database.Admin.Types.Node
 import Gargantext.Database.Query.Table.Node (getNode)
@@ -49,7 +50,6 @@ import Gargantext.Database.Types (Indexed(..))
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
 import Servant
--- import Servant.Job.Async
 import qualified Data.ByteString.Lazy as BSL
 import qualified Data.Csv as Csv
 import qualified Data.HashMap.Strict as HashMap
@@ -180,7 +180,7 @@ reIndexWith cId lId nt lts = do
 
   -- Saving the indexation in database
   _ <- mapM (saveDocNgramsWith lId) ngramsByDoc
-
+  _ <- refreshNgramsMaterialized
   pure ()
 
 toIndexedNgrams :: HashMap Text NgramsId -> Text -> Maybe (Indexed Int Ngrams)

src/Gargantext/Database/Action/Metrics.hs

@@ -35,6 +35,7 @@ import Gargantext.Core.Types.Query (Limit(..))
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Action.Metrics.NgramsByContext (getContextsByNgramsOnlyUser{-, getTficfWith-})
 import Gargantext.Database.Admin.Config (userMaster)
+import Gargantext.Database.Action.Metrics.NgramsByContext (refreshNgramsMaterialized)
 import Gargantext.Database.Admin.Types.Node (ListId, CorpusId)
 import Gargantext.Database.Query.Table.Node (defaultList)
 import Gargantext.Database.Query.Table.Node.Select
@@ -125,7 +126,11 @@ updateNgramsOccurrences' cId maybeListId maybeLimit tabType = do
   let fields = map (\t-> QualifiedIdentifier Nothing t)
              $ map Text.pack ["int4", "int4","text","int4","int4"]
 
-  map (\(Only a) -> a) <$> runPGSQuery queryInsert (Only $ Values fields toInsert)
+  res <- map (\(Only a) -> a) <$> runPGSQuery queryInsert (Only $ Values fields toInsert)
+
+  -- _ <- map (\(Only a) -> a) <$> runPGSQuery [sql|refresh materialized view context_node_ngrams_view;|] ()
+  _ <- refreshNgramsMaterialized
+  pure res
 
 
 

src/Gargantext/Database/Action/Metrics/NgramsByContext.hs

@@ -18,6 +18,7 @@ module Gargantext.Database.Action.Metrics.NgramsByContext
 
 -- import Debug.Trace (trace)
 --import Data.Map.Strict.Patch (PatchMap, Replace, diff)
+import Control.Monad (void)
 import Data.HashMap.Strict (HashMap)
 import Data.Map.Strict (Map)
 import Data.Set (Set)
@@ -29,7 +30,7 @@ import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core
 import Gargantext.Data.HashMap.Strict.Utils as HM
 import Gargantext.Database.Admin.Types.Node (ListId, CorpusId, NodeId(..), ContextId, MasterCorpusId, NodeType(NodeDocument), UserCorpusId, DocId)
-import Gargantext.Database.Prelude (Cmd, runPGSQuery)
+import Gargantext.Database.Prelude (Cmd, runPGSQuery, execPGSQuery)
 import Gargantext.Database.Schema.Ngrams (ngramsTypeId, NgramsType(..))
 import Gargantext.Prelude
 import qualified Data.HashMap.Strict              as HM
@@ -217,12 +218,6 @@ queryNgramsOccurrencesOnlyByContextUser_withSample' = [sql|
       GROUP BY ng.id
   |]
 
-
-
-
-
-
-
 ------------------------------------------------------------------------
 getContextsByNgramsOnlyUser :: HasDBid NodeType
                          => CorpusId
@@ -397,3 +392,17 @@ queryNgramsByContextMaster' = [sql|
   SELECT m.id, m.terms FROM nodesByNgramsMaster m
     RIGHT JOIN contextsByNgramsUser u ON u.id = m.id
   |]
+
+-- | Refreshes the \"context_node_ngrams_view\" materialized view.
+-- This function will be run :
+--  - periodically
+--  - at reindex stage
+--  - at the end of each text flow
+
+refreshNgramsMaterialized :: Cmd err ()
+refreshNgramsMaterialized = void $ execPGSQuery refreshNgramsMaterializedQuery ()
+  where
+    refreshNgramsMaterializedQuery :: DPS.Query
+    refreshNgramsMaterializedQuery = [sql| refresh materialized view context_node_ngrams_view; |] 
+
+

src/Gargantext/Database/GargDB.hs

@@ -19,14 +19,12 @@ module Gargantext.Database.GargDB
 
 import Control.Exception
 import Control.Lens (view)
-import Control.Monad (void)
 import Control.Monad.Reader (MonadReader)
-import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Data.Aeson (ToJSON, toJSON)
 import Data.Text (Text)
 import Data.Tuple.Extra (both)
 import GHC.IO (FilePath)
-import Gargantext.Database.Prelude (HasConfig(..), Cmd, execPGSQuery)
+import Gargantext.Database.Prelude (HasConfig(..))
 import Gargantext.Prelude
 import Gargantext.Prelude.Config
 import Gargantext.Prelude.Crypto.Hash
@@ -145,11 +143,11 @@ writeFile a = do
 
 -- | Example to read a file with Type
 readGargFile :: ( MonadReader  env m
-            , HasConfig    env
-            , MonadBase IO     m
-            , ReadFile         a
-            )
-         => FilePath -> m a
+                , HasConfig    env
+                , MonadBase IO     m
+                , ReadFile         a
+                )
+                 => FilePath -> m a
 readGargFile fp = do
   dataPath <- view $ hasConfig . gc_datafilepath
   liftBase $ readFile' $ toFilePath dataPath fp
@@ -157,9 +155,9 @@ readGargFile fp = do
 ---
 
 rmFile :: ( MonadReader env m
-              , MonadBase IO m
-              , HasConfig env
-              )
+          , MonadBase IO m
+          , HasConfig env
+          )
            => FilePath -> m ()
 rmFile = onDisk_1 SD.removeFile
 
@@ -169,8 +167,11 @@ cpFile = onDisk_2 SD.copyFile
 
 ---
 
-mvFile :: (MonadReader env m, MonadBase IO m, HasConfig env)
-       => FilePath -> FilePath -> m ()
+mvFile :: ( MonadReader env m
+          , MonadBase IO m
+          , HasConfig env
+          )
+           => FilePath -> FilePath -> m ()
 mvFile fp1 fp2 = do
   cpFile fp1 fp2
   rmFile fp1
@@ -178,10 +179,10 @@ mvFile fp1 fp2 = do
 
 ------------------------------------------------------------------------
 onDisk_1 :: ( MonadReader env m
-              , MonadBase IO m
-              , HasConfig env
-              )
-        => (FilePath -> IO ()) -> FilePath -> m ()
+            , MonadBase IO m
+            , HasConfig env
+            )
+             => (FilePath -> IO ()) -> FilePath -> m ()
 onDisk_1 action fp = do
   dataPath <- view $ hasConfig . gc_datafilepath
   liftBase $ action (toFilePath dataPath fp) `catch` handleExists
@@ -209,9 +210,3 @@ onDisk_2 action fp1 fp2 = do
         | isDoesNotExistError e = return ()
         | otherwise = throwIO e
 ------------------------------------------------------------------------
-
--- | Refreshes the \"context_node_ngrams_view\" materialized view. This
--- function will be run periodically.
-refreshNgramsMaterializedView :: Cmd IOException ()
-refreshNgramsMaterializedView =
-  void $ execPGSQuery [sql| refresh materialized view context_node_ngrams_view; |] ()