Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
120
Issues
120
List
Board
Labels
Milestones
Merge Requests
7
Merge Requests
7
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
7ab982ea
Commit
7ab982ea
authored
May 16, 2023
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[FIX] Refresh Index: reindex of term list, textflow and schedule job
parent
80e4a368
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
50 additions
and
38 deletions
+50
-38
API.hs
src/Gargantext/API.hs
+8
-5
List.hs
src/Gargantext/API/Ngrams/List.hs
+2
-2
Metrics.hs
src/Gargantext/Database/Action/Metrics.hs
+6
-1
NgramsByContext.hs
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
+16
-7
GargDB.hs
src/Gargantext/Database/GargDB.hs
+18
-23
No files found.
src/Gargantext/API.hs
View file @
7ab982ea
...
...
@@ -34,7 +34,7 @@ module Gargantext.API
where
import
Control.Concurrent
import
Control.Exception
(
catch
,
finally
,
SomeException
,
displayException
)
import
Control.Exception
(
catch
,
finally
,
SomeException
,
displayException
,
IOException
)
import
Control.Lens
import
Control.Monad.Except
import
Control.Monad.Reader
(
runReaderT
)
...
...
@@ -56,7 +56,8 @@ import Gargantext.API.Ngrams (saveNodeStoryImmediate)
import
Gargantext.API.Routes
import
Gargantext.API.Server
(
server
)
import
Gargantext.Core.NodeStory
import
Gargantext.Database.GargDB
(
refreshNgramsMaterializedView
)
import
Gargantext.Database.Prelude
(
Cmd
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Prelude
hiding
(
putStrLn
)
import
Network.HTTP.Types
hiding
(
Query
)
import
Network.Wai
...
...
@@ -130,11 +131,13 @@ schedulePeriodicActions env =
refreshDBViews
::
Cron
.
Schedule
()
refreshDBViews
=
do
let
doRefresh
=
do
res
<-
DB
.
runCmd
env
refreshNgramsMaterializedView
res
<-
DB
.
runCmd
env
(
refreshNgramsMaterialized
::
Cmd
IOException
()
)
case
res
of
Left
e
->
liftIO
$
putStrLn
$
pack
(
"Refreshing Ngrams materialized view failed: "
<>
displayException
e
)
Right
()
->
pure
()
Cron
.
addJob
doRefresh
"5 * * * *"
Right
()
->
do
_
<-
liftIO
$
putStrLn
$
pack
"Refresh Index Database done"
pure
()
Cron
.
addJob
doRefresh
"* 2 * * *"
----------------------------------------------------------------------
...
...
src/Gargantext/API/Ngrams/List.hs
View file @
7ab982ea
...
...
@@ -38,6 +38,7 @@ import Gargantext.Core.Text.Terms.WithList (buildPatterns, termsInText)
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Database.Action.Flow
(
saveDocNgramsWith
)
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Database.Admin.Types.Hyperdata.Document
import
Gargantext.Database.Admin.Types.Node
import
Gargantext.Database.Query.Table.Node
(
getNode
)
...
...
@@ -49,7 +50,6 @@ import Gargantext.Database.Types (Indexed(..))
import
Gargantext.Prelude
import
Gargantext.Utils.Jobs
(
serveJobsAPI
,
MonadJobStatus
(
..
))
import
Servant
-- import Servant.Job.Async
import
qualified
Data.ByteString.Lazy
as
BSL
import
qualified
Data.Csv
as
Csv
import
qualified
Data.HashMap.Strict
as
HashMap
...
...
@@ -180,7 +180,7 @@ reIndexWith cId lId nt lts = do
-- Saving the indexation in database
_
<-
mapM
(
saveDocNgramsWith
lId
)
ngramsByDoc
_
<-
refreshNgramsMaterialized
pure
()
toIndexedNgrams
::
HashMap
Text
NgramsId
->
Text
->
Maybe
(
Indexed
Int
Ngrams
)
...
...
src/Gargantext/Database/Action/Metrics.hs
View file @
7ab982ea
...
...
@@ -35,6 +35,7 @@ import Gargantext.Core.Types.Query (Limit(..))
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getContextsByNgramsOnlyUser
{-, getTficfWith-}
)
import
Gargantext.Database.Admin.Config
(
userMaster
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
)
import
Gargantext.Database.Query.Table.Node
(
defaultList
)
import
Gargantext.Database.Query.Table.Node.Select
...
...
@@ -125,7 +126,11 @@ updateNgramsOccurrences' cId maybeListId maybeLimit tabType = do
let
fields
=
map
(
\
t
->
QualifiedIdentifier
Nothing
t
)
$
map
Text
.
pack
[
"int4"
,
"int4"
,
"text"
,
"int4"
,
"int4"
]
map
(
\
(
Only
a
)
->
a
)
<$>
runPGSQuery
queryInsert
(
Only
$
Values
fields
toInsert
)
res
<-
map
(
\
(
Only
a
)
->
a
)
<$>
runPGSQuery
queryInsert
(
Only
$
Values
fields
toInsert
)
-- _ <- map (\(Only a) -> a) <$> runPGSQuery [sql|refresh materialized view context_node_ngrams_view;|] ()
_
<-
refreshNgramsMaterialized
pure
res
...
...
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
View file @
7ab982ea
...
...
@@ -18,6 +18,7 @@ module Gargantext.Database.Action.Metrics.NgramsByContext
-- import Debug.Trace (trace)
--import Data.Map.Strict.Patch (PatchMap, Replace, diff)
import
Control.Monad
(
void
)
import
Data.HashMap.Strict
(
HashMap
)
import
Data.Map.Strict
(
Map
)
import
Data.Set
(
Set
)
...
...
@@ -29,7 +30,7 @@ import Gargantext.API.Ngrams.Types (NgramsTerm(..))
import
Gargantext.Core
import
Gargantext.Data.HashMap.Strict.Utils
as
HM
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
,
NodeId
(
..
),
ContextId
,
MasterCorpusId
,
NodeType
(
NodeDocument
),
UserCorpusId
,
DocId
)
import
Gargantext.Database.Prelude
(
Cmd
,
runPGSQuery
)
import
Gargantext.Database.Prelude
(
Cmd
,
runPGSQuery
,
execPGSQuery
)
import
Gargantext.Database.Schema.Ngrams
(
ngramsTypeId
,
NgramsType
(
..
))
import
Gargantext.Prelude
import
qualified
Data.HashMap.Strict
as
HM
...
...
@@ -217,12 +218,6 @@ queryNgramsOccurrencesOnlyByContextUser_withSample' = [sql|
GROUP BY ng.id
|]
------------------------------------------------------------------------
getContextsByNgramsOnlyUser
::
HasDBid
NodeType
=>
CorpusId
...
...
@@ -397,3 +392,17 @@ queryNgramsByContextMaster' = [sql|
SELECT m.id, m.terms FROM nodesByNgramsMaster m
RIGHT JOIN contextsByNgramsUser u ON u.id = m.id
|]
-- | Refreshes the \"context_node_ngrams_view\" materialized view.
-- This function will be run :
-- - periodically
-- - at reindex stage
-- - at the end of each text flow
refreshNgramsMaterialized
::
Cmd
err
()
refreshNgramsMaterialized
=
void
$
execPGSQuery
refreshNgramsMaterializedQuery
()
where
refreshNgramsMaterializedQuery
::
DPS
.
Query
refreshNgramsMaterializedQuery
=
[
sql
|
refresh materialized view context_node_ngrams_view;
|]
src/Gargantext/Database/GargDB.hs
View file @
7ab982ea
...
...
@@ -19,14 +19,12 @@ module Gargantext.Database.GargDB
import
Control.Exception
import
Control.Lens
(
view
)
import
Control.Monad
(
void
)
import
Control.Monad.Reader
(
MonadReader
)
import
Database.PostgreSQL.Simple.SqlQQ
(
sql
)
import
Data.Aeson
(
ToJSON
,
toJSON
)
import
Data.Text
(
Text
)
import
Data.Tuple.Extra
(
both
)
import
GHC.IO
(
FilePath
)
import
Gargantext.Database.Prelude
(
HasConfig
(
..
)
,
Cmd
,
execPGSQuery
)
import
Gargantext.Database.Prelude
(
HasConfig
(
..
))
import
Gargantext.Prelude
import
Gargantext.Prelude.Config
import
Gargantext.Prelude.Crypto.Hash
...
...
@@ -169,7 +167,10 @@ cpFile = onDisk_2 SD.copyFile
---
mvFile
::
(
MonadReader
env
m
,
MonadBase
IO
m
,
HasConfig
env
)
mvFile
::
(
MonadReader
env
m
,
MonadBase
IO
m
,
HasConfig
env
)
=>
FilePath
->
FilePath
->
m
()
mvFile
fp1
fp2
=
do
cpFile
fp1
fp2
...
...
@@ -209,9 +210,3 @@ onDisk_2 action fp1 fp2 = do
|
isDoesNotExistError
e
=
return
()
|
otherwise
=
throwIO
e
------------------------------------------------------------------------
-- | Refreshes the \"context_node_ngrams_view\" materialized view. This
-- function will be run periodically.
refreshNgramsMaterializedView
::
Cmd
IOException
()
refreshNgramsMaterializedView
=
void
$
execPGSQuery
[
sql
|
refresh materialized view context_node_ngrams_view;
|]
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment