Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
120
Issues
120
List
Board
Labels
Milestones
Merge Requests
7
Merge Requests
7
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
7ab982ea
Commit
7ab982ea
authored
May 16, 2023
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[FIX] Refresh Index: reindex of term list, textflow and schedule job
parent
80e4a368
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
50 additions
and
38 deletions
+50
-38
API.hs
src/Gargantext/API.hs
+8
-5
List.hs
src/Gargantext/API/Ngrams/List.hs
+2
-2
Metrics.hs
src/Gargantext/Database/Action/Metrics.hs
+6
-1
NgramsByContext.hs
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
+16
-7
GargDB.hs
src/Gargantext/Database/GargDB.hs
+18
-23
No files found.
src/Gargantext/API.hs
View file @
7ab982ea
...
@@ -34,7 +34,7 @@ module Gargantext.API
...
@@ -34,7 +34,7 @@ module Gargantext.API
where
where
import
Control.Concurrent
import
Control.Concurrent
import
Control.Exception
(
catch
,
finally
,
SomeException
,
displayException
)
import
Control.Exception
(
catch
,
finally
,
SomeException
,
displayException
,
IOException
)
import
Control.Lens
import
Control.Lens
import
Control.Monad.Except
import
Control.Monad.Except
import
Control.Monad.Reader
(
runReaderT
)
import
Control.Monad.Reader
(
runReaderT
)
...
@@ -56,7 +56,8 @@ import Gargantext.API.Ngrams (saveNodeStoryImmediate)
...
@@ -56,7 +56,8 @@ import Gargantext.API.Ngrams (saveNodeStoryImmediate)
import
Gargantext.API.Routes
import
Gargantext.API.Routes
import
Gargantext.API.Server
(
server
)
import
Gargantext.API.Server
(
server
)
import
Gargantext.Core.NodeStory
import
Gargantext.Core.NodeStory
import
Gargantext.Database.GargDB
(
refreshNgramsMaterializedView
)
import
Gargantext.Database.Prelude
(
Cmd
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Prelude
hiding
(
putStrLn
)
import
Gargantext.Prelude
hiding
(
putStrLn
)
import
Network.HTTP.Types
hiding
(
Query
)
import
Network.HTTP.Types
hiding
(
Query
)
import
Network.Wai
import
Network.Wai
...
@@ -130,11 +131,13 @@ schedulePeriodicActions env =
...
@@ -130,11 +131,13 @@ schedulePeriodicActions env =
refreshDBViews
::
Cron
.
Schedule
()
refreshDBViews
::
Cron
.
Schedule
()
refreshDBViews
=
do
refreshDBViews
=
do
let
doRefresh
=
do
let
doRefresh
=
do
res
<-
DB
.
runCmd
env
refreshNgramsMaterializedView
res
<-
DB
.
runCmd
env
(
refreshNgramsMaterialized
::
Cmd
IOException
()
)
case
res
of
case
res
of
Left
e
->
liftIO
$
putStrLn
$
pack
(
"Refreshing Ngrams materialized view failed: "
<>
displayException
e
)
Left
e
->
liftIO
$
putStrLn
$
pack
(
"Refreshing Ngrams materialized view failed: "
<>
displayException
e
)
Right
()
->
pure
()
Right
()
->
do
Cron
.
addJob
doRefresh
"5 * * * *"
_
<-
liftIO
$
putStrLn
$
pack
"Refresh Index Database done"
pure
()
Cron
.
addJob
doRefresh
"* 2 * * *"
----------------------------------------------------------------------
----------------------------------------------------------------------
...
...
src/Gargantext/API/Ngrams/List.hs
View file @
7ab982ea
...
@@ -38,6 +38,7 @@ import Gargantext.Core.Text.Terms.WithList (buildPatterns, termsInText)
...
@@ -38,6 +38,7 @@ import Gargantext.Core.Text.Terms.WithList (buildPatterns, termsInText)
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Database.Action.Flow
(
saveDocNgramsWith
)
import
Gargantext.Database.Action.Flow
(
saveDocNgramsWith
)
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Database.Admin.Types.Hyperdata.Document
import
Gargantext.Database.Admin.Types.Hyperdata.Document
import
Gargantext.Database.Admin.Types.Node
import
Gargantext.Database.Admin.Types.Node
import
Gargantext.Database.Query.Table.Node
(
getNode
)
import
Gargantext.Database.Query.Table.Node
(
getNode
)
...
@@ -49,7 +50,6 @@ import Gargantext.Database.Types (Indexed(..))
...
@@ -49,7 +50,6 @@ import Gargantext.Database.Types (Indexed(..))
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Utils.Jobs
(
serveJobsAPI
,
MonadJobStatus
(
..
))
import
Gargantext.Utils.Jobs
(
serveJobsAPI
,
MonadJobStatus
(
..
))
import
Servant
import
Servant
-- import Servant.Job.Async
import
qualified
Data.ByteString.Lazy
as
BSL
import
qualified
Data.ByteString.Lazy
as
BSL
import
qualified
Data.Csv
as
Csv
import
qualified
Data.Csv
as
Csv
import
qualified
Data.HashMap.Strict
as
HashMap
import
qualified
Data.HashMap.Strict
as
HashMap
...
@@ -180,7 +180,7 @@ reIndexWith cId lId nt lts = do
...
@@ -180,7 +180,7 @@ reIndexWith cId lId nt lts = do
-- Saving the indexation in database
-- Saving the indexation in database
_
<-
mapM
(
saveDocNgramsWith
lId
)
ngramsByDoc
_
<-
mapM
(
saveDocNgramsWith
lId
)
ngramsByDoc
_
<-
refreshNgramsMaterialized
pure
()
pure
()
toIndexedNgrams
::
HashMap
Text
NgramsId
->
Text
->
Maybe
(
Indexed
Int
Ngrams
)
toIndexedNgrams
::
HashMap
Text
NgramsId
->
Text
->
Maybe
(
Indexed
Int
Ngrams
)
...
...
src/Gargantext/Database/Action/Metrics.hs
View file @
7ab982ea
...
@@ -35,6 +35,7 @@ import Gargantext.Core.Types.Query (Limit(..))
...
@@ -35,6 +35,7 @@ import Gargantext.Core.Types.Query (Limit(..))
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Flow.Types
(
FlowCmdM
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getContextsByNgramsOnlyUser
{-, getTficfWith-}
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getContextsByNgramsOnlyUser
{-, getTficfWith-}
)
import
Gargantext.Database.Admin.Config
(
userMaster
)
import
Gargantext.Database.Admin.Config
(
userMaster
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
refreshNgramsMaterialized
)
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
)
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
)
import
Gargantext.Database.Query.Table.Node
(
defaultList
)
import
Gargantext.Database.Query.Table.Node
(
defaultList
)
import
Gargantext.Database.Query.Table.Node.Select
import
Gargantext.Database.Query.Table.Node.Select
...
@@ -125,7 +126,11 @@ updateNgramsOccurrences' cId maybeListId maybeLimit tabType = do
...
@@ -125,7 +126,11 @@ updateNgramsOccurrences' cId maybeListId maybeLimit tabType = do
let
fields
=
map
(
\
t
->
QualifiedIdentifier
Nothing
t
)
let
fields
=
map
(
\
t
->
QualifiedIdentifier
Nothing
t
)
$
map
Text
.
pack
[
"int4"
,
"int4"
,
"text"
,
"int4"
,
"int4"
]
$
map
Text
.
pack
[
"int4"
,
"int4"
,
"text"
,
"int4"
,
"int4"
]
map
(
\
(
Only
a
)
->
a
)
<$>
runPGSQuery
queryInsert
(
Only
$
Values
fields
toInsert
)
res
<-
map
(
\
(
Only
a
)
->
a
)
<$>
runPGSQuery
queryInsert
(
Only
$
Values
fields
toInsert
)
-- _ <- map (\(Only a) -> a) <$> runPGSQuery [sql|refresh materialized view context_node_ngrams_view;|] ()
_
<-
refreshNgramsMaterialized
pure
res
...
...
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
View file @
7ab982ea
...
@@ -18,6 +18,7 @@ module Gargantext.Database.Action.Metrics.NgramsByContext
...
@@ -18,6 +18,7 @@ module Gargantext.Database.Action.Metrics.NgramsByContext
-- import Debug.Trace (trace)
-- import Debug.Trace (trace)
--import Data.Map.Strict.Patch (PatchMap, Replace, diff)
--import Data.Map.Strict.Patch (PatchMap, Replace, diff)
import
Control.Monad
(
void
)
import
Data.HashMap.Strict
(
HashMap
)
import
Data.HashMap.Strict
(
HashMap
)
import
Data.Map.Strict
(
Map
)
import
Data.Map.Strict
(
Map
)
import
Data.Set
(
Set
)
import
Data.Set
(
Set
)
...
@@ -29,7 +30,7 @@ import Gargantext.API.Ngrams.Types (NgramsTerm(..))
...
@@ -29,7 +30,7 @@ import Gargantext.API.Ngrams.Types (NgramsTerm(..))
import
Gargantext.Core
import
Gargantext.Core
import
Gargantext.Data.HashMap.Strict.Utils
as
HM
import
Gargantext.Data.HashMap.Strict.Utils
as
HM
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
,
NodeId
(
..
),
ContextId
,
MasterCorpusId
,
NodeType
(
NodeDocument
),
UserCorpusId
,
DocId
)
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
,
NodeId
(
..
),
ContextId
,
MasterCorpusId
,
NodeType
(
NodeDocument
),
UserCorpusId
,
DocId
)
import
Gargantext.Database.Prelude
(
Cmd
,
runPGSQuery
)
import
Gargantext.Database.Prelude
(
Cmd
,
runPGSQuery
,
execPGSQuery
)
import
Gargantext.Database.Schema.Ngrams
(
ngramsTypeId
,
NgramsType
(
..
))
import
Gargantext.Database.Schema.Ngrams
(
ngramsTypeId
,
NgramsType
(
..
))
import
Gargantext.Prelude
import
Gargantext.Prelude
import
qualified
Data.HashMap.Strict
as
HM
import
qualified
Data.HashMap.Strict
as
HM
...
@@ -217,12 +218,6 @@ queryNgramsOccurrencesOnlyByContextUser_withSample' = [sql|
...
@@ -217,12 +218,6 @@ queryNgramsOccurrencesOnlyByContextUser_withSample' = [sql|
GROUP BY ng.id
GROUP BY ng.id
|]
|]
------------------------------------------------------------------------
------------------------------------------------------------------------
getContextsByNgramsOnlyUser
::
HasDBid
NodeType
getContextsByNgramsOnlyUser
::
HasDBid
NodeType
=>
CorpusId
=>
CorpusId
...
@@ -397,3 +392,17 @@ queryNgramsByContextMaster' = [sql|
...
@@ -397,3 +392,17 @@ queryNgramsByContextMaster' = [sql|
SELECT m.id, m.terms FROM nodesByNgramsMaster m
SELECT m.id, m.terms FROM nodesByNgramsMaster m
RIGHT JOIN contextsByNgramsUser u ON u.id = m.id
RIGHT JOIN contextsByNgramsUser u ON u.id = m.id
|]
|]
-- | Refreshes the \"context_node_ngrams_view\" materialized view.
-- This function will be run :
-- - periodically
-- - at reindex stage
-- - at the end of each text flow
refreshNgramsMaterialized
::
Cmd
err
()
refreshNgramsMaterialized
=
void
$
execPGSQuery
refreshNgramsMaterializedQuery
()
where
refreshNgramsMaterializedQuery
::
DPS
.
Query
refreshNgramsMaterializedQuery
=
[
sql
|
refresh materialized view context_node_ngrams_view;
|]
src/Gargantext/Database/GargDB.hs
View file @
7ab982ea
...
@@ -19,14 +19,12 @@ module Gargantext.Database.GargDB
...
@@ -19,14 +19,12 @@ module Gargantext.Database.GargDB
import
Control.Exception
import
Control.Exception
import
Control.Lens
(
view
)
import
Control.Lens
(
view
)
import
Control.Monad
(
void
)
import
Control.Monad.Reader
(
MonadReader
)
import
Control.Monad.Reader
(
MonadReader
)
import
Database.PostgreSQL.Simple.SqlQQ
(
sql
)
import
Data.Aeson
(
ToJSON
,
toJSON
)
import
Data.Aeson
(
ToJSON
,
toJSON
)
import
Data.Text
(
Text
)
import
Data.Text
(
Text
)
import
Data.Tuple.Extra
(
both
)
import
Data.Tuple.Extra
(
both
)
import
GHC.IO
(
FilePath
)
import
GHC.IO
(
FilePath
)
import
Gargantext.Database.Prelude
(
HasConfig
(
..
)
,
Cmd
,
execPGSQuery
)
import
Gargantext.Database.Prelude
(
HasConfig
(
..
))
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Prelude.Config
import
Gargantext.Prelude.Config
import
Gargantext.Prelude.Crypto.Hash
import
Gargantext.Prelude.Crypto.Hash
...
@@ -145,11 +143,11 @@ writeFile a = do
...
@@ -145,11 +143,11 @@ writeFile a = do
-- | Example to read a file with Type
-- | Example to read a file with Type
readGargFile
::
(
MonadReader
env
m
readGargFile
::
(
MonadReader
env
m
,
HasConfig
env
,
HasConfig
env
,
MonadBase
IO
m
,
MonadBase
IO
m
,
ReadFile
a
,
ReadFile
a
)
)
=>
FilePath
->
m
a
=>
FilePath
->
m
a
readGargFile
fp
=
do
readGargFile
fp
=
do
dataPath
<-
view
$
hasConfig
.
gc_datafilepath
dataPath
<-
view
$
hasConfig
.
gc_datafilepath
liftBase
$
readFile'
$
toFilePath
dataPath
fp
liftBase
$
readFile'
$
toFilePath
dataPath
fp
...
@@ -157,9 +155,9 @@ readGargFile fp = do
...
@@ -157,9 +155,9 @@ readGargFile fp = do
---
---
rmFile
::
(
MonadReader
env
m
rmFile
::
(
MonadReader
env
m
,
MonadBase
IO
m
,
MonadBase
IO
m
,
HasConfig
env
,
HasConfig
env
)
)
=>
FilePath
->
m
()
=>
FilePath
->
m
()
rmFile
=
onDisk_1
SD
.
removeFile
rmFile
=
onDisk_1
SD
.
removeFile
...
@@ -169,8 +167,11 @@ cpFile = onDisk_2 SD.copyFile
...
@@ -169,8 +167,11 @@ cpFile = onDisk_2 SD.copyFile
---
---
mvFile
::
(
MonadReader
env
m
,
MonadBase
IO
m
,
HasConfig
env
)
mvFile
::
(
MonadReader
env
m
=>
FilePath
->
FilePath
->
m
()
,
MonadBase
IO
m
,
HasConfig
env
)
=>
FilePath
->
FilePath
->
m
()
mvFile
fp1
fp2
=
do
mvFile
fp1
fp2
=
do
cpFile
fp1
fp2
cpFile
fp1
fp2
rmFile
fp1
rmFile
fp1
...
@@ -178,10 +179,10 @@ mvFile fp1 fp2 = do
...
@@ -178,10 +179,10 @@ mvFile fp1 fp2 = do
------------------------------------------------------------------------
------------------------------------------------------------------------
onDisk_1
::
(
MonadReader
env
m
onDisk_1
::
(
MonadReader
env
m
,
MonadBase
IO
m
,
MonadBase
IO
m
,
HasConfig
env
,
HasConfig
env
)
)
=>
(
FilePath
->
IO
()
)
->
FilePath
->
m
()
=>
(
FilePath
->
IO
()
)
->
FilePath
->
m
()
onDisk_1
action
fp
=
do
onDisk_1
action
fp
=
do
dataPath
<-
view
$
hasConfig
.
gc_datafilepath
dataPath
<-
view
$
hasConfig
.
gc_datafilepath
liftBase
$
action
(
toFilePath
dataPath
fp
)
`
catch
`
handleExists
liftBase
$
action
(
toFilePath
dataPath
fp
)
`
catch
`
handleExists
...
@@ -209,9 +210,3 @@ onDisk_2 action fp1 fp2 = do
...
@@ -209,9 +210,3 @@ onDisk_2 action fp1 fp2 = do
|
isDoesNotExistError
e
=
return
()
|
isDoesNotExistError
e
=
return
()
|
otherwise
=
throwIO
e
|
otherwise
=
throwIO
e
------------------------------------------------------------------------
------------------------------------------------------------------------
-- | Refreshes the \"context_node_ngrams_view\" materialized view. This
-- function will be run periodically.
refreshNgramsMaterializedView
::
Cmd
IOException
()
refreshNgramsMaterializedView
=
void
$
execPGSQuery
[
sql
|
refresh materialized view context_node_ngrams_view;
|]
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment