Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
153
Issues
153
List
Board
Labels
Milestones
Merge Requests
9
Merge Requests
9
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
dcb737c1
Commit
dcb737c1
authored
2 years ago
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[OPTIM] Ngrams Table query
parent
de1fddb3
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
30 additions
and
37 deletions
+30
-37
gargantext.cabal
gargantext.cabal
+1
-1
Ngrams.hs
src/Gargantext/API/Ngrams.hs
+3
-8
NgramsByContext.hs
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
+26
-28
No files found.
gargantext.cabal
View file @
dcb737c1
...
...
@@ -5,7 +5,7 @@ cabal-version: 1.12
-- see: https://github.com/sol/hpack
name: gargantext
version: 0.0.6.7
version:
0.0.6.7
synopsis: Search, map, share
description: Please see README.md
category: Data
...
...
This diff is collapsed.
Click to expand it.
src/Gargantext/API/Ngrams.hs
View file @
dcb737c1
...
...
@@ -105,7 +105,7 @@ import Gargantext.Core.Mail.Types (HasMail)
import
Gargantext.Core.Types
(
ListType
(
..
),
NodeId
,
ListId
,
DocId
,
Limit
,
Offset
,
TODO
,
assertValid
,
HasInvalidError
)
import
Gargantext.API.Ngrams.Tools
import
Gargantext.Database.Action.Flow.Types
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getOccByNgramsOnlyFast
'
)
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getOccByNgramsOnlyFast
)
import
Gargantext.Database.Admin.Config
(
userMaster
)
import
Gargantext.Database.Admin.Types.Node
(
NodeType
(
..
))
import
Gargantext.Database.Prelude
(
HasConnectionPool
(
..
),
HasConfig
)
...
...
@@ -581,10 +581,9 @@ getTableNgrams _nType nId tabType listId limit_ offset
let
ngrams_terms
=
table
^..
each
.
ne_ngrams
-- printDebug "ngrams_terms" ngrams_terms
t1
<-
getTime
occurrences
<-
getOccByNgramsOnlyFast
'
nId
occurrences
<-
getOccByNgramsOnlyFast
nId
listId
ngramsType
ngrams_terms
--printDebug "occurrences" occurrences
t2
<-
getTime
liftBase
$
hprint
stderr
...
...
@@ -644,19 +643,15 @@ scoresRecomputeTableNgrams nId tabType listId = do
setScores
::
forall
t
.
Each
t
t
NgramsElement
NgramsElement
=>
t
->
m
t
setScores
table
=
do
let
ngrams_terms
=
table
^..
each
.
ne_ngrams
occurrences
<-
getOccByNgramsOnlyFast'
nId
occurrences
<-
getOccByNgramsOnlyFast
nId
listId
ngramsType
ngrams_terms
let
setOcc
ne
=
ne
&
ne_occurrences
.~
sumOf
(
at
(
ne
^.
ne_ngrams
)
.
_Just
)
occurrences
pure
$
table
&
each
%~
setOcc
-- APIs
-- TODO: find a better place for the code above, All APIs stay here
...
...
This diff is collapsed.
Click to expand it.
src/Gargantext/Database/Action/Metrics/NgramsByContext.hs
View file @
dcb737c1
...
...
@@ -20,7 +20,6 @@ module Gargantext.Database.Action.Metrics.NgramsByContext
--import Data.Map.Strict.Patch (PatchMap, Replace, diff)
import
Data.HashMap.Strict
(
HashMap
)
import
Data.Map
(
Map
)
import
Data.Maybe
(
catMaybes
)
import
Data.Set
(
Set
)
import
Data.Text
(
Text
)
import
Data.Tuple.Extra
(
first
,
second
,
swap
)
...
...
@@ -31,8 +30,7 @@ import Gargantext.Core
import
Gargantext.Data.HashMap.Strict.Utils
as
HM
import
Gargantext.Database.Admin.Types.Node
(
ListId
,
CorpusId
,
NodeId
(
..
),
ContextId
,
MasterCorpusId
,
NodeType
(
NodeDocument
),
UserCorpusId
,
DocId
)
import
Gargantext.Database.Prelude
(
Cmd
,
runPGSQuery
)
import
Gargantext.Database.Query.Table.Ngrams
(
selectNgramsId
)
import
Gargantext.Database.Schema.Ngrams
(
ngramsTypeId
,
NgramsType
(
..
),
NgramsId
)
import
Gargantext.Database.Schema.Ngrams
(
ngramsTypeId
,
NgramsType
(
..
))
import
Gargantext.Prelude
import
qualified
Data.HashMap.Strict
as
HM
import
qualified
Data.Map
as
Map
...
...
@@ -107,47 +105,47 @@ getOccByNgramsOnlyFast_withSample cId int nt ngs =
HM
.
fromListWith
(
+
)
<$>
selectNgramsOccurrencesOnlyByContextUser_withSample
cId
int
nt
ngs
getOccByNgramsOnlyFast
'
::
CorpusId
getOccByNgramsOnlyFast
::
CorpusId
->
ListId
->
NgramsType
->
[
NgramsTerm
]
->
Cmd
err
(
HashMap
NgramsTerm
Int
)
getOccByNgramsOnlyFast'
cId
lId
nt
tms
=
do
-- trace (show (cId, lId)) $
mapNgramsIds
<-
selectNgramsId
$
map
unNgramsTerm
tms
HM
.
fromListWith
(
+
)
<$>
catMaybes
<$>
map
(
\
(
nId
,
s
)
->
(,)
<$>
(
NgramsTerm
<$>
(
Map
.
lookup
nId
mapNgramsIds
))
<*>
(
Just
$
round
s
)
)
<$>
run
cId
lId
nt
(
Map
.
keys
mapNgramsIds
)
getOccByNgramsOnlyFast
cId
lId
nt
=
do
HM
.
fromList
<$>
map
(
\
(
t
,
n
)
->
(
NgramsTerm
t
,
round
n
))
<$>
run
cId
lId
nt
where
run
::
CorpusId
->
ListId
->
NgramsType
->
[
NgramsId
]
->
Cmd
err
[(
NgramsId
,
Double
)]
run
cId'
lId'
nt'
tms'
=
runPGSQuery
query
(
Values
fields
((
DPS
.
Only
)
<$>
tms'
)
,
cId'
->
Cmd
err
[(
Text
,
Double
)]
run
cId'
lId'
nt'
=
runPGSQuery
query
(
cId'
,
lId'
,
ngramsTypeId
nt'
)
fields
=
[
QualifiedIdentifier
Nothing
"int4"
]
query
::
DPS
.
Query
query
=
[
sql
|
WITH input_ngrams(id) AS (?)
SELECT ng.terms
-- , ng.id
, round(nng.weight)
-- , ns.version
-- , nng.ngrams_type
-- , ns.ngrams_type_id
FROM ngrams ng
JOIN node_stories ns ON ng.id = ns.ngrams_id
JOIN node_node_ngrams nng ON ns.node_id = nng.node2_id
WHERE nng.node1_id = ?
AND nng.node2_id = ?
AND nng.ngrams_type = ?
AND nng.ngrams_id = ng.id
AND nng.ngrams_type = ns.ngrams_type_id
ORDER BY ng.id ASC;
|]
SELECT ngi.id, nng.weight FROM nodes_contexts nc
JOIN node_node_ngrams nng ON nng.node1_id = nc.node_id
JOIN input_ngrams ngi ON nng.ngrams_id = ngi.id
WHERE nng.node1_id = ?
AND nng.node2_id = ?
AND nng.ngrams_type = ?
AND nc.category > 0
GROUP BY ngi.id, nng.weight
|]
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment