Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
158
Issues
158
List
Board
Labels
Milestones
Merge Requests
9
Merge Requests
9
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
6a3a5ca2
Commit
6a3a5ca2
authored
Jan 18, 2021
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[FIX] ngrams + ngramsPosTag insert
parent
7697da0d
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
26 additions
and
19 deletions
+26
-19
Terms.hs
src/Gargantext/Core/Text/Terms.hs
+11
-11
Ngrams.hs
src/Gargantext/Database/Query/Table/Ngrams.hs
+7
-3
NgramsPostag.hs
src/Gargantext/Database/Query/Table/NgramsPostag.hs
+8
-5
No files found.
src/Gargantext/Core/Text/Terms.hs
View file @
6a3a5ca2
...
@@ -46,7 +46,6 @@ import qualified Data.List as List
...
@@ -46,7 +46,6 @@ import qualified Data.List as List
import
qualified
Data.Set
as
Set
import
qualified
Data.Set
as
Set
import
qualified
Data.Text
as
Text
import
qualified
Data.Text
as
Text
import
qualified
Data.HashMap.Strict
as
HashMap
import
qualified
Data.HashMap.Strict
as
HashMap
import
Gargantext.Core
import
Gargantext.Core
import
Gargantext.Core.Text
(
sentences
,
HasText
(
..
))
import
Gargantext.Core.Text
(
sentences
,
HasText
(
..
))
import
Gargantext.Core.Text.Terms.Eleve
(
mainEleveWith
,
Tries
,
Token
,
buildTries
,
toToken
)
import
Gargantext.Core.Text.Terms.Eleve
(
mainEleveWith
,
Tries
,
Token
,
buildTries
,
toToken
)
...
@@ -151,13 +150,14 @@ extracted2ngrams (EnrichedNgrams ng) = view np_form ng
...
@@ -151,13 +150,14 @@ extracted2ngrams (EnrichedNgrams ng) = view np_form ng
insertExtractedNgrams
::
[
ExtractedNgrams
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertExtractedNgrams
::
[
ExtractedNgrams
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertExtractedNgrams
ngs
=
do
insertExtractedNgrams
ngs
=
do
let
(
s
,
e
)
=
List
.
partition
isSimpleNgrams
ngs
let
(
s
,
e
)
=
List
.
partition
isSimpleNgrams
ngs
m1
<-
if
List
.
null
s
m1
<-
insertNgrams
(
map
unSimpleNgrams
s
)
then
pure
HashMap
.
empty
printDebug
"others"
m1
else
insertNgrams
(
map
unSimpleNgrams
s
)
m2
<-
if
List
.
null
e
m2
<-
insertNgramsPostag
(
map
unEnrichedNgrams
e
)
then
pure
HashMap
.
empty
printDebug
"terms"
m2
else
insertNgramsPostag
(
map
unEnrichedNgrams
e
)
pure
$
m1
<>
m2
let
result
=
HashMap
.
unions
[
m1
,
m2
]
pure
result
isSimpleNgrams
::
ExtractedNgrams
->
Bool
isSimpleNgrams
::
ExtractedNgrams
->
Bool
isSimpleNgrams
(
SimpleNgrams
_
)
=
True
isSimpleNgrams
(
SimpleNgrams
_
)
=
True
...
...
src/Gargantext/Database/Query/Table/Ngrams.hs
View file @
6a3a5ca2
...
@@ -26,6 +26,7 @@ import Data.HashMap.Strict (HashMap)
...
@@ -26,6 +26,7 @@ import Data.HashMap.Strict (HashMap)
import
Data.ByteString.Internal
(
ByteString
)
import
Data.ByteString.Internal
(
ByteString
)
import
Data.Text
(
Text
)
import
Data.Text
(
Text
)
import
qualified
Database.PostgreSQL.Simple
as
PGS
import
qualified
Database.PostgreSQL.Simple
as
PGS
import
qualified
Data.List
as
List
import
qualified
Data.HashMap.Strict
as
HashMap
import
qualified
Data.HashMap.Strict
as
HashMap
import
Gargantext.Core.Types
import
Gargantext.Core.Types
...
@@ -66,7 +67,10 @@ _dbGetNgramsDb = runOpaQuery queryNgramsTable
...
@@ -66,7 +67,10 @@ _dbGetNgramsDb = runOpaQuery queryNgramsTable
-- TODO-ACCESS: access must not be checked here but when insertNgrams is called.
-- TODO-ACCESS: access must not be checked here but when insertNgrams is called.
insertNgrams
::
[
Ngrams
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertNgrams
::
[
Ngrams
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertNgrams
ns
=
HashMap
.
fromList
<$>
map
(
\
(
Indexed
i
t
)
->
(
t
,
i
))
<$>
(
insertNgrams'
ns
)
insertNgrams
ns
=
if
List
.
null
ns
then
pure
HashMap
.
empty
else
HashMap
.
fromList
<$>
map
(
\
(
Indexed
i
t
)
->
(
t
,
i
))
<$>
(
insertNgrams'
ns
)
-- TODO-ACCESS: access must not be checked here but when insertNgrams' is called.
-- TODO-ACCESS: access must not be checked here but when insertNgrams' is called.
insertNgrams'
::
[
Ngrams
]
->
Cmd
err
[
Indexed
Int
Text
]
insertNgrams'
::
[
Ngrams
]
->
Cmd
err
[
Indexed
Int
Text
]
...
@@ -90,7 +94,7 @@ queryInsertNgrams = [sql|
...
@@ -90,7 +94,7 @@ queryInsertNgrams = [sql|
RETURNING id,terms
RETURNING id,terms
)
)
SELECT
terms, id
SELECT
id, terms
FROM ins
FROM ins
UNION ALL
UNION ALL
SELECT c.id, terms
SELECT c.id, terms
...
...
src/Gargantext/Database/Query/Table/NgramsPostag.hs
View file @
6a3a5ca2
...
@@ -28,9 +28,9 @@ import Gargantext.Database.Schema.Prelude
...
@@ -28,9 +28,9 @@ import Gargantext.Database.Schema.Prelude
import
Gargantext.Database.Types
import
Gargantext.Database.Types
import
Gargantext.Prelude
import
Gargantext.Prelude
import
qualified
Data.HashMap.Strict
as
HashMap
import
qualified
Data.HashMap.Strict
as
HashMap
import
qualified
Data.List
as
List
import
qualified
Database.PostgreSQL.Simple
as
PGS
import
qualified
Database.PostgreSQL.Simple
as
PGS
data
NgramsPostag
=
NgramsPostag
{
_np_lang
::
Lang
data
NgramsPostag
=
NgramsPostag
{
_np_lang
::
Lang
,
_np_algo
::
PosTagAlgo
,
_np_algo
::
PosTagAlgo
,
_np_postag
::
POS
,
_np_postag
::
POS
...
@@ -64,7 +64,10 @@ toInsert (NgramsPostag l a p form lem) =
...
@@ -64,7 +64,10 @@ toInsert (NgramsPostag l a p form lem) =
)
)
insertNgramsPostag
::
[
NgramsPostag
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertNgramsPostag
::
[
NgramsPostag
]
->
Cmd
err
(
HashMap
Text
NgramsId
)
insertNgramsPostag
ns
=
HashMap
.
fromList
insertNgramsPostag
ns
=
if
List
.
null
ns
then
pure
HashMap
.
empty
else
HashMap
.
fromList
<$>
map
(
\
(
Indexed
t
i
)
->
(
t
,
i
))
<$>
map
(
\
(
Indexed
t
i
)
->
(
t
,
i
))
<$>
insertNgramsPostag'
(
map
toInsert
ns
)
<$>
insertNgramsPostag'
(
map
toInsert
ns
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment