Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
197
Issues
197
List
Board
Labels
Milestones
Merge Requests
12
Merge Requests
12
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
c72309dd
Verified
Commit
c72309dd
authored
Mar 20, 2024
by
Przemyslaw Kaminski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[refactoring] some fixes of imports according to lsp suggestions
parent
574631df
Pipeline
#5789
passed with stages
in 135 minutes and 34 seconds
Changes
9
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
84 additions
and
95 deletions
+84
-95
NodeStory.hs
src/Gargantext/Core/NodeStory.hs
+3
-3
List.hs
src/Gargantext/Core/Text/List.hs
+32
-33
Group.hs
src/Gargantext/Core/Text/List/Group.hs
+6
-12
Terms.hs
src/Gargantext/Core/Text/Terms.hs
+1
-2
Multi.hs
src/Gargantext/Core/Text/Terms/Multi.hs
+5
-9
En.hs
src/Gargantext/Core/Text/Terms/Multi/Lang/En.hs
+3
-4
Extract.hs
src/Gargantext/Database/Action/Flow/Extract.hs
+10
-7
Search.hs
src/Gargantext/Database/Action/Search.hs
+10
-9
SpacyNLP.hs
src/Gargantext/Utils/SpacyNLP.hs
+14
-16
No files found.
src/Gargantext/Core/NodeStory.hs
View file @
c72309dd
...
...
@@ -235,9 +235,9 @@ fixChildrenInNgrams ns = archiveStateFromList $ nsParents <> nsChildrenFixed
)
)
<$>
nsChildren
-- |
Sometimes, when we upload a new list, a child can be left withou
t
--
a parent. Find such ngrams and set their 'root' and 'parent' to
-- 'Nothing'.
-- |
(#281) Sometimes, when we upload a new list, a child can be lef
t
--
without a parent. Find such ngrams and set their 'root' and
-- '
parent' to '
Nothing'.
fixChildrenWithNoParent
::
NgramsState'
->
NgramsState'
fixChildrenWithNoParent
ns
=
archiveStateFromList
$
nsParents
<>
nsChildrenFixed
where
...
...
src/Gargantext/Core/Text/List.hs
View file @
c72309dd
...
...
@@ -9,14 +9,13 @@ Portability : POSIX
-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE ScopedTypeVariables #-}
module
Gargantext.Core.Text.List
where
import
Control.Lens
hiding
(
both
)
-- ((^.), view, over, set, (_1), (_2))
import
Control.Lens
(
view
,
over
)
-- ((^.), view, over, set, (_1), (_2))
import
Data.HashMap.Strict
(
HashMap
)
import
Data.HashMap.Strict
qualified
as
HashMap
import
Data.HashSet
(
HashSet
)
...
...
@@ -27,26 +26,26 @@ import Data.Set qualified as Set
import
Data.Tuple.Extra
(
both
)
import
Gargantext.API.Ngrams.Types
(
NgramsElement
,
NgramsTerm
(
..
))
import
Gargantext.Core.NLP
(
HasNLPServer
)
import
Gargantext.Core.NodeStory
import
Gargantext.Core.NodeStory
.Types
(
HasNodeStory
)
import
Gargantext.Core.Text
(
size
)
import
Gargantext.Core.Text.List.Group
import
Gargantext.Core.Text.List.Group
(
toGroupedTree
,
setScoresWithMap
)
import
Gargantext.Core.Text.List.Group.Prelude
import
Gargantext.Core.Text.List.Group.WithStem
import
Gargantext.Core.Text.List.Social
import
Gargantext.Core.Text.List.Social.Prelude
import
Gargantext.Core.Text.List.Social
(
FlowSocialListWith
,
flowSocialList
)
import
Gargantext.Core.Text.List.Social.Prelude
(
FlowListScores
,
FlowCont
(
FlowCont
),
flc_scores
)
import
Gargantext.Core.Text.Metrics
(
scored'
,
Scored
(
..
),
scored_speExc
,
scored_genInc
,
normalizeGlobal
,
normalizeLocal
,
scored_terms
)
import
Gargantext.Core.Types
(
ListType
(
..
),
MasterCorpusId
,
UserCorpusId
,
ContextId
)
import
Gargantext.Core.Types.Individu
(
User
(
..
))
import
Gargantext.Core.Types.Main
(
ListType
(
..
)
)
import
Gargantext.Data.HashMap.Strict.Utils
qualified
as
HashMap
import
Gargantext.Database.Action.Metrics.NgramsByContext
(
getContextsByNgramsUser
,
getContextsByNgramsOnlyUser
)
import
Gargantext.Database.Action.Metrics.TFICF
(
getTficf_withSample
)
import
Gargantext.Database.Admin.Types.Node
(
MasterCorpusId
,
UserCorpusId
,
ContextId
)
import
Gargantext.Database.Prelude
(
DBCmd
)
import
Gargantext.Database.Query.Table.Ngrams
(
text2ngrams
)
import
Gargantext.Database.Query.Table.NgramsPostag
(
selectLems
)
import
Gargantext.Database.Query.Table.Node
(
defaultList
)
import
Gargantext.Database.Query.Table.Node.Error
(
HasNodeError
())
import
Gargantext.Database.Query.Tree.Error
(
HasTreeError
)
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
),
Ngrams
(
..
))
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
),
Ngrams
(
..
)
,
text2ngrams
)
import
Gargantext.Prelude
{-
...
...
@@ -81,8 +80,8 @@ buildNgramsLists user uCid mCid mfslw gp = do
pure
$
Map
.
unions
$
[
ngTerms
]
<>
othersTerms
data
MapListSize
=
MapListSize
{
unMapListSize
::
!
Int
}
data
MaxListSize
=
MaxListSize
{
unMaxListSize
::
!
Int
}
newtype
MapListSize
=
MapListSize
{
unMapListSize
::
Int
}
newtype
MaxListSize
=
MaxListSize
{
unMaxListSize
::
Int
}
buildNgramsOthersList
::
(
HasNodeError
err
,
HasNLPServer
env
...
...
@@ -103,7 +102,7 @@ buildNgramsOthersList user uCid mfslw _groupParams (nt, MapListSize mapListSize,
<-
flowSocialList
mfslw
user
nt
(
FlowCont
HashMap
.
empty
$
HashMap
.
fromList
$
List
.
zip
(
HashMap
.
keys
allTerms
)
(
List
.
cycle
[
mempty
]
)
(
repeat
mempty
)
)
let
groupedWithList
=
toGroupedTree
{- groupParams -}
socialLists
allTerms
...
...
@@ -113,7 +112,7 @@ buildNgramsOthersList user uCid mfslw _groupParams (nt, MapListSize mapListSize,
(
mapTerms
,
tailTerms'
)
=
HashMap
.
partition
((
==
Just
MapTerm
)
.
viewListType
)
tailTerms
listSize
=
mapListSize
-
(
List
.
length
mapTerms
)
listSize
=
mapListSize
-
List
.
length
mapTerms
(
mapTerms'
,
candiTerms
)
=
both
HashMap
.
fromList
$
List
.
splitAt
listSize
$
List
.
take
maxListSize
...
...
@@ -121,10 +120,10 @@ buildNgramsOthersList user uCid mfslw _groupParams (nt, MapListSize mapListSize,
$
HashMap
.
toList
tailTerms'
pure
$
Map
.
fromList
[(
nt
,
List
.
take
maxListSize
$
(
toNgramsElement
stopTerms
)
<>
(
toNgramsElement
mapTerms
)
<>
(
toNgramsElement
$
setListType
(
Just
MapTerm
)
mapTerms'
)
<>
(
toNgramsElement
$
setListType
(
Just
CandidateTerm
)
candiTerms
)
pure
$
Map
.
fromList
[(
nt
,
List
.
take
maxListSize
$
toNgramsElement
stopTerms
<>
toNgramsElement
mapTerms
<>
toNgramsElement
(
setListType
(
Just
MapTerm
)
mapTerms'
)
<>
toNgramsElement
(
setListType
(
Just
CandidateTerm
)
candiTerms
)
)]
...
...
@@ -135,7 +134,7 @@ getGroupParams :: ( HasNodeError err
getGroupParams
gp
@
(
GroupWithPosTag
l
nsc
_m
)
ng
=
do
!
hashMap
<-
HashMap
.
fromList
<$>
selectLems
l
nsc
(
HashSet
.
toList
ng
)
-- printDebug "hashMap" hashMap
pure
$
over
gwl_map
(
\
x
->
x
<>
hashMap
)
gp
pure
$
over
gwl_map
(
<>
hashMap
)
gp
getGroupParams
gp
_
=
pure
gp
...
...
@@ -168,7 +167,7 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
<-
flowSocialList
mfslw
user
nt
(
FlowCont
HashMap
.
empty
$
HashMap
.
fromList
$
List
.
zip
(
HashMap
.
keys
allTerms
)
(
List
.
cycle
[
mempty
]
)
(
repeat
mempty
)
)
-- printDebug "[buildNgramsTermsList: Flow Social List / end]" nt
...
...
@@ -187,7 +186,7 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
!
socialLists_Stemmed
=
addScoreStem
groupParams'
ngramsKeys
socialLists
!
groupedWithList
=
toGroupedTree
socialLists_Stemmed
allTerms
!
(
stopTerms
,
candidateTerms
)
=
HashMap
.
partition
((
==
Just
StopTerm
)
.
viewListType
)
$
HashMap
.
filter
(
\
g
->
(
view
gts'_score
g
)
>
1
)
$
HashMap
.
filter
(
\
g
->
view
gts'_score
g
>
1
)
$
view
flc_scores
groupedWithList
!
(
groupedMono
,
groupedMult
)
=
HashMap
.
partitionWithKey
(
\
(
NgramsTerm
t
)
_v
->
size
t
<
2
)
candidateTerms
...
...
@@ -269,8 +268,8 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
!
(
monoScored
,
multScored
)
=
HashMap
.
partitionWithKey
(
\
(
NgramsTerm
t
)
_v
->
size
t
<
2
)
groupedTreeScores_SpeGen
-- filter with max score
partitionWithMaxScore
=
HashMap
.
partition
(
\
g
->
(
view
scored_genInc
$
view
gts'_score
g
)
>
(
view
scored_speExc
$
view
gts'_score
g
)
partitionWithMaxScore
=
HashMap
.
partition
(
\
g
->
view
scored_genInc
(
view
gts'_score
g
)
>
view
scored_speExc
(
view
gts'_score
g
)
)
!
(
monoScoredIncl
,
monoScoredExcl
)
=
partitionWithMaxScore
monoScored
...
...
@@ -285,25 +284,25 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
!
inclSize
=
0.4
::
Double
!
exclSize
=
1
-
inclSize
splitAt''
max'
n'
=
(
both
(
HashMap
.
fromList
))
.
(
List
.
splitAt
(
round
$
n'
*
max'
)
)
sortOn'
f
=
(
List
.
sortOn
(
Down
.
(
view
(
gts'_score
.
f
))
.
snd
)
)
.
HashMap
.
toList
splitAt''
max'
n'
=
both
HashMap
.
fromList
.
List
.
splitAt
(
round
$
n'
*
max'
)
sortOn'
f
=
List
.
sortOn
(
Down
.
view
(
gts'_score
.
f
)
.
snd
)
.
HashMap
.
toList
monoInc_size
n
=
splitAt''
n
$
monoSize
*
inclSize
/
2
multExc_size
n
=
splitAt''
n
$
multSize
*
exclSize
/
2
!
(
mapMonoScoredInclHead
,
monoScoredInclTail
)
=
monoInc_size
mapSize
$
(
sortOn'
scored_genInc
)
monoScoredIncl
!
(
mapMonoScoredExclHead
,
monoScoredExclTail
)
=
monoInc_size
mapSize
$
(
sortOn'
scored_speExc
)
monoScoredExcl
!
(
mapMonoScoredInclHead
,
monoScoredInclTail
)
=
monoInc_size
mapSize
$
sortOn'
scored_genInc
monoScoredIncl
!
(
mapMonoScoredExclHead
,
monoScoredExclTail
)
=
monoInc_size
mapSize
$
sortOn'
scored_speExc
monoScoredExcl
!
(
mapMultScoredInclHead
,
multScoredInclTail
)
=
multExc_size
mapSize
$
(
sortOn'
scored_genInc
)
multScoredIncl
!
(
mapMultScoredExclHead
,
multScoredExclTail
)
=
multExc_size
mapSize
$
(
sortOn'
scored_speExc
)
multScoredExcl
!
(
mapMultScoredInclHead
,
multScoredInclTail
)
=
multExc_size
mapSize
$
sortOn'
scored_genInc
multScoredIncl
!
(
mapMultScoredExclHead
,
multScoredExclTail
)
=
multExc_size
mapSize
$
sortOn'
scored_speExc
multScoredExcl
!
(
canMonoScoredIncHead
,
_
)
=
monoInc_size
canSize
$
(
sortOn'
scored_genInc
)
monoScoredInclTail
!
(
canMonoScoredExclHead
,
_
)
=
monoInc_size
canSize
$
(
sortOn'
scored_speExc
)
monoScoredExclTail
!
(
canMonoScoredIncHead
,
_
)
=
monoInc_size
canSize
$
sortOn'
scored_genInc
monoScoredInclTail
!
(
canMonoScoredExclHead
,
_
)
=
monoInc_size
canSize
$
sortOn'
scored_speExc
monoScoredExclTail
!
(
canMulScoredInclHead
,
_
)
=
multExc_size
canSize
$
(
sortOn'
scored_genInc
)
multScoredInclTail
!
(
canMultScoredExclHead
,
_
)
=
multExc_size
canSize
$
(
sortOn'
scored_speExc
)
multScoredExclTail
!
(
canMulScoredInclHead
,
_
)
=
multExc_size
canSize
$
sortOn'
scored_genInc
multScoredInclTail
!
(
canMultScoredExclHead
,
_
)
=
multExc_size
canSize
$
sortOn'
scored_speExc
multScoredExclTail
------------------------------------------------------------
-- Final Step building the Typed list
...
...
src/Gargantext/Core/Text/List/Group.hs
View file @
c72309dd
...
...
@@ -9,11 +9,8 @@ Portability : POSIX
-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE FunctionalDependencies #-}
{-# LANGUAGE InstanceSigs #-}
module
Gargantext.Core.Text.List.Group
where
...
...
@@ -23,8 +20,8 @@ import Data.HashMap.Strict (HashMap)
import
Data.HashMap.Strict
qualified
as
HashMap
import
Gargantext.API.Ngrams.Types
(
NgramsTerm
(
..
))
import
Gargantext.Core.Text.List.Group.Prelude
import
Gargantext.Core.Text.List.Group.WithScores
import
Gargantext.Core.Text.List.Social.Prelude
import
Gargantext.Core.Text.List.Group.WithScores
(
groupWithScores'
)
import
Gargantext.Core.Text.List.Social.Prelude
(
FlowListScores
,
FlowCont
)
import
Gargantext.Prelude
------------------------------------------------------------------------
toGroupedTree
::
(
Ord
a
,
Monoid
a
,
HasSize
a
)
...
...
@@ -43,9 +40,7 @@ setScoresWithMap :: (Ord a, Ord b, Monoid b) => HashMap NgramsTerm b
->
HashMap
NgramsTerm
(
GroupedTreeScores
b
)
setScoresWithMap
m
=
setScoresWith
(
score
m
)
where
score
m'
t
=
case
HashMap
.
lookup
t
m'
of
Nothing
->
mempty
Just
r
->
r
score
m'
t
=
fromMaybe
mempty
(
HashMap
.
lookup
t
m'
)
setScoresWith
::
(
Ord
a
,
Ord
b
)
=>
(
NgramsTerm
->
b
)
...
...
@@ -58,8 +53,7 @@ setScoresWith f = Map.mapWithKey (\k v -> over gts'_children (setScoresWith f)
)
-}
setScoresWith
f
=
HashMap
.
mapWithKey
(
\
k
v
->
v
{
_gts'_score
=
f
k
,
_gts'_children
=
setScoresWith
f
$
view
gts'_children
v
}
)
,
_gts'_children
=
setScoresWith
f
$
view
gts'_children
v
}
)
------------------------------------------------------------------------
src/Gargantext/Core/Text/Terms.hs
View file @
c72309dd
...
...
@@ -103,8 +103,7 @@ withLang (Unsupervised {..}) ns = Unsupervised { _tt_model = m', .. }
$
fmap
toToken
$
uniText
$
Text
.
intercalate
" . "
$
List
.
concat
$
map
hasText
ns
$
concatMap
hasText
ns
just_m
->
just_m
withLang
l
_
=
l
...
...
src/Gargantext/Core/Text/Terms/Multi.hs
View file @
c72309dd
...
...
@@ -11,25 +11,21 @@ Multi-terms are ngrams where n > 1.
-}
{-# LANGUAGE OverloadedStrings #-}
module
Gargantext.Core.Text.Terms.Multi
(
multiterms
,
multiterms_rake
,
tokenTagsWith
,
tokenTags
,
cleanTextForNLP
)
where
import
Control.Applicative
import
Data.Attoparsec.Text
as
DAT
import
Data.Text
hiding
(
map
,
group
,
filter
,
concat
)
import
Data.Attoparsec.Text
as
DAT
(
digit
,
space
,
notChar
,
string
)
import
Gargantext.Core
(
Lang
(
..
),
NLPServerConfig
(
..
),
PosTagAlgo
(
..
))
import
Gargantext.Core.Text.Terms.Multi.Lang.En
qualified
as
En
import
Gargantext.Core.Text.Terms.Multi.Lang.Fr
qualified
as
Fr
import
Gargantext.Core.Text.Terms.Multi.PosTagging
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
import
Gargantext.Core.Text.Terms.Multi.PosTagging
(
corenlp
,
tokens2tokensTags
)
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
(
PosSentences
(
_sentences
),
Sentence
(
_sentenceTokens
)
)
import
Gargantext.Core.Text.Terms.Multi.RAKE
(
multiterms_rake
)
import
Gargantext.Core.Types
import
Gargantext.Core.Types
(
POS
(
NP
),
Terms
(
Terms
),
TermsWithCount
,
TokenTag
(
TokenTag
,
_my_token_pos
)
)
import
Gargantext.Core.Utils
(
groupWithCounts
)
import
Gargantext.Prelude
import
Gargantext.Utils.SpacyNLP
qualified
as
SpacyNLP
import
Replace.Attoparsec.Text
as
RAT
import
Replace.Attoparsec.Text
as
RAT
(
streamEdit
)
-------------------------------------------------------------------
type
NLP_API
=
Lang
->
Text
->
IO
PosSentences
...
...
src/Gargantext/Core/Text/Terms/Multi/Lang/En.hs
View file @
c72309dd
...
...
@@ -17,8 +17,8 @@ module Gargantext.Core.Text.Terms.Multi.Lang.En (groupTokens)
where
import
Gargantext.Prelude
import
Gargantext.Core.Types
import
Gargantext.Core.Text.Terms.Multi.Group
import
Gargantext.Core.Types
(
POS
(
CC
,
IN
,
DT
,
NP
,
JJ
),
TokenTag
)
import
Gargantext.Core.Text.Terms.Multi.Group
(
group2
)
------------------------------------------------------------------------
-- | Rule grammar to group tokens
...
...
@@ -31,8 +31,7 @@ groupTokens ntags = group2 NP NP
-- $ group2 VB NP
$
group2
JJ
NP
$
group2
JJ
JJ
$
group2
JJ
CC
$
ntags
$
group2
JJ
CC
ntags
------------------------------------------------------------------------
--groupNgrams ((x,_,"PERSON"):(y,yy,"PERSON"):xs) = groupNgrams ((x <> " " <> y,yy,"PERSON"):xs)
...
...
src/Gargantext/Database/Action/Flow/Extract.hs
View file @
c72309dd
...
...
@@ -20,15 +20,16 @@ module Gargantext.Database.Action.Flow.Extract
import
Control.Lens
((
^.
),
_Just
,
view
)
import
Data.HashMap.Strict
qualified
as
HashMap
import
Data.Map.Strict
qualified
as
DM
import
Gargantext.Core
(
Lang
,
NLPServerConfig
,
PosTagAlgo
(
CoreNLP
))
import
Gargantext.Core
(
Lang
,
NLPServerConfig
,
PosTagAlgo
(
CoreNLP
))
import
Gargantext.Core.Text
(
HasText
(
..
))
import
Gargantext.Core.Text.Corpus.Parsers
(
splitOn
)
import
Gargantext.Core.Text.Terms
(
ExtractNgramsT
,
ExtractedNgrams
(
..
),
TermType
,
cleanExtractedNgrams
,
enrichedTerms
,
extractNgramsT
,
extractTerms
,
tt_lang
)
import
Gargantext.Core.Types
(
POS
(
NP
),
TermsCount
)
import
Gargantext.Database.Admin.Types.Hyperdata
(
HyperdataContact
,
HyperdataDocument
,
cw_lastName
,
hc_who
,
hd_authors
,
hd_bdd
,
hd_institutes
,
hd_source
)
import
Gargantext.Database.Admin.Types.Node
import
Gargantext.Database.Admin.Types.Hyperdata.Contact
(
HyperdataContact
,
cw_lastName
,
hc_who
)
import
Gargantext.Database.Admin.Types.Hyperdata.Document
(
HyperdataDocument
,
hd_authors
,
hd_bdd
,
hd_institutes
,
hd_source
)
import
Gargantext.Database.Admin.Types.Node
(
Node
)
import
Gargantext.Database.Prelude
(
DBCmd
)
import
Gargantext.Database.Schema.Ngrams
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
),
text2ngrams
)
import
Gargantext.Database.Schema.Node
(
NodePoly
(
..
))
import
Gargantext.Prelude
...
...
@@ -49,6 +50,9 @@ instance ExtractNgramsT HyperdataContact
pure
$
HashMap
.
fromList
$
[(
SimpleNgrams
a'
,
(
DM
.
singleton
Authors
1
,
1
))
|
a'
<-
authors
]
-- | Main ngrams extraction functionality.
-- For NgramsTerms, this calls NLP server under the hood.
-- For Sources, Institutes, Authors, this uses simple split on " ".
instance
ExtractNgramsT
HyperdataDocument
where
extractNgramsT
::
NLPServerConfig
...
...
@@ -72,9 +76,8 @@ instance ExtractNgramsT HyperdataDocument
$
maybe
[
"Nothing"
]
(
splitOn
Authors
(
doc
^.
hd_bdd
))
$
doc
^.
hd_authors
termsWithCounts'
<-
map
(
\
(
t
,
cnt
)
->
(
enrichedTerms
(
lang
^.
tt_lang
)
CoreNLP
NP
t
,
cnt
))
<$>
concat
<$>
liftBase
(
extractTerms
ncs
lang
$
hasText
doc
)
termsWithCounts'
<-
map
(
first
(
enrichedTerms
(
lang
^.
tt_lang
)
CoreNLP
NP
))
.
concat
<$>
liftBase
(
extractTerms
ncs
lang
$
hasText
doc
)
pure
$
HashMap
.
fromList
$
[(
SimpleNgrams
source
,
(
DM
.
singleton
Sources
1
,
1
))
]
...
...
src/Gargantext/Database/Action/Search.hs
View file @
c72309dd
...
...
@@ -23,7 +23,7 @@ module Gargantext.Database.Action.Search (
import
Control.Arrow
(
returnA
)
import
Control.Lens
((
^.
),
view
)
import
Data.BoolExpr
import
Data.BoolExpr
(
BoolExpr
(
..
),
Signed
(
Negative
,
Positive
)
)
import
Data.List
qualified
as
List
import
Data.Map.Strict
qualified
as
Map
import
Data.Profunctor.Product
(
p4
)
...
...
@@ -31,25 +31,26 @@ import Data.Set qualified as Set
import
Data.Text
(
unpack
)
import
Data.Text
qualified
as
T
import
Data.Time
(
UTCTime
)
import
Gargantext.Core
import
Gargantext.Core
(
Lang
(
EN
),
HasDBid
(
toDBid
)
)
import
Gargantext.Core.Text.Corpus.Query
qualified
as
API
import
Gargantext.Core.Text.Terms.Mono.Stem
(
stem
,
StemmingAlgorithm
(
..
))
import
Gargantext.Core.Types
import
Gargantext.Core.Types.Query
(
IsTrash
,
Limit
,
Offset
)
import
Gargantext.Database.Admin.Types.Hyperdata
(
HyperdataDocument
(
..
),
HyperdataContact
(
..
))
import
Gargantext.Database.Admin.Types.Hyperdata.Contact
(
HyperdataContact
(
..
)
)
import
Gargantext.Database.Admin.Types.Hyperdata.Document
(
HyperdataDocument
(
..
)
)
import
Gargantext.Database.Prelude
(
runOpaQuery
,
runCountOpaQuery
,
DBCmd
)
import
Gargantext.Database.Query.Facet
import
Gargantext.Database.Query.Filter
import
Gargantext.Database.Query.Table.Context
import
Gargantext.Database.Query.Filter
(
limit'
,
offset'
)
import
Gargantext.Database.Query.Table.Context
(
queryContextSearchTable
)
import
Gargantext.Database.Query.Table.ContextNodeNgrams
(
queryContextNodeNgramsTable
)
import
Gargantext.Database.Query.Table.Node
import
Gargantext.Database.Query.Table.Node
(
queryNodeSearchTable
,
defaultList
)
import
Gargantext.Database.Query.Table.Node.Error
(
HasNodeError
())
import
Gargantext.Database.Query.Table.NodeContext
import
Gargantext.Database.
Query.Table.NodeContext_NodeContext
import
Gargantext.Database.
Schema.NodeContext_NodeContext
(
NodeContext_NodeContextRead
,
queryNodeContext_NodeContextTable
,
ncnc_nodecontext2
,
ncnc_nodecontext1
)
import
Gargantext.Database.Schema.Context
import
Gargantext.Database.Schema.ContextNodeNgrams
(
ContextNodeNgramsPoly
(
..
))
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
))
import
Gargantext.Database.Schema.Node
import
Gargantext.Database.Schema.Node
(
NodePolySearch
(
_ns_hyperdata
,
_ns_search
,
_ns_typename
,
_ns_id
)
)
import
Gargantext.Prelude
hiding
(
groupBy
)
import
Opaleye
hiding
(
Order
)
import
Opaleye
qualified
as
O
hiding
(
Order
)
...
...
@@ -59,7 +60,7 @@ import Opaleye qualified as O hiding (Order)
--
queryToTsSearch
::
API
.
Query
->
Field
SqlTSQuery
queryToTsSearch
q
=
sqlToTSQuery
$
T
.
unpack
$
(
API
.
interpretQuery
q
transformAST
)
queryToTsSearch
q
=
sqlToTSQuery
$
T
.
unpack
$
API
.
interpretQuery
q
transformAST
where
-- It's important to understand how things work under the hood: When we perform
...
...
src/Gargantext/Utils/SpacyNLP.hs
View file @
c72309dd
...
...
@@ -13,7 +13,6 @@ Server to be used: https://gitlab.iscpif.fr/gargantext/spacy-server
-}
{-# LANGUAGE TemplateHaskell #-}
module
Gargantext.Utils.SpacyNLP
(
module
Gargantext
.
Utils
.
SpacyNLP
.
Types
...
...
@@ -24,9 +23,8 @@ module Gargantext.Utils.SpacyNLP (
)
where
import
Data.Aeson
(
encode
)
import
Data.Text
hiding
(
map
,
group
,
filter
,
concat
,
zip
)
import
Gargantext.Core
(
Lang
(
..
))
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
(
PosSentences
(
PosSentences
),
Sentence
(
Sentence
),
Token
(
Token
)
)
import
Gargantext.Prelude
import
Network.HTTP.Simple
(
parseRequest
,
httpJSON
,
setRequestBodyLBS
,
getResponseBody
,
Response
)
import
Network.URI
(
URI
(
..
))
...
...
@@ -42,22 +40,22 @@ spacyRequest uri txt = do
----------------------------------------------------------------
spacyTagsToToken
::
SpacyTags
->
Token
spacyTagsToToken
st
=
Token
(
_spacyTags_index
st
)
(
_spacyTags_normalized
st
)
(
_spacyTags_text
st
)
(
_spacyTags_lemma
st
)
(
_spacyTags_head_index
st
)
(
_spacyTags_char_offset
st
)
(
Just
$
_spacyTags_pos
st
)
(
Just
$
_spacyTags_ent_type
st
)
(
Just
$
_spacyTags_prefix
st
)
(
Just
$
_spacyTags_suffix
st
)
spacyTagsToToken
st
=
Token
(
_spacyTags_index
st
)
(
_spacyTags_normalized
st
)
(
_spacyTags_text
st
)
(
_spacyTags_lemma
st
)
(
_spacyTags_head_index
st
)
(
_spacyTags_char_offset
st
)
(
Just
$
_spacyTags_pos
st
)
(
Just
$
_spacyTags_ent_type
st
)
(
Just
$
_spacyTags_prefix
st
)
(
Just
$
_spacyTags_suffix
st
)
spacyDataToPosSentences
::
SpacyData
->
PosSentences
spacyDataToPosSentences
(
SpacyData
ds
)
=
PosSentences
$
map
(
\
(
i
,
ts
)
->
Sentence
i
ts
)
$
zip
[
1
..
]
$
map
(
\
(
SpacyText
_
tags
)
->
map
spacyTagsToToken
tags
)
ds
$
zipWith
Sentence
[
1
..
]
(
map
(
\
(
SpacyText
_
tags
)
->
map
spacyTagsToToken
tags
)
ds
)
-----------------------------------------------------------------
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment