Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Grégoire Locqueville
haskell-gargantext
Commits
bac174eb
Verified
Commit
bac174eb
authored
Feb 28, 2024
by
Przemyslaw Kaminski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[refactor] this compiles fine with stack now...
parent
5f6cce5b
Changes
18
Hide whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
82 additions
and
121 deletions
+82
-121
Main.hs
bin/gargantext-phylo-profile/Main.hs
+15
-3
Common.hs
bin/gargantext-phylo/Phylo/Common.hs
+19
-6
cabal.project
cabal.project
+2
-1
gargantext.cabal
gargantext.cabal
+1
-4
Ngrams.hs
src/Gargantext/API/Ngrams.hs
+4
-4
Types.hs
src/Gargantext/API/Ngrams/List/Types.hs
+5
-5
New.hs
src/Gargantext/API/Node/Corpus/New.hs
+5
-4
Types.hs
src/Gargantext/API/Node/Document/Export/Types.hs
+3
-4
Public.hs
src/Gargantext/API/Public.hs
+10
-9
Types.hs
src/Gargantext/Core/Ngrams/Types.hs
+3
-6
FrameWrite.hs
src/Gargantext/Core/Text/Corpus/Parsers/FrameWrite.hs
+1
-1
Tokenize.hs
src/Gargantext/Core/Text/Terms/Tokenize.hs
+2
-3
Types.hs
src/Gargantext/Core/Text/Terms/Tokenize/Types.hs
+1
-1
List.hs
src/Gargantext/Database/Action/Flow/List.hs
+1
-1
Servant.hs
src/Gargantext/Utils/Servant.hs
+1
-2
SpacyNLP.hs
src/Gargantext/Utils/SpacyNLP.hs
+6
-64
Types.hs
src/Gargantext/Utils/SpacyNLP/Types.hs
+1
-0
stack.yaml
stack.yaml
+2
-3
No files found.
bin/gargantext-phylo-profile/Main.hs
View file @
bac174eb
{-|
Module : Main
Description : Phylomemy profile Main module
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE OverloadedStrings #-}
module
Main
where
module
Main
where
import
Common
import
Common
import
Data.Aeson
import
Data.Aeson
import
Data.List
(
nub
)
import
Data.List
(
nub
)
import
Data.Text
qualified
as
T
import
GHC.IO.Encoding
import
GHC.Stack
import
Gargantext.Core.Viz.Phylo
import
Gargantext.Core.Viz.Phylo
import
Gargantext.Core.Viz.Phylo.API.Tools
import
Gargantext.Core.Viz.Phylo.API.Tools
import
Gargantext.Core.Viz.Phylo.PhyloExport
(
toPhyloExport
,
dotToFile
)
import
Gargantext.Core.Viz.Phylo.PhyloExport
(
toPhyloExport
,
dotToFile
)
import
Gargantext.Core.Viz.Phylo.PhyloMaker
(
toPhylo
,
toPhyloWithoutLink
)
import
Gargantext.Core.Viz.Phylo.PhyloMaker
(
toPhylo
,
toPhyloWithoutLink
)
import
Gargantext.Core.Viz.Phylo.PhyloTools
(
printIOMsg
,
printIOComment
,
setConfig
,
toPeriods
,
getTimePeriod
,
getTimeStep
)
import
Gargantext.Core.Viz.Phylo.PhyloTools
(
printIOMsg
,
printIOComment
,
setConfig
,
toPeriods
,
getTimePeriod
,
getTimeStep
)
import
GHC.IO.Encoding
import
GHC.Stack
import
Paths_gargantext
import
Paths_gargantext
import
Prelude
import
Prelude
import
qualified
Data.Text
as
T
import
Shelly
import
Shelly
import
System.Directory
import
System.Directory
...
...
bin/gargantext-phylo/Phylo/Common.hs
View file @
bac174eb
{-|
Module : Common
Description : Phylomemy common module
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE OverloadedStrings #-}
module
Common
where
module
Common
where
import
Control.Concurrent.Async
(
mapConcurrently
)
import
Control.Concurrent.Async
(
mapConcurrently
)
import
Crypto.Hash.SHA256
(
hash
)
import
Crypto.Hash.SHA256
(
hash
)
import
Data.Aeson
import
Data.Aeson
(
eitherDecode
)
import
Data.ByteString.Char8
qualified
as
C8
import
Data.ByteString.Char8
qualified
as
C8
import
Data.List
(
nub
,
tail
)
import
Data.List
(
nub
,
tail
)
import
Data.List.Split
import
Data.List.Split
(
splitOn
)
import
Data.Maybe
(
fromJust
)
import
Data.Maybe
(
fromJust
)
import
Data.Text
(
unpack
,
replace
,
pack
)
import
Data.Text
(
unpack
,
replace
,
pack
)
import
Data.Text
qualified
as
T
import
Data.Text
qualified
as
T
import
Data.Vector
qualified
as
Vector
import
Data.Vector
qualified
as
Vector
import
Gargantext.API.Ngrams.Prelude
(
toTermList
)
import
Gargantext.API.Ngrams.Prelude
(
toTermList
)
import
Gargantext.
API.Ngrams.Types
import
Gargantext.
Core.Ngrams.Types
(
NgramsList
)
import
Gargantext.Core.Text.Context
(
TermList
)
import
Gargantext.Core.Text.Context
(
TermList
)
import
Gargantext.Core.Text.Corpus.Parsers
(
FileFormat
(
..
),
FileType
(
..
),
parseFile
)
import
Gargantext.Core.Text.Corpus.Parsers
(
parseFile
)
import
Gargantext.Core.Text.Corpus.Parsers.CSV
(
csv_title
,
csv_abstract
,
csv_publication_year
,
csv_publication_month
,
csv_publication_day
,
csv'_source
,
csv'_title
,
csv'_abstract
,
csv'_publication_year
,
csv'_publication_month
,
csv'_publication_day
,
csv'_weight
)
import
Gargantext.Core.Text.Corpus.Parsers.CSV
(
csv_title
,
csv_abstract
,
csv_publication_year
,
csv_publication_month
,
csv_publication_day
,
csv'_source
,
csv'_title
,
csv'_abstract
,
csv'_publication_year
,
csv'_publication_month
,
csv'_publication_day
,
csv'_weight
)
import
Gargantext.Core.Text.Corpus.Parsers.CSV
qualified
as
Csv
import
Gargantext.Core.Text.Corpus.Parsers.CSV
qualified
as
Csv
import
Gargantext.Core.Text.Corpus.Parsers.Types
(
FileFormat
(
..
),
FileType
(
..
))
import
Gargantext.Core.Text.List.Formats.CSV
(
csvMapTermList
)
import
Gargantext.Core.Text.List.Formats.CSV
(
csvMapTermList
)
import
Gargantext.Core.Text.Terms.WithList
(
Patterns
,
buildPatterns
,
extractTermsWithList
)
import
Gargantext.Core.Text.Terms.WithList
(
Patterns
,
buildPatterns
,
extractTermsWithList
)
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Core.Viz.Phylo
import
Gargantext.Core.Viz.Phylo
import
Gargantext.Core.Viz.Phylo.API.Tools
import
Gargantext.Core.Viz.Phylo.API.Tools
(
toPhyloDate
,
toPhyloDate'
,
readJson
)
import
Gargantext.Core.Viz.Phylo.PhyloTools
(
toPeriods
,
getTimePeriod
,
getTimeStep
)
import
Gargantext.Core.Viz.Phylo.PhyloTools
(
toPeriods
,
getTimePeriod
,
getTimeStep
)
import
Gargantext.Database.Admin.Types.Hyperdata
(
HyperdataDocument
(
..
)
)
import
Gargantext.Database.Admin.Types.Hyperdata
.Document
(
HyperdataDocument
(
..
)
)
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
))
import
Gargantext.Database.Schema.Ngrams
(
NgramsType
(
..
))
import
Gargantext.Prelude
hiding
(
hash
,
replace
)
import
Gargantext.Prelude
hiding
(
hash
,
replace
)
import
Prelude
qualified
import
Prelude
qualified
import
System.Directory
(
listDirectory
)
import
System.Directory
(
listDirectory
)
data
Backup
=
BackupPhyloWithoutLink
|
BackupPhylo
deriving
(
Show
)
data
Backup
=
BackupPhyloWithoutLink
|
BackupPhylo
deriving
(
Show
)
---------------
---------------
...
...
cabal.project
View file @
bac174eb
...
@@ -131,7 +131,7 @@ source-repository-package
...
@@ -131,7 +131,7 @@ source-repository-package
source
-
repository
-
package
source
-
repository
-
package
type
:
git
type
:
git
location
:
https
://
gitlab
.
iscpif
.
fr
/
gargantext
/
crawlers
/
openalex
.
git
location
:
https
://
gitlab
.
iscpif
.
fr
/
gargantext
/
crawlers
/
openalex
.
git
tag
:
d
d23048307f2cef0655cced2005cd0eaf69f8b6
a
tag
:
d
c0a1be8f2c706fb67b5eb7fb1c667eea911a0b
a
source
-
repository
-
package
source
-
repository
-
package
type
:
git
type
:
git
...
@@ -215,6 +215,7 @@ allow-newer: *
...
@@ -215,6 +215,7 @@ allow-newer: *
package
gargantext
package
gargantext
ghc
-
options
:
-
fwrite
-
ide
-
info
-
hiedir
=
".stack-work/hiedb"
ghc
-
options
:
-
fwrite
-
ide
-
info
-
hiedir
=
".stack-work/hiedb"
optimization
:
2
package
hmatrix
package
hmatrix
ghc
-
options
:
-
O2
-
fsimpl
-
tick
-
factor
=
10000
-
fdicts
-
cheap
-
fdicts
-
strict
-
flate
-
dmd
-
anal
-
fno
-
state
-
hack
ghc
-
options
:
-
O2
-
fsimpl
-
tick
-
factor
=
10000
-
fdicts
-
cheap
-
fdicts
-
strict
-
flate
-
dmd
-
anal
-
fno
-
state
-
hack
...
...
gargantext.cabal
View file @
bac174eb
...
@@ -65,7 +65,6 @@ common defaults
...
@@ -65,7 +65,6 @@ common defaults
base >=4.7 && <5
base >=4.7 && <5
, aeson ^>= 2.2.1.0
, aeson ^>= 2.2.1.0
optimization: 2
common optimized
common optimized
ghc-options:
ghc-options:
-O2
-O2
...
@@ -470,7 +469,6 @@ library
...
@@ -470,7 +469,6 @@ library
, bytestring ^>= 0.10.12.0
, bytestring ^>= 0.10.12.0
, case-insensitive ^>= 1.2.1.0
, case-insensitive ^>= 1.2.1.0
, cassava ^>= 0.5.2.0
, cassava ^>= 0.5.2.0
, cborg ^>= 0.2.6.0
, cereal ^>= 0.5.8.2
, cereal ^>= 0.5.8.2
, conduit ^>= 1.3.4.2
, conduit ^>= 1.3.4.2
, conduit-extra ^>= 1.3.5
, conduit-extra ^>= 1.3.5
...
@@ -601,8 +599,8 @@ library
...
@@ -601,8 +599,8 @@ library
, simple-reflect ^>= 0.3.3
, simple-reflect ^>= 0.3.3
, singletons ^>= 2.7
, singletons ^>= 2.7
, singletons-th >= 3.1
, singletons-th >= 3.1
, snowball ^>= 1.0.0.1
, split ^>= 0.2.3.4
, split ^>= 0.2.3.4
, stemmer ^>= 0.5.2
, stm ^>= 2.5.0.1
, stm ^>= 2.5.0.1
, swagger2 ^>= 2.6
, swagger2 ^>= 2.6
, taggy-lens ^>= 0.1.2
, taggy-lens ^>= 0.1.2
...
@@ -818,7 +816,6 @@ executable gargantext-server
...
@@ -818,7 +816,6 @@ executable gargantext-server
, text ^>= 1.2.4.1
, text ^>= 1.2.4.1
, unordered-containers ^>= 0.2.16.0
, unordered-containers ^>= 0.2.16.0
, vector ^>= 0.7.3
, vector ^>= 0.7.3
optimization: 2
executable gargantext-upgrade
executable gargantext-upgrade
import:
import:
...
...
src/Gargantext/API/Ngrams.hs
View file @
bac174eb
...
@@ -31,7 +31,7 @@ module Gargantext.API.Ngrams
...
@@ -31,7 +31,7 @@ module Gargantext.API.Ngrams
,
TableNgramsApiPut
,
TableNgramsApiPut
,
commitStatePatch
,
commitStatePatch
,
searchTableNgrams
,
searchTableNgrams
,
getTableNgrams
,
getTableNgrams
,
getTableNgramsCorpus
,
getTableNgramsCorpus
...
@@ -164,7 +164,7 @@ commitStatePatch listId (Versioned _p_version p) = do
...
@@ -164,7 +164,7 @@ commitStatePatch listId (Versioned _p_version p) = do
-- )
-- )
let
newA
=
Versioned
(
a'
^.
a_version
)
q'
let
newA
=
Versioned
(
a'
^.
a_version
)
q'
-- NOTE Now is the only good time to save the archive history. We
-- NOTE Now is the only good time to save the archive history. We
-- have the handle to the MVar and we need to save its exact
-- have the handle to the MVar and we need to save its exact
-- snapshot. Node Story archive is a linear table, so it's only
-- snapshot. Node Story archive is a linear table, so it's only
...
@@ -380,11 +380,11 @@ searchTableNgrams versionedTableMap NgramsSearchQuery{..} =
...
@@ -380,11 +380,11 @@ searchTableNgrams versionedTableMap NgramsSearchQuery{..} =
-- | For each input root, extends its occurrence count with
-- | For each input root, extends its occurrence count with
-- the information found in the subitems.
-- the information found in the subitems.
withInners
::
Map
NgramsTerm
NgramsElement
->
Set
NgramsElement
->
Set
NgramsElement
withInners
::
Map
NgramsTerm
NgramsElement
->
Set
NgramsElement
->
Set
NgramsElement
withInners
tblMap
roots
=
Set
.
map
addSubitemsOccurrences
root
s
withInners
tblMap
=
Set
.
map
addSubitemsOccurrence
s
where
where
addSubitemsOccurrences
::
NgramsElement
->
NgramsElement
addSubitemsOccurrences
::
NgramsElement
->
NgramsElement
addSubitemsOccurrences
e
=
addSubitemsOccurrences
e
=
e
&
ne_occurrences
.~
(
foldl'
alterOccurrences
(
e
^.
ne_occurrences
)
(
e
^.
ne_children
)
)
e
&
ne_occurrences
.~
foldl'
alterOccurrences
(
e
^.
ne_occurrences
)
(
e
^.
ne_nre
.
nre_children
)
alterOccurrences
::
Set
ContextId
->
NgramsTerm
->
Set
ContextId
alterOccurrences
::
Set
ContextId
->
NgramsTerm
->
Set
ContextId
alterOccurrences
occs
t
=
case
Map
.
lookup
t
tblMap
of
alterOccurrences
occs
t
=
case
Map
.
lookup
t
tblMap
of
...
...
src/Gargantext/API/Ngrams/List/Types.hs
View file @
bac174eb
...
@@ -9,18 +9,18 @@ Portability : POSIX
...
@@ -9,18 +9,18 @@ Portability : POSIX
-}
-}
{-# LANGUAGE TypeOperators #-}
module
Gargantext.API.Ngrams.List.Types
where
module
Gargantext.API.Ngrams.List.Types
where
import
Data.Aeson
import
Data.Aeson
(
FromJSON
(
parseJSON
),
ToJSON
(
toJSON
),
eitherDecode'
,
genericParseJSON
,
genericToJSON
)
import
Data.Text.Encoding
qualified
as
E
import
Data.Swagger
(
ToSchema
(
..
))
import
Data.Text
qualified
as
T
import
Gargantext.API.Node.Corpus.New.Types
(
FileType
(
..
))
import
Gargantext.API.Node.Corpus.New.Types
(
FileType
(
..
))
import
Gargantext.Core.Ngrams.Types
(
NgramsList
)
import
Gargantext.Core.Ngrams.Types
(
NgramsList
)
import
Gargantext.Core.Utils.Prefix
(
unPrefixSwagger
)
import
Gargantext.Core.Utils.Prefix
(
unPrefixSwagger
)
import
Protolude
import
Protolude
import
Servant.Job.Utils
(
jsonOptions
)
import
Servant.Job.Utils
(
jsonOptions
)
import
Web.FormUrlEncoded
(
FromForm
(
..
),
ToForm
,
parseUnique
)
import
Web.FormUrlEncoded
(
FromForm
(
..
),
ToForm
,
parseUnique
)
import
Data.Swagger.Schema
(
genericDeclareNamedSchema
)
...
@@ -51,7 +51,7 @@ instance FromForm WithJsonFile where
...
@@ -51,7 +51,7 @@ instance FromForm WithJsonFile where
fromForm
f
=
do
fromForm
f
=
do
d'
<-
parseUnique
"_wjf_data"
f
::
Either
Text
Text
d'
<-
parseUnique
"_wjf_data"
f
::
Either
Text
Text
d
<-
case
eitherDecode'
(
toUtf8Lazy
d'
)
of
d
<-
case
eitherDecode'
(
toUtf8Lazy
d'
)
of
Left
s
->
Left
$
pack
s
Left
s
->
Left
$
T
.
pack
s
Right
v
->
Right
v
Right
v
->
Right
v
n
<-
parseUnique
"_wjf_name"
f
n
<-
parseUnique
"_wjf_name"
f
pure
$
WithJsonFile
{
_wjf_data
=
d
pure
$
WithJsonFile
{
_wjf_data
=
d
...
...
src/Gargantext/API/Node/Corpus/New.hs
View file @
bac174eb
...
@@ -47,7 +47,7 @@ import Gargantext.API.Node.Types (NewWithFile(..), NewWithForm, wf_data, wf_file
...
@@ -47,7 +47,7 @@ import Gargantext.API.Node.Types (NewWithFile(..), NewWithForm, wf_data, wf_file
import
Gargantext.Core
(
withDefaultLanguage
,
defaultLanguage
)
import
Gargantext.Core
(
withDefaultLanguage
,
defaultLanguage
)
import
Gargantext.Core.Ngrams.Types
(
Versioned
(
..
))
import
Gargantext.Core.Ngrams.Types
(
Versioned
(
..
))
import
Gargantext.Core.NodeStory
(
HasNodeStoryImmediateSaver
,
HasNodeArchiveStoryImmediateSaver
,
currentVersion
,
NgramsStatePatch
'
)
import
Gargantext.Core.NodeStory
(
HasNodeStoryImmediateSaver
,
HasNodeArchiveStoryImmediateSaver
,
currentVersion
,
NgramsStatePatch
'
)
import
Gargantext.Core.Text.Corpus.
API
qualified
as
API
import
Gargantext.Core.Text.Corpus.
Query
qualified
as
API
import
Gargantext.Core.Text.Corpus.Parsers
qualified
as
Parser
(
parseFormatC
)
import
Gargantext.Core.Text.Corpus.Parsers
qualified
as
Parser
(
parseFormatC
)
import
Gargantext.Core.Text.Corpus.Parsers.Types
qualified
as
Parser
(
FileFormat
(
..
))
import
Gargantext.Core.Text.Corpus.Parsers.Types
qualified
as
Parser
(
FileFormat
(
..
))
import
Gargantext.Core.Text.Terms
(
TermType
(
..
))
import
Gargantext.Core.Text.Terms
(
TermType
(
..
))
...
@@ -58,7 +58,8 @@ import Gargantext.Database.Action.Flow.Types (FlowCmdM)
...
@@ -58,7 +58,8 @@ import Gargantext.Database.Action.Flow.Types (FlowCmdM)
import
Gargantext.Database.Action.Mail
(
sendMail
)
import
Gargantext.Database.Action.Mail
(
sendMail
)
import
Gargantext.Database.Action.Node
(
mkNodeWithParent
)
import
Gargantext.Database.Action.Node
(
mkNodeWithParent
)
import
Gargantext.Database.Action.User
(
getUserId
)
import
Gargantext.Database.Action.User
(
getUserId
)
import
Gargantext.Database.Admin.Types.Hyperdata
(
HyperdataFile
,
hff_name
,
hff_path
,
toHyperdataDocument
)
import
Gargantext.Database.Admin.Types.Hyperdata.Document
(
toHyperdataDocument
)
import
Gargantext.Database.Admin.Types.Hyperdata.File
(
HyperdataFile
,
hff_name
,
hff_path
)
import
Gargantext.Database.Admin.Types.Node
(
CorpusId
,
NodeType
(
..
),
ParentId
)
import
Gargantext.Database.Admin.Types.Node
(
CorpusId
,
NodeType
(
..
),
ParentId
)
import
Gargantext.Database.GargDB
qualified
as
GargDB
import
Gargantext.Database.GargDB
qualified
as
GargDB
import
Gargantext.Database.Prelude
(
hasConfig
)
import
Gargantext.Database.Prelude
(
hasConfig
)
...
@@ -68,7 +69,7 @@ import Gargantext.Database.Schema.Node (node_hyperdata)
...
@@ -68,7 +69,7 @@ import Gargantext.Database.Schema.Node (node_hyperdata)
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Prelude.Config
(
gc_max_docs_parsers
)
import
Gargantext.Prelude.Config
(
gc_max_docs_parsers
)
import
Gargantext.System.Logging
(
logLocM
,
LogLevel
(
..
))
import
Gargantext.System.Logging
(
logLocM
,
LogLevel
(
..
))
import
Gargantext.Utils.Jobs
(
JobHandle
,
MonadJobStatus
(
..
)
)
import
Gargantext.Utils.Jobs
.Monad
(
JobHandle
,
MonadJobStatus
(
..
)
)
import
Servant
((
:>
)(),
Capture
,
FormUrlEncoded
,
JSON
,
Summary
)
import
Servant
((
:>
)(),
Capture
,
FormUrlEncoded
,
JSON
,
Summary
)
import
Test.QuickCheck.Arbitrary
(
Arbitrary
(
..
))
import
Test.QuickCheck.Arbitrary
(
Arbitrary
(
..
))
...
@@ -265,7 +266,7 @@ addToCorpusWithForm user cid nwf jobHandle = do
...
@@ -265,7 +266,7 @@ addToCorpusWithForm user cid nwf jobHandle = do
let
ff
=
unFileFormat
$
nwf
^.
wf_fileformat
let
ff
=
unFileFormat
$
nwf
^.
wf_fileformat
let
data
'
=
case
ff
of
let
data
'
=
case
ff
of
Parser
.
Plain
->
cs
(
nwf
^.
wf_data
)
Parser
.
Plain
->
cs
(
nwf
^.
wf_data
)
Parser
.
ZIP
->
case
BSB64
.
decode
$
TE
.
encodeUtf8
(
nwf
^.
wf_data
)
of
Parser
.
ZIP
->
case
BSB64
.
decode
$
encodeUtf8
(
nwf
^.
wf_data
)
of
Left
err
->
panicTrace
$
T
.
pack
"[addToCorpusWithForm] error decoding base64: "
<>
T
.
pack
err
Left
err
->
panicTrace
$
T
.
pack
"[addToCorpusWithForm] error decoding base64: "
<>
T
.
pack
err
Right
decoded
->
decoded
Right
decoded
->
decoded
eDocsC
<-
liftBase
$
parseC
ff
data
'
eDocsC
<-
liftBase
$
parseC
ff
data
'
...
...
src/Gargantext/API/Node/Document/Export/Types.hs
View file @
bac174eb
...
@@ -14,11 +14,10 @@ Portability : POSIX
...
@@ -14,11 +14,10 @@ Portability : POSIX
module
Gargantext.API.Node.Document.Export.Types
where
module
Gargantext.API.Node.Document.Export.Types
where
import
Data.Csv
(
DefaultOrdered
(
..
),
ToNamedRecord
(
..
),
(
.=
),
header
,
namedRecord
)
import
Data.Csv
(
DefaultOrdered
(
..
),
ToNamedRecord
(
..
),
(
.=
),
header
,
namedRecord
)
import
Data.Swagger
import
Data.Swagger
(
genericDeclareNamedSchema
,
ToParamSchema
(
..
),
ToSchema
(
..
)
)
import
Data.Text.Encoding
qualified
as
TE
import
Gargantext.Core.Types
(
Node
,
TODO
)
import
Gargantext.Core.Types
import
Gargantext.Core.Utils.Prefix
(
unPrefix
,
unPrefixSwagger
)
import
Gargantext.Core.Utils.Prefix
(
unPrefix
,
unPrefixSwagger
)
import
Gargantext.Database.Admin.Types.Hyperdata
(
HyperdataDocument
(
..
)
)
import
Gargantext.Database.Admin.Types.Hyperdata
.Document
(
HyperdataDocument
(
..
)
)
import
Gargantext.Database.Schema.Node
(
NodePoly
(
..
))
import
Gargantext.Database.Schema.Node
(
NodePoly
(
..
))
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Servant
import
Servant
...
...
src/Gargantext/API/Public.hs
View file @
bac174eb
...
@@ -21,28 +21,29 @@ import Data.Aeson (Options(..), defaultOptions, genericParseJSON, genericToJSON)
...
@@ -21,28 +21,29 @@ import Data.Aeson (Options(..), defaultOptions, genericParseJSON, genericToJSON)
import
Data.List
qualified
as
List
import
Data.List
qualified
as
List
import
Data.Map.Strict
qualified
as
Map
import
Data.Map.Strict
qualified
as
Map
import
Data.Set
qualified
as
Set
import
Data.Set
qualified
as
Set
import
Data.Swagger
hiding
(
title
,
url
)
import
Data.Swagger
(
genericDeclareNamedSchema
,
ToSchema
(
..
)
)
import
Gargantext.API.Ngrams.Types
(
NgramsTerm
(
..
))
import
Gargantext.API.Node.File
(
FileApi
,
fileApi
)
import
Gargantext.API.Node.File
import
Gargantext.API.Prelude
(
serverError
,
GargServer
,
GargServerC
)
import
Gargantext.API.Prelude
import
Gargantext.Core
(
Lang
)
import
Gargantext.Core
(
Lang
)
import
Gargantext.Core.Ngrams.Types
(
NgramsTerm
(
..
))
import
Gargantext.Core.Text.Terms.Tokenize
qualified
as
Tokenize
import
Gargantext.Core.Text.Terms.Tokenize
qualified
as
Tokenize
import
Gargantext.Core.Text.Terms.Tokenize.Types
qualified
as
Tokenize
import
Gargantext.Core.Text.Terms.Tokenize.Types
qualified
as
Tokenize
import
Gargantext.Core.Types
(
TokenTag
(
..
))
import
Gargantext.Core.Types
(
TokenTag
(
..
))
import
Gargantext.Core.Utils.DateUtils
(
utc2year
)
import
Gargantext.Core.Utils.DateUtils
(
utc2year
)
import
Gargantext.Database.Admin.Types.Hyperdata
import
Gargantext.Database.Admin.Types.Hyperdata.Corpus
(
hc_fields
)
import
Gargantext.Database.Admin.Types.Hyperdata.CorpusField
import
Gargantext.Database.Admin.Types.Hyperdata.Folder
(
HyperdataFolder
)
import
Gargantext.Database.Admin.Types.Node
import
Gargantext.Database.Admin.Types.Hyperdata.CorpusField
(
CodeType
(
JSON
),
cf_authors
,
cf_desc
,
cf_query
,
cf_title
,
HyperdataField
(
HyperdataField
),
hf_data
)
import
Gargantext.Database.Admin.Types.Node
(
NodeId
(
UnsafeMkNodeId
),
Node
,
unNodeId
)
import
Gargantext.Database.Prelude
(
Cmd
,
DBCmd
)
import
Gargantext.Database.Prelude
(
Cmd
,
DBCmd
)
import
Gargantext.Database.Query.Table.Node.Error
(
HasNodeError
(
..
))
import
Gargantext.Database.Query.Table.Node.Error
(
HasNodeError
(
..
))
import
Gargantext.Database.Query.Table.NodeNode
(
selectPublicNodes
)
import
Gargantext.Database.Query.Table.NodeNode
(
selectPublicNodes
)
import
Gargantext.Database.Schema.Node
-- (NodePoly(..))
import
Gargantext.Database.Schema.Node
(
NodePoly
(
_node_id
),
node_date
,
node_hyperdata
)
-- (NodePoly(..))
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Core.Utils.Prefix
(
unPrefix
,
unPrefixSwagger
)
import
Gargantext.Core.Utils.Prefix
(
unPrefix
,
unPrefixSwagger
)
import
Gargantext.Utils.Aeson
qualified
as
GUA
import
Gargantext.Utils.Aeson
qualified
as
GUA
import
Servant
import
Servant
import
Test.QuickCheck
(
elements
)
import
Test.QuickCheck
(
elements
)
import
Test.QuickCheck.Arbitrary
import
Test.QuickCheck.Arbitrary
(
Arbitrary
(
arbitrary
)
)
------------------------------------------------------------------------
------------------------------------------------------------------------
type
API
=
API_Home
type
API
=
API_Home
...
...
src/Gargantext/Core/Ngrams/Types.hs
View file @
bac174eb
...
@@ -35,13 +35,9 @@ module Gargantext.Core.Ngrams.Types
...
@@ -35,13 +35,9 @@ module Gargantext.Core.Ngrams.Types
,
NgramsElement
(
..
)
,
NgramsElement
(
..
)
,
mkNgramsElement
,
mkNgramsElement
,
ne_children
,
ne_list
,
ne_ngrams
,
ne_ngrams
,
ne_nre
,
ne_occurrences
,
ne_occurrences
,
ne_parent
,
ne_root
,
ne_size
,
NgramsList
,
NgramsList
...
@@ -122,7 +118,8 @@ import Codec.Serialise (Serialise())
...
@@ -122,7 +118,8 @@ import Codec.Serialise (Serialise())
import
Control.Category
((
>>>
))
import
Control.Category
((
>>>
))
import
Control.Lens
(
makePrisms
,
Iso
'
,
iso
,
from
,
(
.=
),
(
?=
),
(
#
),
to
,
folded
,
{-withIndex, ifolded,-}
view
,
use
,
(
^?
),
(
%~
),
(
%=
),
at
,
_Just
,
Each
(
..
),
itraverse_
,
both
,
forOf_
,
(
?~
),
over
)
import
Control.Lens
(
makePrisms
,
Iso
'
,
iso
,
from
,
(
.=
),
(
?=
),
(
#
),
to
,
folded
,
{-withIndex, ifolded,-}
view
,
use
,
(
^?
),
(
%~
),
(
%=
),
at
,
_Just
,
Each
(
..
),
itraverse_
,
both
,
forOf_
,
(
?~
),
over
)
import
Control.Monad.State
import
Control.Monad.State
import
Data.Aeson
(
FromJSONKey
(
..
),
FromJSONKeyFunction
(
..
),
ToJSONKey
(
..
),
decode
,
encode
,
genericFromJSONKey
,
defaultJSONKeyOptions
,
genericToJSONKey
,
defaultJSONKeyOptions
,
genericParseJSON
,
genericToEncoding
,
genericToJSON
)
import
Data.Aeson
qualified
as
Aeson
import
Data.Aeson
(
FromJSONKey
(
..
),
FromJSONKeyFunction
(
..
),
ToJSONKey
(
..
),
decode
,
encode
,
genericFromJSONKey
,
defaultJSONKeyOptions
,
genericToJSONKey
,
defaultJSONKeyOptions
,
genericParseJSON
,
genericToEncoding
,
genericToJSON
,
(
.:
),
(
.:?
),
withObject
,
object
)
import
Data.Foldable
import
Data.Foldable
import
Data.HashMap.Strict.InsOrd
qualified
as
InsOrdHashMap
import
Data.HashMap.Strict.InsOrd
qualified
as
InsOrdHashMap
import
Data.Map.Strict
qualified
as
Map
import
Data.Map.Strict
qualified
as
Map
...
...
src/Gargantext/Core/Text/Corpus/Parsers/FrameWrite.hs
View file @
bac174eb
...
@@ -15,7 +15,7 @@ module Gargantext.Core.Text.Corpus.Parsers.FrameWrite
...
@@ -15,7 +15,7 @@ module Gargantext.Core.Text.Corpus.Parsers.FrameWrite
import
Data.Either
import
Data.Either
import
Data.List
qualified
as
List
import
Data.List
qualified
as
List
import
Data.Maybe
import
Data.Maybe
import
Data.Text
hiding
(
foldl
)
import
Data.Text
hiding
(
foldl
'
)
import
Data.Text
qualified
as
DT
import
Data.Text
qualified
as
DT
import
Gargantext.Core.Text
(
sentences
)
import
Gargantext.Core.Text
(
sentences
)
import
Gargantext.Prelude
hiding
(
ByteString
,
(
<|>
),
many
,
try
)
import
Gargantext.Prelude
hiding
(
ByteString
,
(
<|>
),
many
,
try
)
...
...
src/Gargantext/Core/Text/Terms/Tokenize.hs
View file @
bac174eb
...
@@ -10,7 +10,6 @@ Portability : POSIX
...
@@ -10,7 +10,6 @@ Portability : POSIX
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE TemplateHaskell #-}
module
Gargantext.Core.Text.Terms.Tokenize
module
Gargantext.Core.Text.Terms.Tokenize
where
where
...
@@ -22,11 +21,11 @@ import Data.Interval qualified as I
...
@@ -22,11 +21,11 @@ import Data.Interval qualified as I
import
Data.IntervalSet
qualified
as
IS
import
Data.IntervalSet
qualified
as
IS
import
Data.Set
qualified
as
Set
import
Data.Set
qualified
as
Set
import
Data.Text
qualified
as
T
import
Data.Text
qualified
as
T
import
Gargantext.API.Ngrams.Types
(
NgramsTerm
(
..
))
import
Gargantext.Core
(
Lang
,
NLPServerConfig
(
..
))
--, PosTagAlgo(CoreNLP))
import
Gargantext.Core
(
Lang
,
NLPServerConfig
(
..
))
--, PosTagAlgo(CoreNLP))
import
Gargantext.Core.Ngrams.Types
(
NgramsTerm
(
..
))
import
Gargantext.Core.NLP
(
nlpServerGet
)
import
Gargantext.Core.NLP
(
nlpServerGet
)
import
Gargantext.Core.Text.Terms.Multi
(
tokenTagsNoGroup
)
import
Gargantext.Core.Text.Terms.Multi
(
tokenTagsNoGroup
)
import
Gargantext.Core.Text.Terms.Tokenize.Types
import
Gargantext.Core.Text.Terms.Tokenize.Types
(
HighlightResult
(
..
),
HighlightedTerm
(
..
),
HasTokenizer
,
htToInterval
,
intervalToNt
,
hrToInterval
)
import
Gargantext.Core.Types
(
TokenTag
(
..
),
POS
(
..
))
--, my_token_offset_end)
import
Gargantext.Core.Types
(
TokenTag
(
..
),
POS
(
..
))
--, my_token_offset_end)
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Utils.Array
(
window
)
import
Gargantext.Utils.Array
(
window
)
...
...
src/Gargantext/Core/Text/Terms/Tokenize/Types.hs
View file @
bac174eb
...
@@ -16,7 +16,7 @@ module Gargantext.Core.Text.Terms.Tokenize.Types
...
@@ -16,7 +16,7 @@ module Gargantext.Core.Text.Terms.Tokenize.Types
where
where
import
Control.Monad.Trans.Control
(
MonadBaseControl
)
import
Control.Monad.Trans.Control
(
MonadBaseControl
)
import
Data.Aeson.TH
(
defaultOptions
,
deriveJSON
)
import
Data.Aeson.TH
(
defaultOptions
)
import
Data.ExtendedReal
(
Extended
(
..
))
import
Data.ExtendedReal
(
Extended
(
..
))
import
Data.Interval
((
<=..<=
))
import
Data.Interval
((
<=..<=
))
import
Data.Interval
qualified
as
I
import
Data.Interval
qualified
as
I
...
...
src/Gargantext/Database/Action/Flow/List.hs
View file @
bac174eb
...
@@ -24,7 +24,7 @@ import Data.Map.Strict.Patch qualified as PM
...
@@ -24,7 +24,7 @@ import Data.Map.Strict.Patch qualified as PM
import
Gargantext.Core.Ngrams.Tools
(
getNodeStory
)
import
Gargantext.Core.Ngrams.Tools
(
getNodeStory
)
import
Gargantext.Core.Ngrams.Types
import
Gargantext.Core.Ngrams.Types
(
NgramsTerm
(
NgramsTerm
),
(
NgramsTerm
(
NgramsTerm
),
NgramsRepoElement
,
NgramsRepoElement
(
..
)
,
NgramsElement
(
..
),
NgramsElement
(
..
),
ne_ngrams
,
ne_ngrams
,
NgramsPatch
(
NgramsReplace
),
NgramsPatch
(
NgramsReplace
),
...
...
src/Gargantext/Utils/Servant.hs
View file @
bac174eb
...
@@ -14,14 +14,13 @@ import qualified Data.ByteString.Lazy.Char8 as BSC
...
@@ -14,14 +14,13 @@ import qualified Data.ByteString.Lazy.Char8 as BSC
import
Data.Csv
(
defaultEncodeOptions
,
encodeByNameWith
,
encodeDefaultOrderedByName
,
header
,
namedRecord
,
(
.=
),
DefaultOrdered
,
EncodeOptions
(
..
),
NamedRecord
,
Quoting
(
QuoteNone
),
ToNamedRecord
)
import
Data.Csv
(
defaultEncodeOptions
,
encodeByNameWith
,
encodeDefaultOrderedByName
,
header
,
namedRecord
,
(
.=
),
DefaultOrdered
,
EncodeOptions
(
..
),
NamedRecord
,
Quoting
(
QuoteNone
),
ToNamedRecord
)
import
qualified
Data.Map.Strict
as
Map
import
qualified
Data.Map.Strict
as
Map
import
qualified
Data.Text
as
T
import
qualified
Data.Text
as
T
import
qualified
Data.Text.Encoding
as
TE
import
Gargantext.Core.Ngrams.Types
(
mSetToList
,
NgramsRepoElement
(
..
),
NgramsTableMap
,
NgramsTerm
(
..
),
unNgramsTerm
)
import
Gargantext.Core.Ngrams.Types
(
mSetToList
,
NgramsRepoElement
(
..
),
NgramsTableMap
,
NgramsTerm
(
..
),
unNgramsTerm
)
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Gargantext.Core.Types.Main
(
ListType
(
..
))
import
Network.HTTP.Media
((
//
),
(
/:
))
import
Network.HTTP.Media
((
//
),
(
/:
))
import
qualified
Prelude
import
qualified
Prelude
import
Protolude
import
Protolude
import
Protolude.Partial
(
read
)
import
Protolude.Partial
(
read
)
import
Servant
import
Servant
(
Accept
(
contentType
),
MimeRender
(
..
),
MimeUnrender
(
mimeUnrender
)
)
data
CSV
=
CSV
data
CSV
=
CSV
...
...
src/Gargantext/Utils/SpacyNLP.hs
View file @
bac174eb
...
@@ -13,82 +13,24 @@ Server to be used: https://gitlab.iscpif.fr/gargantext/spacy-server
...
@@ -13,82 +13,24 @@ Server to be used: https://gitlab.iscpif.fr/gargantext/spacy-server
-}
-}
{-# LANGUAGE TemplateHaskell #-}
module
Gargantext.Utils.SpacyNLP
(
module
Gargantext.Utils.SpacyNLP
(
module
Gargantext
.
Utils
.
SpacyNLP
.
Types
spacyRequest
,
spacyRequest
,
spacyTagsToToken
,
spacyTagsToToken
,
spacyDataTo
PosSentence
s
,
spacyDataTo
TokenTag
s
,
nlp
,
nlp
TokenTags
)
where
)
where
import
Data.Aeson
(
encode
)
import
Data.Aeson
(
encode
)
import
Data.Aeson.TH
(
deriveJSON
)
import
Data.Set
qualified
as
Set
import
Data.Set
qualified
as
Set
import
Data.Text
hiding
(
map
,
group
,
filter
,
concat
,
zip
)
import
Data.Text
(
splitOn
,
toLower
)
import
Gargantext.Core
(
Lang
(
..
))
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
(
Token
(
Token
)
)
import
Gargantext.Core.Text.Terms.Multi.PosTagging.Types
import
Gargantext.Core.Types
(
TokenTag
(
..
))
import
Gargantext.Core.Types
(
POS
(
..
),
NER
(
..
),
TokenTag
(
..
))
import
Gargantext.Core.Utils.Prefix
(
unPrefix
)
import
Gargantext.Prelude
import
Gargantext.Prelude
import
Gargantext.Utils.SpacyNLP.Types
import
Gargantext.Utils.SpacyNLP.Types
import
Network.HTTP.Simple
(
parseRequest
,
httpJSON
,
setRequestBodyLBS
,
getResponseBody
,
Response
)
import
Network.HTTP.Simple
(
parseRequest
,
httpJSON
,
setRequestBodyLBS
,
getResponseBody
,
Response
)
import
Network.URI
(
URI
(
..
))
import
Network.URI
(
URI
(
..
))
data
SpacyData
=
SpacyData
{
_spacy_data
::
!
[
SpacyText
]
}
deriving
(
Show
)
data
SpacyText
=
SpacyText
{
_spacy_text
::
!
Text
,
_spacy_tags
::
!
[
SpacyTags
]
}
deriving
(
Show
)
-- | https://spacy.io/api/token/#attributes
data
SpacyTags
=
SpacyTags
{
_spacyTags_text
::
!
Text
,
_spacyTags_text_with_ws
::
!
Text
,
_spacyTags_whitespace
::
!
Text
,
_spacyTags_head
::
!
Text
,
_spacyTags_head_index
::
!
Int
,
_spacyTags_left_edge
::
!
Text
,
_spacyTags_right_edge
::
!
Text
,
_spacyTags_index
::
Int
,
_spacyTags_ent_type
::
!
NER
,
_spacyTags_ent_iob
::
!
Text
,
_spacyTags_lemma
::
!
Text
,
_spacyTags_normalized
::
!
Text
,
_spacyTags_shape
::
!
Text
,
_spacyTags_prefix
::
!
Text
,
_spacyTags_suffix
::
!
Text
,
_spacyTags_is_alpha
::
Bool
,
_spacyTags_is_ascii
::
Bool
,
_spacyTags_is_digit
::
Bool
,
_spacyTags_is_title
::
Bool
,
_spacyTags_is_punct
::
Bool
,
_spacyTags_is_left_punct
::
Bool
,
_spacyTags_is_right_punct
::
Bool
,
_spacyTags_is_space
::
Bool
,
_spacyTags_is_bracket
::
Bool
,
_spacyTags_is_quote
::
Bool
,
_spacyTags_is_currency
::
Bool
,
_spacyTags_like_url
::
Bool
,
_spacyTags_like_num
::
Bool
,
_spacyTags_like_email
::
Bool
,
_spacyTags_is_oov
::
Bool
,
_spacyTags_is_stop
::
Bool
,
_spacyTags_pos
::
POS
,
_spacyTags_tag
::
POS
,
_spacyTags_dep
::
!
Text
,
_spacyTags_lang
::
!
Text
,
_spacyTags_prob
::
!
Int
,
_spacyTags_char_offset
::
!
Int
}
deriving
(
Show
)
data
SpacyRequest
=
SpacyRequest
{
_spacyRequest_text
::
!
Text
}
deriving
(
Show
)
spacyRequest
::
URI
->
Text
->
IO
SpacyData
spacyRequest
::
URI
->
Text
->
IO
SpacyData
spacyRequest
uri
txt
=
do
spacyRequest
uri
txt
=
do
req
<-
parseRequest
$
"POST "
<>
show
(
uri
{
uriPath
=
"/pos"
})
req
<-
parseRequest
$
"POST "
<>
show
(
uri
{
uriPath
=
"/pos"
})
...
...
src/Gargantext/Utils/SpacyNLP/Types.hs
View file @
bac174eb
...
@@ -29,6 +29,7 @@ data SpacyData = SpacyData { _spacy_data :: ![SpacyText]}
...
@@ -29,6 +29,7 @@ data SpacyData = SpacyData { _spacy_data :: ![SpacyText]}
data
SpacyText
=
SpacyText
{
_spacy_text
::
!
Text
data
SpacyText
=
SpacyText
{
_spacy_text
::
!
Text
,
_spacy_tags
::
!
[
SpacyTags
]
,
_spacy_tags
::
!
[
SpacyTags
]
}
deriving
(
Show
)
}
deriving
(
Show
)
-- | https://spacy.io/api/token/#attributes
data
SpacyTags
=
data
SpacyTags
=
SpacyTags
{
_spacyTags_text
::
!
Text
SpacyTags
{
_spacyTags_text
::
!
Text
,
_spacyTags_text_with_ws
::
!
Text
,
_spacyTags_text_with_ws
::
!
Text
...
...
stack.yaml
View file @
bac174eb
"
allow-newer"
:
true
"
allow-newer"
:
true
"
extra-deps"
:
"
extra-deps"
:
-
corenlp-types-0.1.0.0@sha256:96538c47dddc96dff11922f8bfa1fd27586fbcce42f6ca2a7fc06cd652a36850,3557
-
"
HSvm-0.1.1.3.22"
-
"
HSvm-0.1.1.3.22"
-
"
KMP-0.2.0.0"
-
"
KMP-0.2.0.0"
-
"
MissingH-1.4.3.0"
-
"
MissingH-1.4.3.0"
...
@@ -40,7 +41,7 @@
...
@@ -40,7 +41,7 @@
-
"
servant-flatten-0.2"
-
"
servant-flatten-0.2"
-
"
servant-server-0.20"
-
"
servant-server-0.20"
-
"
snap-server-1.1.2.1"
-
"
snap-server-1.1.2.1"
-
"
s
temmer-0.5.2
"
-
"
s
nowball-1.0.0.1
"
-
"
taggy-0.2.1"
-
"
taggy-0.2.1"
-
"
taggy-lens-0.1.2"
-
"
taggy-lens-0.1.2"
-
"
tomland-1.3.3.2"
-
"
tomland-1.3.3.2"
...
@@ -247,8 +248,6 @@ flags:
...
@@ -247,8 +248,6 @@ flags:
"
bytestring--lt-0_10_4"
:
false
"
bytestring--lt-0_10_4"
:
false
"
cassava-conduit"
:
"
cassava-conduit"
:
small_base
:
true
small_base
:
true
cborg
:
"
optimize-gmp"
:
true
cereal
:
cereal
:
"
bytestring-builder"
:
false
"
bytestring-builder"
:
false
"
cipher-aes"
:
"
cipher-aes"
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment