Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
145
Issues
145
List
Board
Labels
Milestones
Merge Requests
6
Merge Requests
6
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
09038c19
Commit
09038c19
authored
Feb 06, 2018
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[PROTOLUDE] cleaning map and imports.
parent
79f3c204
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
44 additions
and
37 deletions
+44
-37
User.hs
src/Gargantext/Database/User.hs
+10
-8
CoreNLP.hs
src/Gargantext/Ngrams/CoreNLP.hs
+6
-4
Parser.hs
src/Gargantext/Ngrams/Parser.hs
+7
-6
Date.hs
src/Gargantext/Parsers/Date.hs
+6
-5
Prelude.hs
src/Gargantext/Prelude.hs
+15
-14
No files found.
src/Gargantext/Database/User.hs
View file @
09038c19
{-# OPTIONS_GHC -fno-warn-name-shadowing #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE FunctionalDependencies #-}
{-# LANGUAGE Arrows #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE FunctionalDependencies #-}
{-# LANGUAGE Arrows #-}
{-# LANGUAGE NoImplicitPrelude #-}
module
Gargantext.Database.User
where
import
Prelude
import
Gargantext.Prelude
import
GHC.Show
(
Show
(
..
))
import
Data.Eq
(
Eq
(
..
))
import
Data.Time
(
UTCTime
)
import
Data.Text
(
Text
)
import
Data.Maybe
(
Maybe
)
...
...
@@ -120,4 +122,4 @@ users = do
usersLight
::
IO
[
UserLight
]
usersLight
=
do
conn
<-
PGS
.
connect
infoGargandb
pm
toUserLight
<$>
runQuery
conn
queryUserTable
map
toUserLight
<$>
runQuery
conn
queryUserTable
src/Gargantext/Ngrams/CoreNLP.hs
View file @
09038c19
...
...
@@ -2,6 +2,7 @@
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE NoImplicitPrelude #-}
module
Gargantext.Ngrams.CoreNLP
where
...
...
@@ -9,6 +10,7 @@ import Data.Aeson
import
Data.Aeson.TH
(
deriveJSON
)
import
GHC.Generics
import
Data.Monoid
((
<>
))
import
GHC.Show
(
Show
(
..
))
import
Gargantext.Types.Main
(
Language
(
..
))
import
Gargantext.Prelude
...
...
@@ -69,7 +71,7 @@ instance FromJSON Sentences
--
corenlpPretty
::
String
->
IO
()
corenlpPretty
::
Text
->
IO
()
corenlpPretty
txt
=
do
url
<-
parseRequest
"POST http://localhost:9000/?properties={
\"
annotators
\"
:
\"
tokenize,ssplit,pos,ner
\"
,
\"
outputFormat
\"
:
\"
json
\"
}"
let
request
=
setRequestBodyJSON
txt
url
...
...
@@ -80,7 +82,7 @@ corenlpPretty txt = do
-- print $ getResponseHeader "Content-Type" response
S8
.
putStrLn
$
Yaml
.
encode
(
getResponseBody
response
::
Sentences
)
corenlp
::
Language
->
String
->
IO
Sentences
corenlp
::
Language
->
Text
->
IO
Sentences
corenlp
lang
txt
=
do
let
properties
=
case
lang
of
EN
->
"{
\"
annotators
\"
:
\"
tokenize,ssplit,pos,ner
\"
,
\"
outputFormat
\"
:
\"
json
\"
}"
...
...
@@ -99,8 +101,8 @@ corenlp lang txt = do
-- Named Entity Recognition example
-- parseWith _tokenNer "Hello world of Peter."
-- [[("``","O"),("Hello","O"),("world","O"),("of","O"),("Peter","PERSON"),(".","O"),("''","O")]]
tokenWith
::
(
Token
->
t
)
->
Language
->
String
->
IO
[[(
Text
,
t
)]]
tokenWith
f
lang
s
=
pm
(
pm
(
\
t
->
(
_tokenWord
t
,
f
t
)))
<$>
pm
_sentenceTokens
<$>
sentences
<$>
corenlp
lang
s
tokenWith
::
(
Token
->
t
)
->
Language
->
Text
->
IO
[[(
Text
,
t
)]]
tokenWith
f
lang
s
=
map
(
map
(
\
t
->
(
_tokenWord
t
,
f
t
)))
<$>
map
_sentenceTokens
<$>
sentences
<$>
corenlp
lang
s
src/Gargantext/Ngrams/Parser.hs
View file @
09038c19
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE NoImplicitPrelude #-}
module
Gargantext.Ngrams.Parser
where
import
Gargantext.Prelude
import
Gargantext.Ngrams.CoreNLP
import
Data.Text
hiding
(
map
)
import
Gargantext.Types.Main
(
Language
(
..
),
Ngrams
)
import
qualified
Gargantext.Ngrams.Lang.En
as
En
...
...
@@ -30,13 +31,13 @@ import qualified Gargantext.Ngrams.Lang.Fr as Fr
-- TODO for scientific papers: add maesures
-- TODO add the p score regex
extractNgrams
::
Language
->
String
->
IO
[[
Ngrams
]]
extractNgrams
lang
s
=
pm
(
groupNgrams
lang
)
<$>
extractNgrams'
lang
s
extractNgrams
::
Language
->
Text
->
IO
[[
Ngrams
]]
extractNgrams
lang
s
=
map
(
groupNgrams
lang
)
<$>
extractNgrams'
lang
s
extractNgrams'
::
Language
->
String
->
IO
[[
Ngrams
]]
extractNgrams'
lang
t
=
pm
(
pm
token2text
)
<$>
pm
_sentenceTokens
extractNgrams'
::
Language
->
Text
->
IO
[[
Ngrams
]]
extractNgrams'
lang
t
=
map
(
map
token2text
)
<$>
map
_sentenceTokens
<$>
sentences
<$>
corenlp
lang
t
...
...
src/Gargantext/Parsers/Date.hs
View file @
09038c19
...
...
@@ -15,6 +15,7 @@ DGP.parseDate1 DGP.FR "12 avril 2010" == "2010-04-12T00:00:00.000+00:00"
-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE NoImplicitPrelude #-}
module
Gargantext.Parsers.Date
(
parseDate1
,
Lang
(
FR
,
EN
),
parseDate
)
where
...
...
@@ -33,7 +34,7 @@ import Duckling.Types (jsonValue, Entity)
import
Duckling.Api
(
analyze
,
parse
)
import
qualified
Data.HashSet
as
HashSet
import
qualified
Data.Aeson
as
Json
import
Data.HashMap.Strict
as
HM
import
Data.HashMap.Strict
as
HM
hiding
(
map
)
import
Data.Text
(
Text
)
-- import Duckling.Engine (parseAndResolve)
...
...
@@ -59,13 +60,13 @@ import Safe (headMay)
-- parseDate1 :: Context -> Text -> SomeErrorHandling Text
parseDate1
::
Lang
->
Text
->
IO
Text
parseDate1
lang
text
=
do
maybeJson
<-
pm
jsonValue
<$>
parseDateWithDuckling
lang
text
maybeJson
<-
map
jsonValue
<$>
parseDateWithDuckling
lang
text
case
headMay
maybeJson
of
Just
(
Json
.
Object
object
)
->
case
HM
.
lookup
"value"
object
of
Just
(
Json
.
String
date
)
->
pure
date
Just
_
->
error
"ParseDate ERROR: should be a json String"
Nothing
->
error
"ParseDate ERROR: no date found"
_
->
error
"ParseDate ERROR: type error"
Just
_
->
panic
"ParseDate ERROR: should be a json String"
Nothing
->
panic
"ParseDate ERROR: no date found"
_
->
panic
"ParseDate ERROR: type error"
...
...
src/Gargantext/Prelude.hs
View file @
09038c19
...
...
@@ -14,7 +14,9 @@ module Gargantext.Prelude
where
import
Protolude
(
Bool
(
True
,
False
),
Int
,
Double
,
Integer
,
Fractional
,
Num
,
Maybe
,
Floating
,
Char
,
Fractional
,
Num
,
Maybe
(
Just
,
Nothing
)
,
Floating
,
Char
,
IO
,
pure
,
(
<$>
),
panic
,
Ord
,
Integral
,
Foldable
,
RealFrac
,
Monad
,
filter
,
reverse
,
map
,
zip
,
drop
,
take
,
zipWith
,
sum
,
fromIntegral
,
length
,
fmap
...
...
@@ -33,18 +35,17 @@ import qualified Data.Map as Map
import
qualified
Data.Vector
as
V
import
Safe
(
headMay
)
pf
::
(
a
->
Bool
)
->
[
a
]
->
[
a
]
pf
=
filter
pr
::
[
a
]
->
[
a
]
pr
=
reverse
pm
::
(
a
->
b
)
->
[
a
]
->
[
b
]
pm
=
map
--
pm :: (a -> b) -> [a] -> [b]
--
pm = map
pm2
::
(
t
->
b
)
->
[[
t
]]
->
[[
b
]]
pm2
fun
=
pm
(
pm
fun
)
pm2
fun
=
map
(
map
fun
)
pz
::
[
a
]
->
[
b
]
->
[(
a
,
b
)]
pz
=
zip
...
...
@@ -73,14 +74,14 @@ sumMaybe :: Num a => [Maybe a] -> Maybe a
sumMaybe
=
fmap
sum
.
M
.
sequence
variance
::
Floating
a
=>
[
a
]
->
a
variance
xs
=
mean
$
pm
(
\
x
->
(
x
-
m
)
**
2
)
xs
where
variance
xs
=
mean
$
map
(
\
x
->
(
x
-
m
)
**
2
)
xs
where
m
=
mean
xs
deviation
::
[
Double
]
->
Double
deviation
=
sqrt
.
variance
movingAverage
::
Fractional
b
=>
Int
->
[
b
]
->
[
b
]
movingAverage
steps
xs
=
pm
mean
$
chunkAlong
steps
1
xs
movingAverage
steps
xs
=
map
mean
$
chunkAlong
steps
1
xs
ma
::
[
Double
]
->
[
Double
]
ma
=
movingAverage
3
...
...
@@ -90,7 +91,7 @@ ma = movingAverage 3
chunkAlong
::
Int
->
Int
->
[
a
]
->
[[
a
]]
chunkAlong
a
b
l
=
only
(
while
dropAlong
)
where
only
=
pm
(
take
a
)
only
=
map
(
take
a
)
while
=
takeWhile
(
\
x
->
length
x
>=
a
)
dropAlong
=
L
.
scanl
(
\
x
_y
->
drop
b
x
)
l
([
1
..
]
::
[
Integer
])
...
...
@@ -172,18 +173,18 @@ scale :: [Double] -> [Double]
scale
=
scaleMinMax
scaleMinMax
::
[
Double
]
->
[
Double
]
scaleMinMax
xs
=
pm
(
\
x
->
(
x
-
mi
/
(
ma
-
mi
+
1
)
))
xs'
scaleMinMax
xs
=
map
(
\
x
->
(
x
-
mi
/
(
ma
-
mi
+
1
)
))
xs'
where
ma
=
maximum
xs'
mi
=
minimum
xs'
xs'
=
pm
abs
xs
xs'
=
map
abs
xs
scaleNormalize
::
[
Double
]
->
[
Double
]
scaleNormalize
xs
=
pm
(
\
x
->
(
x
-
v
/
(
m
+
1
)))
xs'
scaleNormalize
xs
=
map
(
\
x
->
(
x
-
v
/
(
m
+
1
)))
xs'
where
v
=
variance
xs'
m
=
mean
xs'
xs'
=
pm
abs
xs
xs'
=
map
abs
xs
...
...
@@ -191,9 +192,9 @@ normalize :: [Double] -> [Double]
normalize
as
=
normalizeWith
identity
as
normalizeWith
::
Fractional
b
=>
(
a
->
b
)
->
[
a
]
->
[
b
]
normalizeWith
extract
bs
=
pm
(
\
x
->
x
/
(
sum
bs'
))
bs'
normalizeWith
extract
bs
=
map
(
\
x
->
x
/
(
sum
bs'
))
bs'
where
bs'
=
pm
extract
bs
bs'
=
map
extract
bs
-- Zip functions to add
zipFst
::
([
b
]
->
[
a
])
->
[
b
]
->
[(
a
,
b
)]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment