Merge remote-tracking branch...

Merge remote-tracking branch 'origin/184-dev-add-support-for-multiple-languages-in-ini-file' into dev-merge

gargantext.cabal

@@ -127,6 +127,7 @@ library
       Gargantext.API.GraphQL.AsyncTask
       Gargantext.API.GraphQL.Context
       Gargantext.API.GraphQL.IMT
+      Gargantext.API.GraphQL.NLP
       Gargantext.API.GraphQL.Node
       Gargantext.API.GraphQL.Team
       Gargantext.API.GraphQL.TreeFirstLevel
@@ -214,11 +215,12 @@ library
       Gargantext.Core.Text.Metrics.FrequentItemSet
       Gargantext.Core.Text.Metrics.SpeGen.IncExc
       Gargantext.Core.Text.Metrics.Utils
-      Gargantext.Core.Text.Samples.CH
+      Gargantext.Core.Text.Samples.CN
       Gargantext.Core.Text.Samples.DE
       Gargantext.Core.Text.Samples.EN
+      Gargantext.Core.Text.Samples.ES
       Gargantext.Core.Text.Samples.FR
-      Gargantext.Core.Text.Samples.SP
+      Gargantext.Core.Text.Samples.PL
       Gargantext.Core.Text.Terms.Mono.Stem
       Gargantext.Core.Text.Terms.Mono.Stem.En
       Gargantext.Core.Text.Terms.Mono.Token

src/Gargantext/API/GraphQL.hs

@@ -38,6 +38,7 @@ import qualified Gargantext.API.GraphQL.Annuaire as GQLA
 import qualified Gargantext.API.GraphQL.AsyncTask as GQLAT
 import qualified Gargantext.API.GraphQL.Context as GQLCTX
 import qualified Gargantext.API.GraphQL.IMT as GQLIMT
+import qualified Gargantext.API.GraphQL.NLP as GQLNLP
 import qualified Gargantext.API.GraphQL.Node as GQLNode
 import qualified Gargantext.API.GraphQL.User as GQLUser
 import qualified Gargantext.API.GraphQL.UserInfo as GQLUserInfo
@@ -70,6 +71,7 @@ data Query m
     , contexts_for_ngrams :: GQLCTX.ContextsForNgramsArgs -> m [GQLCTX.ContextGQL]
     , imt_schools         :: GQLIMT.SchoolsArgs -> m [GQLIMT.School]
     , job_logs            :: GQLAT.JobLogArgs -> m (Map Int JobLog)
+    , languages           :: GQLNLP.LanguagesArgs -> m [GQLNLP.Lang]
     , nodes               :: GQLNode.NodeArgs -> m [GQLNode.Node]
     , node_parent         :: GQLNode.NodeParentArgs -> m [GQLNode.Node]
     , user_infos          :: GQLUserInfo.UserInfoArgs -> m [GQLUserInfo.UserInfo]
@@ -112,6 +114,7 @@ rootResolver =
                             , contexts_for_ngrams = GQLCTX.resolveContextsForNgrams
                             , imt_schools         = GQLIMT.resolveSchools
                             , job_logs            = GQLAT.resolveJobLogs
+                            , languages           = GQLNLP.resolveLanguages
                             , nodes               = GQLNode.resolveNodes
                             , node_parent         = GQLNode.resolveNodeParent
                             , user_infos          = GQLUserInfo.resolveUserInfos

src/Gargantext/API/GraphQL/NLP.hs

+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+
+module Gargantext.API.GraphQL.NLP
+  ( Lang(..)
+  , LanguagesArgs(..)
+  , resolveLanguages
+  )
+  where
+
+import Data.Morpheus.Types
+  ( GQLType
+  , Resolver
+  , QUERY
+  )
+import Gargantext.API.Prelude (GargM, GargError)
+import Gargantext.Core (Lang(..), allLangs)
+import Gargantext.Prelude
+import GHC.Generics (Generic)
+
+data LanguagesArgs
+  = LanguagesArgs
+    { } deriving (Generic, GQLType)
+
+type GqlM e env = Resolver QUERY e (GargM env GargError)
+
+resolveLanguages
+  :: LanguagesArgs -> GqlM e env [Lang]
+resolveLanguages LanguagesArgs { } = pure $ allLangs

src/Gargantext/API/Node/Corpus/New.hs

@@ -44,7 +44,7 @@ import Gargantext.API.Node.Corpus.New.Types
 import Gargantext.API.Node.Corpus.Searx
 import Gargantext.API.Node.Corpus.Types
 import Gargantext.API.Node.Types
-import Gargantext.Core (Lang(..){-, allLangs-})
+import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)

src/Gargantext/API/Node/Corpus/Searx.hs

@@ -47,6 +47,11 @@ import qualified Gargantext.Database.Query.Table.Node.Document.Add  as Doc  (add
 langToSearx :: Lang -> Text
 langToSearx EN = "en-US"
 langToSearx FR = "fr-FR"
+langToSearx DE = "de-FR"
+langToSearx ES = "es-FR"
+langToSearx IT = "it-FR"
+langToSearx PL = "pl-FR"
+langToSearx CN = "cn-FR"
 langToSearx All = "en-US"
 
 data SearxResult = SearxResult

src/Gargantext/API/Node/Types.hs

@@ -15,7 +15,7 @@ import GHC.Generics (Generic)
 import Servant.Job.Utils (jsonOptions)
 import Web.FormUrlEncoded          (FromForm, ToForm)
 
-import Gargantext.Core (Lang(..){-, allLangs-})
+import Gargantext.Core (Lang(..))
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Prelude
 import qualified Gargantext.Database.GargDB as GargDB

src/Gargantext/Core.hs

@@ -28,30 +28,37 @@ import Servant.API
 -- | Language of a Text
 -- For simplicity, we suppose text has an homogenous language
 --
--- Next steps: | DE | IT | SP
---
 --  - EN == english
 --  - FR == french
---  - DE == deutch  (not implemented yet)
---  - IT == italian (not implemented yet)
---  - SP == spanish (not implemented yet)
+--  - DE == deutch
+--  - IT == italian
+--  - ES == spanish
+--  - PL == polish
+--  - CN == chinese
 --
 --  ... add your language and help us to implement it (:
 
 -- | All languages supported
--- TODO : DE | SP | CH
-data Lang = EN | FR | All
-  deriving (Show, Eq, Ord, Bounded, Enum, Generic, GQLType)
+-- NOTE: Use international country codes
+-- https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes
+data Lang = EN | FR | DE | IT | PL | ES | CN | All
+  deriving (Show, Eq, Ord, Enum, Bounded, Generic, GQLType)
 
 instance ToJSON Lang
 instance FromJSON Lang
-instance ToSchema Lang
+instance ToSchema Lang where
+  declareNamedSchema = genericDeclareNamedSchemaUnrestricted defaultSchemaOptions
 instance FromHttpApiData Lang
   where
     parseUrlPiece "EN"  = pure EN
     parseUrlPiece "FR"  = pure FR
+    parseUrlPiece "DE"  = pure DE
+    parseUrlPiece "ES"  = pure ES
+    parseUrlPiece "IT"  = pure IT
+    parseUrlPiece "PL"  = pure PL
+    parseUrlPiece "CN"  = pure CN
     parseUrlPiece "All" = pure All
-    parseUrlPiece _     = Left "Unexpected value of OrderBy"
+    parseUrlPiece _     = Left "Unexpected value of Lang"
 instance ToHttpApiData Lang where
   toUrlPiece = pack . show
 instance Hashable Lang
@@ -63,14 +70,27 @@ class HasDBid a where
   toDBid   :: a   -> Int
   fromDBid :: Int -> a
 
+-- NOTE: We try to use numeric codes for countries
+-- https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes
+-- https://en.wikipedia.org/wiki/ISO_3166-1_numeric#004
 instance HasDBid Lang where
   toDBid All = 0
   toDBid FR  = 1
   toDBid EN  = 2
+  toDBid DE  = 276
+  toDBid ES  = 724
+  toDBid IT  = 380
+  toDBid PL  = 616
+  toDBid CN  = 156
 
   fromDBid 0 = All
   fromDBid 1 = FR
   fromDBid 2 = EN
+  fromDBid 276 = DE
+  fromDBid 724 = ES
+  fromDBid 380 = IT
+  fromDBid 616 = PL
+  fromDBid 156 = CN
   fromDBid _ = panic "HasDBid lang, not implemented"
 
 ------------------------------------------------------------------------

src/Gargantext/Core/NLP.hs

@@ -4,7 +4,7 @@ import Control.Lens (Getter, at, non)
 import qualified Data.Map.Strict as Map
 import Data.Maybe (fromJust)
 import Network.URI (URI(..), parseURI)
-import Gargantext.Core (Lang(..), NLPServerConfig(..), PosTagAlgo(..))
+import Gargantext.Core (Lang(..), NLPServerConfig(..), PosTagAlgo(..), allLangs)
 import Gargantext.Prelude.NLP.Types (NLPConfig(..))
 import Gargantext.Utils.Tuple (uncurryMaybeSecond)
 import Protolude hiding (All)
@@ -53,6 +53,8 @@ nlpServerConfigFromURI _ = Nothing
 
 nlpServerMap :: NLPConfig -> NLPServerMap
 nlpServerMap (NLPConfig { .. }) =
-  Map.fromList $ catMaybes [ uncurryMaybeSecond (EN, nlpServerConfigFromURI _nlp_en)
-                           , uncurryMaybeSecond (FR, nlpServerConfigFromURI _nlp_fr)
-                           , uncurryMaybeSecond (All, nlpServerConfigFromURI _nlp_all) ]
+  Map.fromList $ catMaybes $
+    [ uncurryMaybeSecond (All, nlpServerConfigFromURI _nlp_all) ] ++
+    ((\lang ->
+        uncurryMaybeSecond (lang, Map.lookup (show lang) _nlp_languages >>= nlpServerConfigFromURI ))
+      <$> allLangs)

src/Gargantext/Core/Text/Learn.hs

@@ -39,11 +39,12 @@ import Gargantext.Core (Lang(..), allLangs)
 import Gargantext.Core.Text.Terms.Mono (words)
 import Gargantext.Core.Text.Metrics.Count (occurrencesWith)
 
-import qualified Gargantext.Core.Text.Samples.FR as FR
+import qualified Gargantext.Core.Text.Samples.CN as CN
+import qualified Gargantext.Core.Text.Samples.DE as DE
 import qualified Gargantext.Core.Text.Samples.EN as EN
---import qualified Gargantext.Core.Text.Samples.DE as DE
---import qualified Gargantext.Core.Text.Samples.SP as SP
---import qualified Gargantext.Core.Text.Samples.CH as CH
+import qualified Gargantext.Core.Text.Samples.ES as ES
+import qualified Gargantext.Core.Text.Samples.FR as FR
+import qualified Gargantext.Core.Text.Samples.PL as PL
 
 ------------------------------------------------------------------------
 data Candidate = Candidate { stop :: Double
@@ -112,6 +113,10 @@ detectLangDefault = detectCat 99 eventLang
     textSample :: Lang -> String
     textSample EN = EN.textSample
     textSample FR = FR.textSample
+    textSample DE = DE.textSample
+    textSample ES = ES.textSample
+    textSample CN = CN.textSample
+    textSample PL = PL.textSample
     textSample _  = panic "[G.C.T.L:detectLangDefault] not impl yet"
     --textSample DE = DE.textSample
     --textSample SP = SP.textSample

src/Gargantext/Core/Text/Samples/CH.hs → src/Gargantext/Core/Text/Samples/CN.hs

 {-|
-Module      : Gargantext.Core.Text.Samples.CH
+Module      : Gargantext.Core.Text.Samples.CN
 Description : Sample of Chinese Text
 Copyright   : (c) CNRS, 2017 - present
 License     : AGPL + CECILL v3
@@ -14,10 +14,9 @@ Page  : text mining
 
 
 
-module Gargantext.Core.Text.Samples.CH where
+module Gargantext.Core.Text.Samples.CN where
 
 import Data.String (String)
 
 textSample :: String
 textSample = "文本挖掘有时也被称为文字探勘、文本数据挖掘等，大致相当于文字分析，一般指文本处理过程中产生高质量的信息。高质量的信息通常通过分类和预测来产生，如模式识别。文本挖掘通常涉及输入文本的处理过程（通常进行分析，同时加上一些衍生语言特征以及消除杂音，随后插入到数据库中） ，产生结构化数据，并最终评价和解释输出。'高品质'的文本挖掘通常是指某种组合的相关性，新颖性和趣味性。典型的文本挖掘方法包括文本分类，文本聚类，概念/实体挖掘，生产精确分类，观点分析，文档摘要和实体关系模型（即，学习已命名实体之间的关系） 。 文本分析包括了信息检索、词典分析来研究词语的频数分布、模式识别、标签 注释、信息抽取，数据挖掘技术包括链接和关联分析、可视化和预测分析。本质上，首要的任务是，通过自然语言处理和分析方法，将文本转化为数据进行分析"
-

src/Gargantext/Core/Text/Samples/SP.hs → src/Gargantext/Core/Text/Samples/ES.hs

 {-|
-Module      : Gargantext.Core.Text.Samples.SP
+Module      : Gargantext.Core.Text.Samples.ES
 Description : Sample of Spanish Text
 Copyright   : (c) CNRS, 2017 - present
 License     : AGPL + CECILL v3
@@ -13,9 +13,9 @@ Page  : text mining
 
 
 
-module Gargantext.Core.Text.Samples.SP where
+module Gargantext.Core.Text.Samples.ES where
 
 import Data.String (String)
 
-textMining :: String
-textMining = "La minería de textos se refiere al proceso de derivar información nueva de textos. A comienzos de los años ochenta surgieron los primeros esfuerzos de minería de textos que necesitaban una gran cantidad de esfuerzo humano, pero los avances tecnológicos han permitido que esta área progrese de manera rápida en la última década. La minería de textos es un área multidisciplinar basada en la recuperación de información, minería de datos, aprendizaje automático, estadísticas y la lingüística computacional. Como la mayor parte de la información (más de un 80%) se encuentra actualmente almacenada como texto, se cree que la minería de textos tiene un gran valor comercial."
+textSample :: String
+textSample = "La minería de textos se refiere al proceso de derivar información nueva de textos. A comienzos de los años ochenta surgieron los primeros esfuerzos de minería de textos que necesitaban una gran cantidad de esfuerzo humano, pero los avances tecnológicos han permitido que esta área progrese de manera rápida en la última década. La minería de textos es un área multidisciplinar basada en la recuperación de información, minería de datos, aprendizaje automático, estadísticas y la lingüística computacional. Como la mayor parte de la información (más de un 80%) se encuentra actualmente almacenada como texto, se cree que la minería de textos tiene un gran valor comercial."

src/Gargantext/Core/Text/Samples/PL.hs

+{-|
+Module      : Gargantext.Core.Text.Samples.PL
+Description : Sample of Polish Text
+Copyright   : (c) CNRS, 2017 - present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+Source: Wikipedia
+Page  : text mining
+
+-}
+
+
+
+module Gargantext.Core.Text.Samples.PL where
+
+import Data.String (String)
+
+textSample :: String
+textSample = "Text mining (eksploracja tekstu) – ogólna nazwa metod eksploracji danych służących do wydobywania danych z tekstu i ich późniejszej obróbki. Metody text mining stosowane są np. do statystycznego przetwarzania: artykułów prasowych, wiadomości poczty elektronicznej, otwartych odpowiedzi na pytania ankietowe, opisów dolegliwości, podawanych przez pacjentów, komentarzy do sesji giełdowych i zdarzeń dotyczące spółek, życiorysów zawodowych i listów motywacyjnych, tekstów reklamacji konsumenckich. Text mining może polegać na znalezieniu kluczowych fraz, zdań, które zostają następnie zakodowane pod postacią zmiennych numerycznych. Później stosuje się metody statystyki i eksploracji danych w celu odkrycia zależności pomiędzy zmiennymi. Ze względu na to, że powstające zmienne są zwykle nominalne, szczególnie użyteczna jest analiza koszykowa."

src/Gargantext/Core/Text/Terms/Multi/PosTagging.hs

@@ -27,6 +27,8 @@ module Gargantext.Core.Text.Terms.Multi.PosTagging
 
 import Data.Aeson
 import Data.ByteString.Lazy.Internal (ByteString)
+import qualified Data.ByteString.Lazy.Char8 as BSL
+import qualified Data.Map as Map
 import Data.Set (fromList)
 import Data.Text (Text, splitOn, pack, toLower)
 import Gargantext.Core (Lang(..))
@@ -79,14 +81,43 @@ corenlp' :: ( FromJSON a
             )
           => URI -> Lang -> p -> IO (Response a)
 corenlp' uri lang txt = do
-    let properties = case lang of
-            EN -> "{\"annotators\": \"tokenize,ssplit,pos,ner\", \"outputFormat\": \"json\"}"
-            FR -> "{\"annotators\": \"tokenize,ssplit,pos,lemma,ner\", \"parse.model\":\"edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz\", \"pos.model\":\"edu/stanford/nlp/models/pos-tagger/french/french.tagger\", \"tokenize.language\":\"fr\", \"outputFormat\": \"json\"}"
-            _  -> panic $ pack "not implemented yet"
-    req <- parseRequest $ "POST " <> show (uri { uriQuery = "?properties=" <> properties })
+  req <- parseRequest $
+         "POST " <> show (uri { uriQuery = "?properties=" <> (BSL.unpack $ encode $ toJSON $ Map.fromList properties) })
    -- curl -XPOST 'http://localhost:9000/?properties=%7B%22annotators%22:%20%22tokenize,ssplit,pos,ner%22,%20%22outputFormat%22:%20%22json%22%7D' -d 'hello world, hello' | jq .
-    let request = setRequestBodyLBS (cs txt) req
-    httpJSON request
+  httpJSON $ setRequestBodyLBS (cs txt) req
+  where
+    properties_ :: [(Text, Text)]
+    properties_ = case lang of
+-- TODO: Add: Aeson.encode $ Aeson.toJSON $ Map.fromList [()] instead of these hardcoded JSON strings
+            EN -> [ ("annotators", "tokenize,ssplit,pos,ner" ) ]
+            FR -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/french.tagger")
+                  , ("tokenize.language", "fr") ]
+            DE -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/german-hgc.tagger")
+                  , ("tokenize.language", "de") ]
+            ES -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/spanish.tagger")
+                  , ("tokenize.language", "es") ]
+            IT -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  -- , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/french.tagger")
+                  , ("tokenize.language", "it") ]
+            PL -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  -- , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/french.tagger")
+                  , ("tokenize.language", "pl") ]
+            CN -> [ ("annotators", "tokenize,ssplit,pos,lemma,ner")
+                  -- , ("parse.model", "edu/stanford/nlp/models/lexparser/frenchFactored.ser.gz")
+                  , ("pos.model", "edu/stanford/nlp/models/pos-tagger/french/chinese-distsim.tagger")
+                  , ("tokenize.language", "zh") ]
+            l  -> panic $ pack $ "corenlp for language " <> show l <> " is not implemented yet"
+
+    properties = properties_ <> [ ("outputFormat", "json") ]
+
 
 
 corenlp :: URI -> Lang -> Text -> IO PosSentences

src/Gargantext/Utils/JohnSnowNLP.hs

@@ -38,17 +38,38 @@ data JSSpell = JSPOS Lang | JSLemma Lang
 instance ToJSON JSSpell where
   toJSON (JSPOS EN)    = "en.pos"
   toJSON (JSPOS FR)    = "fr.pos"
+  toJSON (JSPOS DE)    = "de.pos"
+  toJSON (JSPOS ES)    = "es.pos"
+  toJSON (JSPOS IT)    = "it.pos"
+  toJSON (JSPOS PL)    = "pl.pos"
+  toJSON (JSPOS CN)    = "cn.pos"
   toJSON (JSPOS All)   = "pos"
+
   toJSON (JSLemma EN)  = "en.lemma"
   toJSON (JSLemma FR)  = "fr.lemma"
+  toJSON (JSLemma DE)  = "de.lemma"
+  toJSON (JSLemma ES)  = "es.lemma"
+  toJSON (JSLemma IT)  = "it.lemma"
+  toJSON (JSLemma PL)  = "pl.lemma"
+  toJSON (JSLemma CN)  = "cn.lemma"
   toJSON (JSLemma All) = "lemma"
 
 instance FromJSON JSSpell where
   parseJSON (String "en.pos")   = pure $ JSPOS EN
   parseJSON (String "fr.pos")   = pure $ JSPOS FR
+  parseJSON (String "de.pos")   = pure $ JSPOS DE
+  parseJSON (String "es.pos")   = pure $ JSPOS ES
+  parseJSON (String "it.pos")   = pure $ JSPOS IT
+  parseJSON (String "pl.pos")   = pure $ JSPOS PL
+  parseJSON (String "cn.pos")   = pure $ JSPOS CN
   parseJSON (String "pos")      = pure $ JSPOS All
   parseJSON (String "en.lemma") = pure $ JSLemma EN
   parseJSON (String "fr.lemma") = pure $ JSLemma FR
+  parseJSON (String "de.lemma") = pure $ JSLemma DE
+  parseJSON (String "es.lemma") = pure $ JSLemma ES
+  parseJSON (String "it.lemma") = pure $ JSLemma IT
+  parseJSON (String "pl.lemma") = pure $ JSLemma PL
+  parseJSON (String "cn.lemma") = pure $ JSLemma CN
   parseJSON (String "lemma")    = pure $ JSLemma All
   parseJSON s =
     prependFailure "parsing spell failed, "

stack.yaml

@@ -40,7 +40,7 @@ extra-deps:
 #- git: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude.git
 #  commit: 791c2a7046a3760f8ae5fabdbd708f61caa63741
 - git: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude
-  commit: 175d4b295be2a0f56edc4eb6c7d8227d81bc2841
+  commit: 8f97fef4dfd941d773914ad058d8e02ce2bb1a3e  # e250af8a495191b4bd06c9f465656b87bea7bf7f
 - git: https://gitlab.iscpif.fr/gargantext/gargantext-graph.git
   commit: 588e104fe7593210956610cab0041fd16584a4ce
   # Data Mining Libs