[refactor] move some of API stuff to Core

Overall idea is: Core and Database contain base stuff and don't import
API. And API imports from Core and Database.

bin/gargantext-import/Main.hs

@@ -16,13 +16,13 @@ Import a corpus binary.
 module Main where
 
 import Data.Either
-import qualified Data.Text as Text
-
-import Gargantext.API.Dev (withDevEnv, runCmdGargDev)
+import Data.Text qualified as Text
 import Gargantext.API.Admin.EnvTypes (DevEnv(..), DevJobHandle(..))
+import Gargantext.API.Dev (withDevEnv, runCmdGargDev)
 import Gargantext.API.Errors.Types
 import Gargantext.API.Node () -- instances
 import Gargantext.Core (Lang(..))
+import Gargantext.Core.Text.Corpus.Parsers.Types (FileFormat(..), FileType(..))
 import Gargantext.Core.Text.Terms (TermType(..))
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Types.Query (Limit)
@@ -30,9 +30,9 @@ import Gargantext.Database.Action.Flow (flowCorpusFile, flowAnnuaire)
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Admin.Types.Node (CorpusId)
 import Gargantext.Prelude
-import Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), FileType(..))
 import Gargantext.Utils.Jobs (MonadJobStatus, JobHandle)
 
+
 main :: IO ()
 main = do
   [fun, user, name, iniPath, limit, corpusPath] <- getArgs

bin/gargantext-phylo/Main.hs

@@ -28,11 +28,12 @@ import Data.Text qualified as T
 import Data.Vector qualified as Vector
 import GHC.IO.Encoding
 import Gargantext.API.Ngrams.Prelude (toTermList)
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Text.Context (TermList)
-import Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), FileType(..), parseFile)
+import Gargantext.Core.Text.Corpus.Parsers (parseFile)
 import Gargantext.Core.Text.Corpus.Parsers.CSV (csv_title, csv_abstract, csv_publication_year, csv_publication_month, csv_publication_day, csv'_source, csv'_title, csv'_abstract, csv'_publication_year, csv'_publication_month, csv'_publication_day, csv'_weight)
 import Gargantext.Core.Text.Corpus.Parsers.CSV qualified as Csv
+import Gargantext.Core.Text.Corpus.Parsers.Types (FileFormat(..), FileType(..))
 import Gargantext.Core.Text.List.Formats.CSV (csvMapTermList)
 import Gargantext.Core.Text.Terms.WithList (Patterns, buildPatterns, extractTermsWithList)
 import Gargantext.Core.Types.Main (ListType(..))

gargantext.cabal

@@ -107,7 +107,6 @@ library
       Gargantext.API.Ngrams.List.Types
       Gargantext.API.Ngrams.Prelude
       Gargantext.API.Ngrams.Tools
-      Gargantext.API.Ngrams.Types
       Gargantext.API.Node
       Gargantext.API.Node.Corpus.New
       Gargantext.API.Node.Corpus.Types
@@ -120,6 +119,7 @@ library
       Gargantext.Core
       Gargantext.Core.Mail.Types
       Gargantext.Core.Methods.Similarities
+      Gargantext.Core.Ngrams.Types
       Gargantext.Core.NLP
       Gargantext.Core.NodeStory
       Gargantext.Core.NodeStory.DB
@@ -137,6 +137,7 @@ library
       Gargantext.Core.Text.Corpus.Parsers.CSV
       Gargantext.Core.Text.Corpus.Parsers.Date
       Gargantext.Core.Text.Corpus.Parsers.Date.Parsec
+      Gargantext.Core.Text.Corpus.Parsers.Types
       Gargantext.Core.Text.List.Formats.CSV
       Gargantext.Core.Text.Metrics
       Gargantext.Core.Text.Metrics.CharByChar

src/Gargantext/API/Admin/Orchestrator/Types.hs

+{-|
+Module      : Gargantext.API.Admin.Orchestrator.Types
+Description : Types for Orchestrator
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 {-# LANGUAGE TemplateHaskell    #-}
 {-# LANGUAGE TypeOperators      #-}

src/Gargantext/API/Metrics.hs

@@ -24,8 +24,8 @@ import Data.Time (UTCTime)
 import Data.Vector (Vector)
 import Gargantext.API.HashedResponse
 import Gargantext.API.Ngrams.NgramsTree
-import Gargantext.API.Ngrams.Types
 import Gargantext.API.Prelude (GargServer)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Metrics (Scored(..), {-normalizeGlobal,-} normalizeLocal)
 import Gargantext.Core.Types (CorpusId, ListId, ListType(..))

src/Gargantext/API/Ngrams.hs

@@ -55,8 +55,8 @@ import Gargantext.API.Admin.Types (HasSettings)
 import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Metrics qualified as Metrics
 import Gargantext.API.Ngrams.Tools (getNgramsTableMap, getNodeStory, setNgramsTableScores, tableNgramsPull)
-import Gargantext.API.Ngrams.Types
 import Gargantext.API.Prelude (GargM)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory, HasNodeArchiveStoryImmediateSaver(..), HasNodeStoryImmediateSaver(..), NgramsStatePatch', a_history, a_state, a_version, currentVersion)
 import Gargantext.Core.NodeStory.Utils (saveNodeStory)
 -- import Gargantext.Core.Types (Terms(..))

src/Gargantext/API/Ngrams/List.hs

@@ -34,9 +34,9 @@ import Gargantext.API.Errors.Types
 import Gargantext.API.Ngrams (setListNgrams)
 import Gargantext.API.Ngrams.List.Types
 import Gargantext.API.Ngrams.Prelude (getNgramsList)
-import Gargantext.API.Ngrams.Types
 import Gargantext.API.Prelude (GargServer, GargM, serverError, HasServerError)
 import Gargantext.API.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory
 import Gargantext.Core.Types.Main (ListType(..))
 import Gargantext.Database.Action.Flow (reIndexWith)

src/Gargantext/API/Ngrams/List/Types.hs

@@ -13,20 +13,17 @@ Portability : POSIX
 
 module Gargantext.API.Ngrams.List.Types where
 
---import Control.Lens hiding (elements, Indexed)
 import Data.Aeson
-import qualified Data.ByteString.Lazy as BSL
+import Data.ByteString.Lazy qualified as BSL
 import Data.Swagger (ToSchema, declareNamedSchema, genericDeclareNamedSchema)
 import Data.Text
-import qualified Data.Text.Encoding as E
-import Servant.Job.Utils (jsonOptions)
-import Web.FormUrlEncoded (FromForm(..), ToForm, parseUnique)
-
-import Protolude
-
-import Gargantext.API.Ngrams.Types (NgramsList)
+import Data.Text.Encoding qualified as E
 import Gargantext.API.Node.Corpus.New.Types (FileType(..))
+import Gargantext.Core.Ngrams.Types (NgramsList)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
+import Protolude
+import Servant.Job.Utils (jsonOptions)
+import Web.FormUrlEncoded (FromForm(..), ToForm, parseUnique)
 
 
 

src/Gargantext/API/Ngrams/NgramsTree.hs

@@ -21,7 +21,7 @@ import Data.List qualified as List
 import Data.Set qualified as Set
 import Data.Swagger
 import Data.Tree
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Types (ListType(..), NodeId)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Prelude

src/Gargantext/API/Ngrams/Prelude.hs

@@ -22,7 +22,7 @@ import Data.Map.Strict qualified as Map
 import Data.Text qualified as Text
 import Data.Validity
 import Gargantext.API.Ngrams.Tools (getNgramsTableMap)
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Context (TermList)
 import Gargantext.Core.Text.List.Social.Prelude

src/Gargantext/API/Ngrams/Tools.hs

@@ -42,7 +42,7 @@ import Data.Set qualified as Set
 import Data.Text qualified as T
 import Data.Text.Lazy.IO as DTL
 import Formatting (hprint, int, (%))
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (ArchiveList, HasNodeStory, NodeListStory, a_history, a_state, a_version, hasNodeStory, nse_getter, nse_getter_multi, unNodeStory)
 import Gargantext.Core.Types (ListType(..), NodeId, ListId)
 import Gargantext.Database.Action.Metrics.NgramsByContext (getOccByNgramsOnlyFast)

src/Gargantext/API/Node.hs

@@ -39,7 +39,6 @@ import Gargantext.API.Auth.PolicyCheck
 import Gargantext.API.Errors.Types
 import Gargantext.API.Metrics
 import Gargantext.API.Ngrams (TableNgramsApi, apiNgramsTableCorpus)
-import Gargantext.API.Ngrams.Types (TabType(..))
 import Gargantext.API.Node.DocumentUpload qualified as DocumentUpload
 import Gargantext.API.Node.File
 import Gargantext.API.Node.FrameCalcUpload qualified as FrameCalcUpload
@@ -49,6 +48,7 @@ import Gargantext.API.Node.Update qualified as Update
 import Gargantext.API.Prelude
 import Gargantext.API.Search qualified as Search
 import Gargantext.API.Table
+import Gargantext.Core.Ngrams.Types (TabType(..))
 import Gargantext.Core.Types (NodeTableResult)
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Types.Main (Tree, NodeTree)

src/Gargantext/API/Node/Corpus/Export.hs

@@ -22,10 +22,10 @@ import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Data.Text (pack)
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, mapTermListRoot, getRepo)
-import Gargantext.API.Ngrams.Types
 import Gargantext.API.Node.Corpus.Export.Types
 import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
 import Gargantext.API.Prelude (GargNoServer)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory
 import Gargantext.Core.Types
 import Gargantext.Database.Action.Metrics.NgramsByContext (getNgramsByContextOnlyUser)

src/Gargantext/API/Node/Corpus/New/Types.hs

+{-|
+Module      : Gargantext.API.Node.Corpus.New.Types
+Description : New corpus API Types
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+{-# LANGUAGE TemplateHaskell    #-}
+
+
 module Gargantext.API.Node.Corpus.New.Types where
 
-import Data.Aeson
-import Data.Swagger
+import Control.Lens (makeLenses)
+import Data.Aeson (FromJSON(..), ToJSON(..), genericParseJSON, genericToJSON)
+import Data.Swagger (ToParamSchema, ToSchema(..), genericDeclareNamedSchema)
 import Data.Text (pack)
+import Gargantext.API.Node.Corpus.Types (Database, Datafield)
+import Gargantext.Core (Lang(..))
+import Gargantext.Core.Text.Corpus.Query qualified as Corpus
+import Gargantext.Core.Text.Corpus.Parsers.Types qualified as Parsers
+import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
+import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Prelude
-import Servant
+import Servant (FromHttpApiData(..), ToHttpApiData(..))
+import Servant.Job.Utils (jsonOptions)
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
 
-data FileType = CSV
-              | CSV_HAL
-              | Istex
-              | PresseRIS
-              | WOS
-              | Iramuteq
-              | JSON
+
+-- | This is just a wrapper around G.C.T.C.Parsers -> FileType for API requests
+newtype FileType = FileType { unFileType :: Parsers.FileType }
   deriving (Eq, Show, Generic)
 instance ToSchema FileType
-instance Arbitrary FileType where arbitrary = elements [CSV, PresseRIS]
+instance Arbitrary FileType where arbitrary = elements [ FileType Parsers.CsvGargV3
+                                                       , FileType Parsers.RisPresse]
 instance ToParamSchema FileType
 instance FromJSON FileType
 instance ToJSON FileType
 
 instance FromHttpApiData FileType where
-  parseUrlPiece "CSV"       = pure CSV
-  parseUrlPiece "CSV_HAL"   = pure CSV_HAL
-  parseUrlPiece "Istex"     = pure Istex
-  parseUrlPiece "PresseRis" = pure PresseRIS
-  parseUrlPiece "WOS"       = pure WOS
-  parseUrlPiece "Iramuteq"  = pure Iramuteq
-  parseUrlPiece "JSON"      = pure JSON
+  parseUrlPiece "CSV"       = pure $ FileType Parsers.CsvGargV3
+  parseUrlPiece "CSV_HAL"   = pure $ FileType Parsers.CsvHal
+  parseUrlPiece "Iramuteq"  = pure $ FileType Parsers.Iramuteq
+  parseUrlPiece "Istex"     = pure $ FileType Parsers.Istex
+  parseUrlPiece "JSON"      = pure $ FileType Parsers.JSON
+  parseUrlPiece "PresseRis" = pure $ FileType Parsers.RisPresse
+  parseUrlPiece "WOS"       = pure $ FileType Parsers.WOS
   parseUrlPiece s           = panicTrace $ "[G.A.A.Node.Corpus.New] File Type not implemented (yet): " <> s
 instance ToHttpApiData FileType where
   toUrlPiece = pack . show
 
-data FileFormat = Plain | ZIP
+
+
+newtype FileFormat = FileFormat { unFileFormat :: Parsers.FileFormat }
   deriving (Eq, Show, Generic)
 instance ToSchema FileFormat
-instance Arbitrary FileFormat where arbitrary = elements [ Plain, ZIP ]
+instance Arbitrary FileFormat where arbitrary = elements [ FileFormat Parsers.Plain
+                                                         , FileFormat Parsers.ZIP ]
 instance ToParamSchema FileFormat
 instance FromJSON FileFormat
 instance ToJSON FileFormat
 instance FromHttpApiData FileFormat where
-  parseUrlPiece "Plain" = pure Plain
-  parseUrlPiece "ZIP"   = pure ZIP
-  parseUrlPiece _       = pure Plain -- TODO error here
+  parseUrlPiece "Plain" = pure $ FileFormat Parsers.Plain
+  parseUrlPiece "ZIP"   = pure $ FileFormat Parsers.ZIP
+  parseUrlPiece _       = pure $ FileFormat Parsers.Plain -- TODO error here
 instance ToHttpApiData FileFormat where
   toUrlPiece = pack . show
+
+
+------------------------------------------------------------------------
+data WithQuery = WithQuery
+  { _wq_query        :: !Corpus.RawQuery
+  , _wq_databases    :: !Database
+  , _wq_datafield    :: !(Maybe Datafield)
+  , _wq_lang         :: !Lang
+  , _wq_node_id      :: !Int
+  , _wq_flowListWith :: !FlowSocialListWith
+  , _wq_pubmedAPIKey :: !(Maybe Text)
+  , _wq_epoAPIUser   :: !(Maybe Text)
+  , _wq_epoAPIToken  :: !(Maybe Text)
+  }
+  deriving (Show, Eq, Generic)
+
+makeLenses ''WithQuery
+instance FromJSON WithQuery where
+  parseJSON = genericParseJSON $ jsonOptions "_wq_"
+instance ToJSON WithQuery where
+  toJSON = genericToJSON $ jsonOptions "_wq_"
+instance ToSchema WithQuery where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wq_")
+
+instance Arbitrary WithQuery where
+  arbitrary = WithQuery <$> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+
+------------------------------------------------------------------------
+ 

src/Gargantext/API/Node/DocumentsFromWriteNodes.hs

@@ -28,9 +28,9 @@ import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
 import Gargantext.API.Admin.Types (HasSettings)
 import Gargantext.API.Errors.Types
 import Gargantext.API.Ngrams (commitStatePatch)
-import Gargantext.API.Ngrams.Types (Versioned(..))
 import Gargantext.API.Prelude (GargM)
 import Gargantext.Core (Lang(..))
+import Gargantext.Core.Ngrams.Types (Versioned(..))
 import Gargantext.Core.NodeStory (HasNodeStoryImmediateSaver, HasNodeArchiveStoryImmediateSaver, currentVersion)
 import Gargantext.Core.Text.Corpus.Parsers.Date (split')
 import Gargantext.Core.Text.Corpus.Parsers.FrameWrite

src/Gargantext/API/Node/File.hs

@@ -8,7 +8,7 @@ Stability   : experimental
 Portability : POSIX
 -}
 
-{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-unused-matches #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
 
 {-# LANGUAGE AllowAmbiguousTypes     #-}
 {-# LANGUAGE TypeOperators     #-}
@@ -20,15 +20,11 @@ module Gargantext.API.Node.File where
 import Control.Lens ((^.))
 import Data.ByteString qualified as BS
 import Data.ByteString.Lazy qualified as BSL
+import Data.Either
 import Data.MIME.Types qualified as DMT
 import Data.Swagger
-import Data.Text qualified as T
 import Data.Text
-import Servant
-import qualified Gargantext.Database.GargDB as GargDB
-import qualified Network.HTTP.Media as M
-
-import Data.Either
+import Data.Text qualified as T
 import Gargantext.API.Admin.Auth.Types
 import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
 import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
@@ -41,11 +37,15 @@ import Gargantext.Database.Action.Flow.Types
 import Gargantext.Database.Action.Node (mkNodeWithParent)
 import Gargantext.Database.Admin.Types.Hyperdata.File
 import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.GargDB qualified as GargDB
 import Gargantext.Database.Query.Table.Node (getNodeWith)
 import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
 import Gargantext.Database.Schema.Node (node_hyperdata)
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
+import Network.HTTP.Media qualified as M
+import Servant
+
 
 data RESPONSE deriving Typeable
 

src/Gargantext/API/Node/FrameCalcUpload.hs

@@ -16,36 +16,37 @@ Portability : POSIX
 module Gargantext.API.Node.FrameCalcUpload where
 
 import Control.Lens ((^.))
-import Data.Aeson
+import Data.Aeson (FromJSON, ToJSON)
 import Data.ByteString.Lazy qualified as BSL
 import Data.ByteString.UTF8 qualified as BSU8
-import Data.Swagger
+import Data.Swagger (ToSchema)
 import Data.Text qualified as T
-import Network.HTTP.Client (newManager, httpLbs, parseRequest, responseBody)
-import Network.HTTP.Client.TLS (tlsManagerSettings)
-import Servant
-import Web.FormUrlEncoded (FromForm)
-
-import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser, auth_node_id)
 import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
 import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
-import Gargantext.API.Errors.Types
+import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Node.Corpus.New (addToCorpusWithForm)
 import Gargantext.API.Node.Corpus.New.Types (FileFormat(..), FileType(..))
 import Gargantext.API.Node.Types (NewWithForm(..))
-import Gargantext.API.Prelude
+import Gargantext.API.Prelude (GargM)
 import Gargantext.Core (Lang)
 import Gargantext.Core.NodeStory (HasNodeArchiveStoryImmediateSaver)
 import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
+import Gargantext.Core.Text.Corpus.Parsers.Types (FileFormat(..), FileType(..))
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Database.Action.Flow.Types
-import Gargantext.Database.Admin.Types.Hyperdata.Frame
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Action.Flow.Types (FlowCmdM)
+import Gargantext.Database.Admin.Types.Hyperdata.Frame (HyperdataFrame(..))
+import Gargantext.Database.Admin.Types.Node (NodeId, NodeType(NodeCorpus))
 import Gargantext.Database.Prelude (HasConfig)
 import Gargantext.Database.Query.Table.Node (getClosestParentIdByType, getNodeWith)
 import Gargantext.Database.Schema.Node (node_hyperdata)
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
+import Network.HTTP.Client (newManager, httpLbs, parseRequest, responseBody)
+import Network.HTTP.Client.TLS (tlsManagerSettings)
+import Servant ((:>)(), JSON, ServerT, Summary)
+import Web.FormUrlEncoded (FromForm)
+
 
 data FrameCalcUpload = FrameCalcUpload {
   _wf_lang      :: !(Maybe Lang)
@@ -110,8 +111,8 @@ frameCalcUploadAsync authenticatedUser nId (FrameCalcUpload _wf_lang _wf_selecti
       -- FIXME(adn) Audit this conversion.
       addToCorpusWithForm (RootId userNodeId)
                           cId
-                          (NewWithForm { _wf_filetype = CSV
-                                       , _wf_fileformat = Plain
+                          (NewWithForm { _wf_filetype = FileType CsvGargV3
+                                       , _wf_fileformat = FileFormat Plain
                                        , _wf_data = body
                                        , _wf_lang
                                        , _wf_name = "calc-upload.csv"

src/Gargantext/API/Node/Update.hs

@@ -25,9 +25,9 @@ import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
 import Gargantext.API.Admin.Types (HasSettings)
 import Gargantext.API.Errors.Types
 import Gargantext.API.Metrics qualified as Metrics
-import Gargantext.API.Ngrams.Types qualified as NgramsTypes
 import Gargantext.API.Prelude (GargM, simuLogs)
 import Gargantext.Core.Methods.Similarities (GraphMetric(..))
+import Gargantext.Core.Ngrams.Types qualified as NgramsTypes
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Types.Main (ListType(..))
 import Gargantext.Core.Viz.Graph.API (recomputeGraph)

src/Gargantext/API/Table.hs

@@ -36,8 +36,8 @@ import Data.Maybe
 import Data.Swagger
 import Data.Text qualified as T
 import Gargantext.API.HashedResponse
-import Gargantext.API.Ngrams.Types (TabType(..))
 import Gargantext.API.Prelude (GargServer)
+import Gargantext.Core.Ngrams.Types (TabType(..))
 import Gargantext.Core.Text.Corpus.Query (RawQuery, parseQuery, getRawQuery)
 import Gargantext.Core.Types (TableResult(..))
 import Gargantext.Core.Types.Query (Offset, Limit)

src/Gargantext/API/Ngrams/Types.hs → src/Gargantext/Core/Ngrams/Types.hs

 {-|
-Module      : Gargantext.API.Ngrams.Types
+Module      : Gargantext.Core.Ngrams.Types
 Description : Ngrams List Types
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -15,7 +15,7 @@ Portability : POSIX
 {-# LANGUAGE TypeFamilies      #-}
 {-# OPTIONS -fno-warn-orphans #-}
 
-module Gargantext.API.Ngrams.Types
+module Gargantext.Core.Ngrams.Types
   ( AddRem
   , addPatch
   
@@ -841,7 +841,7 @@ instance ToHttpApiData (Map TableNgrams.NgramsType (Versioned NgramsTableMap)) w
 
 ngramsTypeFromTabType :: TabType -> TableNgrams.NgramsType
 ngramsTypeFromTabType tabType =
-  let here = "Garg.API.Ngrams: " :: Text in
+  let here = "Garg.Core.Ngrams: " :: Text in
     case tabType of
       Sources    -> TableNgrams.Sources
       Authors    -> TableNgrams.Authors

src/Gargantext/Core/NodeStory.hs

@@ -70,7 +70,7 @@ import Data.Set qualified as Set
 import Database.PostgreSQL.Simple qualified as PGS
 import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.ToField qualified as PGS
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory.DB
 import Gargantext.Core.NodeStory.Types
 import Gargantext.Core.Types (ListId, NodeId(..))

src/Gargantext/Core/NodeStory/DB.hs

@@ -36,8 +36,8 @@ import Data.Monoid
 import Database.PostgreSQL.Simple qualified as PGS
 import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
-import Gargantext.API.Ngrams.Types
 import Gargantext.Core (toDBid)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory.Types
 import Gargantext.Core.Types (NodeId(..), NodeType)
 import Gargantext.Database.Admin.Config ()

src/Gargantext/Core/NodeStory/Types.hs

@@ -58,7 +58,7 @@ import Data.Profunctor.Product.TH (makeAdaptorAndInstance)
 import Data.Semigroup
 import Data.Set qualified as Set
 import Database.PostgreSQL.Simple.FromField (FromField(fromField), fromJSONField)
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Types (NodeId(..))
 import Gargantext.Core.Utils.Prefix (unPrefix)
 import Gargantext.Database.Admin.Config ()

src/Gargantext/Core/Text/Convert.hs

@@ -16,8 +16,9 @@ Format Converter.
 module Gargantext.Core.Text.Convert (risPress2csvWrite)
     where
 
-import Gargantext.Core.Text.Corpus.Parsers (parseFile, FileFormat(..), FileType(..))
+import Gargantext.Core.Text.Corpus.Parsers (parseFile)
 import Gargantext.Core.Text.Corpus.Parsers.CSV (writeDocs2Csv)
+import Gargantext.Core.Text.Corpus.Parsers.Types (FileFormat(..), FileType(..))
 import Gargantext.Prelude
 
 

src/Gargantext/Core/Text/Corpus/API/OpenAlex.hs

@@ -7,17 +7,18 @@ Maintainer  : team@gargantext.org
 Stability   : experimental
 Portability : POSIX
 -}
+
 module Gargantext.Core.Text.Corpus.API.OpenAlex where
 
 import Conduit
 import Data.LanguageCodes qualified as ISO639
-import qualified Data.Text as T
+import Data.Text qualified as T
 import Gargantext.Core (iso639ToText)
 import Gargantext.Core.Text.Corpus.Query as Corpus
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import OpenAlex qualified as OA
+import OpenAlex.Types qualified as OA
 import Protolude
-import qualified OpenAlex as OA
-import qualified OpenAlex.Types as OA
 import Servant.Client (ClientError)
 
 

src/Gargantext/Core/Text/Corpus/Parsers.hs

@@ -20,8 +20,15 @@ please follow the types.
 
 {-# LANGUAGE PackageImports    #-}
 
-module Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), FileType(..), clean, parseFile, cleanText, parseFormatC, splitOn, etale)
-    where
+module Gargantext.Core.Text.Corpus.Parsers (
+    clean
+  , parseFile
+  , cleanText
+  , parseFormatC
+  , splitOn
+  , etale
+  )
+where
 
 import "zip" Codec.Archive.Zip (EntrySelector, withArchive, getEntry, getEntries, unEntrySelector)
 import Conduit
@@ -35,7 +42,6 @@ import Data.List (lookup)
 import Data.Map qualified as DM
 import Data.Text qualified as DT
 import Data.Tuple.Extra (both) -- , first, second)
-import Gargantext.API.Node.Corpus.New.Types (FileFormat(..))
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.Corpus.Parsers.CSV (parseHal, parseCsv, parseCsvC)
 import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
@@ -45,6 +51,7 @@ import Gargantext.Core.Text.Corpus.Parsers.JSON (parseJSONC, parseIstex)
 import Gargantext.Core.Text.Corpus.Parsers.RIS qualified as RIS
 import Gargantext.Core.Text.Corpus.Parsers.RIS.Presse (presseEnrich)
 import Gargantext.Core.Text.Corpus.Parsers.WOS qualified as WOS
+import Gargantext.Core.Text.Corpus.Parsers.Types
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
 import Gargantext.Database.Query.Table.Ngrams (NgramsType(..))
 import Gargantext.Prelude hiding (show, undefined)
@@ -62,18 +69,6 @@ type ParseError = Text
 --                              } deriving (Show)
 
 
--- | According to the format of Input file,
--- different parser are available.
-data FileType = WOS
-              | RIS
-              | RisPresse
-              | CsvGargV3
-              | CsvHal
-              | Iramuteq
-              | JSON
-              | Istex
-  deriving (Show, Eq)
-
 -- Implemented (ISI Format)
 --                | DOC        -- Not Implemented / import Pandoc
 --                | ODT        -- Not Implemented / import Pandoc

src/Gargantext/Core/Text/Corpus/Parsers/Types.hs

+{-|
+Module      : Gargantext.Core.Text.Corpus.Parsers.Types
+Description : Parsers types
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+module Gargantext.Core.Text.Corpus.Parsers.Types where
+
+import Data.Aeson
+import Data.Swagger
+import Gargantext.Prelude
+import Test.QuickCheck (elements)
+import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
+
+
+
+-- | According to the format of Input file,
+-- different parser are available.
+data FileType = CsvGargV3
+              | CsvHal
+              | Iramuteq
+              | Istex
+              | JSON
+              | RIS
+              | RisPresse
+              | WOS
+  deriving (Show, Eq, Generic)
+instance ToSchema FileType
+instance Arbitrary FileType where arbitrary = elements [ CsvGargV3
+                                                       , RisPresse]
+instance ToParamSchema FileType
+instance FromJSON FileType
+instance ToJSON FileType
+
+
+data FileFormat =
+    Plain
+  | ZIP
+  deriving (Show, Eq, Generic)
+instance ToSchema FileFormat
+instance Arbitrary FileFormat where arbitrary = elements [ Plain
+                                                         , ZIP ]
+instance ToParamSchema FileFormat
+instance FromJSON FileFormat
+instance ToJSON FileFormat
+
+

src/Gargantext/Core/Text/List.hs

@@ -25,7 +25,7 @@ import Data.List qualified as List
 import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Data.Tuple.Extra (both)
-import Gargantext.API.Ngrams.Types (NgramsElement, NgramsTerm(..))
+import Gargantext.Core.Ngrams.Types (NgramsElement, NgramsTerm(..))
 import Gargantext.Core.NLP (HasNLPServer)
 import Gargantext.Core.NodeStory
 import Gargantext.Core.Text (size)

src/Gargantext/Core/Text/List/Group.hs

@@ -21,7 +21,7 @@ module Gargantext.Core.Text.List.Group
 import Control.Lens (view)
 import Data.HashMap.Strict (HashMap)
 import Data.HashMap.Strict qualified as HashMap
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Text.List.Group.Prelude
 import Gargantext.Core.Text.List.Group.WithScores
 import Gargantext.Core.Text.List.Social.Prelude

src/Gargantext/Core/Text/List/Group/Prelude.hs

@@ -23,7 +23,7 @@ import Data.List qualified as List
 import Data.Monoid
 import Data.Semigroup
 import Data.Set qualified as Set
-import Gargantext.API.Ngrams.Types (NgramsElement, mkNgramsElement, NgramsTerm(..), RootParent(..), mSetFromList)
+import Gargantext.Core.Ngrams.Types (NgramsElement, mkNgramsElement, NgramsTerm(..), RootParent(..), mSetFromList)
 import Gargantext.Core.Types (ListType(..))
 import Gargantext.Prelude
 import Prelude (foldl1)

src/Gargantext/Core/Text/List/Group/WithScores.hs

@@ -19,7 +19,7 @@ import Control.Lens (view, set, over)
 import Data.HashMap.Strict (HashMap)
 import Data.HashMap.Strict qualified as HashMap
 import Data.Semigroup
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Text.List.Group.Prelude
 import Gargantext.Core.Text.List.Social.Prelude
 import Gargantext.Prelude

src/Gargantext/Core/Text/List/Group/WithStem.hs

@@ -27,8 +27,8 @@ import Data.Map.Strict qualified as Map
 import Data.Map.Strict.Patch qualified as PatchMap
 import Data.Patch.Class qualified as Patch (Replace(..))
 import Data.Text qualified as Text
-import Gargantext.API.Ngrams.Types
 import Gargantext.Core (Lang(..), Form, Lem, NLPServerConfig)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Text.List.Group.Prelude
 import Gargantext.Core.Text.List.Social.Patch
 import Gargantext.Core.Text.List.Social.Prelude

src/Gargantext/Core/Text/List/Merge.hs

@@ -20,7 +20,7 @@ module Gargantext.Core.Text.List.Merge
 
 import Control.Lens (view)
 import Data.Map.Strict.Patch hiding (PatchMap)
-import Gargantext.API.Ngrams.Types (NgramsRepoElement, NgramsTerm, PatchMap, Versioned(..), v_data, v_version)
+import Gargantext.Core.Ngrams.Types (NgramsRepoElement, NgramsTerm, PatchMap, Versioned(..), v_data, v_version)
 import Gargantext.Prelude hiding (diff)
 
 type List = Map NgramsTerm NgramsRepoElement

src/Gargantext/Core/Text/List/Social.hs

@@ -23,7 +23,7 @@ import Data.Swagger
 import Data.Text qualified as T
 import Data.Vector qualified as V
 import GHC.Generics
-import Gargantext.API.Ngrams.Types (NgramsTerm, NgramsPatch)
+import Gargantext.Core.Ngrams.Types (NgramsTerm, NgramsPatch)
 import Gargantext.Core.NodeStory (getNodesArchiveHistory)
 import Gargantext.Core.Text.List.Social.Find (findListsId)
 import Gargantext.Core.Text.List.Social.Patch (addScorePatches)

src/Gargantext/Core/Text/List/Social/Patch.hs

@@ -18,7 +18,7 @@ import Data.List qualified as List
 import Data.Map.Strict qualified as Map
 import Data.Patch.Class qualified as Patch (Replace(..))
 import Gargantext.API.Ngrams.Prelude (unMSet, patchMSet_toList)
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Text.List.Social.Prelude
 import Gargantext.Core.Types (ListId)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))

src/Gargantext/Core/Text/List/Social/Prelude.hs

@@ -24,7 +24,7 @@ import Data.HashMap.Strict qualified as HashMap
 import Data.Map.Strict qualified as Map
 import Data.Map.Strict.Patch qualified as PatchMap
 import Data.Monoid
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Types.Main
 import Gargantext.Data.HashMap.Strict.Utils qualified as HashMap
 import Gargantext.Prelude

src/Gargantext/Core/Text/Terms/WithList.hs

@@ -22,8 +22,8 @@ import Data.List qualified as List
 import Data.Ord
 import Data.Text (concat)
 import Data.Text qualified as Text
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core (Lang(ZH))
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Text.Context
 import Gargantext.Core.Text.Terms.Mono (monoTextsBySentence)
 import Gargantext.Core.Types (TermsCount)

src/Gargantext/Core/Viz/Chart.hs

@@ -21,7 +21,7 @@ import Data.Set qualified as Set
 import Data.Vector qualified as V
 import Gargantext.API.Ngrams.NgramsTree
 import Gargantext.API.Ngrams.Tools
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Metrics.Count (occurrencesWith)
 import Gargantext.Core.Types

src/Gargantext/Core/Viz/Graph.hs

@@ -22,7 +22,7 @@ import Data.Aeson qualified as DA
 import Data.ByteString.Lazy as DBL (readFile, writeFile)
 import Data.HashMap.Strict (HashMap, lookup)
 import Data.Text qualified as Text
-import Gargantext.API.Ngrams.Types (NgramsTerm(..), NgramsRepoElement(..), mSetToList)
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..), NgramsRepoElement(..), mSetToList)
 import Gargantext.Core.Viz.Graph.Types (Attributes(..), AttributesV3(..), Edge(..), EdgeV3(..), Graph(..), GraphV3(..), Node(..), NodeV3(..), graph_nodes)
 import Gargantext.Database.Admin.Types.Hyperdata.Prelude (set)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))

src/Gargantext/Core/Viz/Graph/API.hs

@@ -26,9 +26,9 @@ import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
 import Gargantext.API.Admin.Orchestrator.Types
 import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, getCoocByNgrams'', getRepo, groupNodesByNgrams, mapTermListRoot)
-import Gargantext.API.Ngrams.Types (Diagonal(..))
 import Gargantext.API.Prelude (GargM, GargServer)
 import Gargantext.Core.Methods.Similarities (Similarity(..), GraphMetric(..), withMetric)
+import Gargantext.Core.Ngrams.Types (Diagonal(..))
 import Gargantext.Core.NodeStory (HasNodeStory, NodeListStory, a_version, unNodeStory)
 import Gargantext.Core.Types.Main (ListType(MapTerm))
 import Gargantext.Core.Viz.Graph.GEXF ()  -- MimeRender instances

src/Gargantext/Core/Viz/Graph/Tools.hs

@@ -28,8 +28,8 @@ import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Data.Text qualified as Text
 import Data.Vector.Storable qualified as Vec
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Methods.Similarities (Similarity(..), measure)
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Statistics (Dimension(..), pcaReduceTo)
 import Gargantext.Core.Viz.Graph.Bridgeness (bridgeness, Bridgeness(..), Partitions, nodeId2comId, {-recursiveClustering,-} recursiveClustering', setNodes2clusterNodes)
 import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex, map2mat, mat2map, Index, MatrixShape(..))

src/Gargantext/Core/Viz/Graph/Types.hs

@@ -22,8 +22,8 @@ import Data.HashSet qualified as HashSet
 import Data.Swagger (ToSchema(..), genericDeclareNamedSchema)
 import Data.Text (pack)
 import Database.PostgreSQL.Simple.FromField (FromField(..))
-import Gargantext.API.Ngrams.Types (NgramsTerm)
 import Gargantext.Core.Methods.Similarities (GraphMetric)
+import Gargantext.Core.Ngrams.Types (NgramsTerm)
 import Gargantext.Core.Types (ListId)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.Admin.Types.Hyperdata.Prelude (Hyperdata)

src/Gargantext/Core/Viz/Phylo/API/Tools.hs

@@ -28,8 +28,8 @@ import Data.Text (pack)
 import Data.Time.Calendar (fromGregorian, diffGregorianDurationClip, cdMonths, diffDays, showGregorian)
 import Data.Time.Clock.POSIX(posixSecondsToUTCTime)
 import Gargantext.API.Ngrams.Prelude (getTermList)
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core (withDefaultLanguage, Lang)
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Terms.WithList (Patterns, buildPatterns, termsInText)
 import Gargantext.Core.Types (Context, nodeId2ContextId)

src/Gargantext/Core/Viz/Phylo/Legacy/LegacyMain.hs

@@ -25,8 +25,8 @@ import Data.Proxy
 import Data.Set qualified as Set
 import Data.Text qualified as Text
 import Gargantext.API.Ngrams.Tools (getTermsWith)
-import Gargantext.API.Ngrams.Types
 import Gargantext.Core (HasDBid, withDefaultLanguage)
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Context (TermList)
 import Gargantext.Core.Text.Terms.WithList

src/Gargantext/Database/Action/Flow.hs

@@ -57,7 +57,8 @@ import Gargantext.Core.Ext.IMTUser (readFile_Annuaire)
 import Gargantext.Core.NLP (HasNLPServer, nlpServerGet)
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Corpus.API qualified as API
-import Gargantext.Core.Text.Corpus.Parsers (parseFile, FileFormat, FileType)
+import Gargantext.Core.Text.Corpus.Parsers (parseFile)
+import Gargantext.Core.Text.Corpus.Parsers.Types (FileFormat, FileType)
 import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
 import Gargantext.Core.Text.Terms (ExtractedNgrams(SimpleNgrams), TermType(..), tt_lang)
 import Gargantext.Core.Text.Terms.Mono.Stem.En (stemIt)
@@ -101,11 +102,11 @@ getDataText :: (HasNodeError err)
             -> Maybe EPO.AuthKey
             -> Maybe API.Limit
             -> DBCmd err (Either API.GetCorpusError DataText)
-getDataText (ExternalOrigin api) la q mPubmedAPIKey mAuthKey li = do
+getDataText (ExternalOrigin api) lang q mPubmedAPIKey mAuthKey limit = do
   cfg <- view hasConfig
-  eRes <- liftBase $ API.get api (_tt_lang la) q mPubmedAPIKey mAuthKey (_gc_epo_api_url cfg) li
+  eRes <- liftBase $ API.get api (_tt_lang lang) q mPubmedAPIKey mAuthKey (_gc_epo_api_url cfg) limit
   pure $ DataNew <$> eRes
-getDataText (InternalOrigin _) _la q _ _ _li = do
+getDataText (InternalOrigin _) _lang q _ _ _limit = do
   (_masterUserId, _masterRootId, cId) <- getOrMk_RootWithCorpus
                                            (UserName userMaster)
                                            (Left "")

src/Gargantext/Database/Action/Flow/List.hs

@@ -26,7 +26,7 @@ import Data.Map.Strict (toList)
 import Data.Map.Strict qualified as Map
 import Data.Map.Strict.Patch qualified as PM
 import Gargantext.API.Ngrams.Tools (getNodeStory)
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.NodeStory (HasNodeStory, a_history, a_state, a_version)
 import Gargantext.Core.NodeStory.Utils (saveNodeStory)
 import Gargantext.Core.Types (HasValidationError(..), assertValid)

src/Gargantext/Database/Action/Flow/Pairing.hs

@@ -27,8 +27,8 @@ import Data.List qualified as List
 import Data.Set qualified as Set
 import Data.Text qualified as Text
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, getRepo, groupNodesByNgrams, mapTermListRoot)
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core (toDBid)
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Metrics.CharByChar (levenshtein)
 import Gargantext.Core.Types (TableResult(..))

src/Gargantext/Database/Action/Flow/Utils.hs

@@ -34,9 +34,9 @@ import Data.HashMap.Strict qualified as HashMap
 import Data.List qualified as List
 import Data.Map.Strict qualified as DM
 import Data.Text qualified as T
-import Gargantext.API.Ngrams.Types qualified as NT
 import Gargantext.Core (Lang, NLPServerConfig, toDBid)
 import Gargantext.Core.Flow.Types (UniqId, uniqId)
+import Gargantext.Core.Ngrams.Types qualified as NT
 import Gargantext.Core.NLP (HasNLPServer, nlpServerGet)
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.List (buildNgramsLists)

src/Gargantext/Database/Action/Metrics.hs

@@ -28,9 +28,8 @@ import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.ToField (toField, Action{-, ToField-})
 import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, groupNodesByNgrams, getCoocByNgrams, mapTermListRoot, getRepo)
-import Gargantext.API.Ngrams.Types (Diagonal(..), RootTerm)
-import Gargantext.API.Ngrams.Types (TabType(..), ngramsTypeFromTabType, NgramsTerm(..))
 import Gargantext.Core (HasDBid(toDBid))
+import Gargantext.Core.Ngrams.Types (Diagonal(..), RootTerm, TabType(..), ngramsTypeFromTabType, NgramsTerm(..))
 import Gargantext.Core.NodeStory
 import Gargantext.Core.Text.Metrics (scored, Scored(..), {-localMetrics, toScored-})
 import Gargantext.Core.Types (ListType(..), NodeType(..), ContextId, contextId2NodeId)

src/Gargantext/Database/Action/Metrics/NgramsByContext.hs

@@ -26,8 +26,8 @@ import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.ToField qualified as DPS
 import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
 import Database.PostgreSQL.Simple.Types qualified as DPST
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core
+import Gargantext.Core.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Data.HashMap.Strict.Utils as HM
 import Gargantext.Database.Admin.Types.Node (ListId, CorpusId, NodeId(..), ContextId (..), MasterCorpusId, NodeType(NodeDocument), UserCorpusId, DocId)
 import Gargantext.Database.Prelude (DBCmd, runPGSQuery)

src/Gargantext/Database/Action/Metrics/TFICF.hs

@@ -17,8 +17,8 @@ module Gargantext.Database.Action.Metrics.TFICF
 import Data.HashMap.Strict (HashMap)
 import Data.HashMap.Strict qualified as HM
 import Data.Set qualified as Set
-import Gargantext.API.Ngrams.Types
 import Gargantext.Core
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Text.Metrics.TFICF
 import Gargantext.Database.Action.Metrics.NgramsByContext (getContextsByNgramsUser, {-getOccByNgramsOnlyFast,-} getOccByNgramsOnlyFast_withSample)
 import Gargantext.Database.Admin.Types.Node -- (ListId, CorpusId, NodeId)

src/Gargantext/Database/Action/Search.hs

@@ -38,18 +38,18 @@ import Gargantext.Core.Types
 import Gargantext.Core.Types.Query (IsTrash, Limit, Offset)
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..), HyperdataContact(..))
 import Gargantext.Database.Prelude (runOpaQuery, runCountOpaQuery, DBCmd)
-import Gargantext.Database.Query.Facet
-import Gargantext.Database.Query.Filter
-import Gargantext.Database.Query.Table.Context
+import Gargantext.Database.Query.Facet (Facet(..), FacetDoc, FacetDocRead, FacetPaired(..), FacetPairedRead, OrderBy, _fp_score, filterWith)
+import Gargantext.Database.Query.Filter (limit', offset')
+import Gargantext.Database.Query.Table.Context (queryContextSearchTable)
 import Gargantext.Database.Query.Table.ContextNodeNgrams (queryContextNodeNgramsTable)
-import Gargantext.Database.Query.Table.Node
+import Gargantext.Database.Query.Table.Node (defaultList, queryNodeSearchTable)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError())
-import Gargantext.Database.Query.Table.NodeContext
-import Gargantext.Database.Query.Table.NodeContext_NodeContext
-import Gargantext.Database.Schema.Context
+import Gargantext.Database.Query.Table.NodeContext (NodeContextRead, nc_category, nc_context_id, nc_id, nc_node_id, nc_score, queryNodeContextTable)
+import Gargantext.Database.Query.Table.NodeContext_NodeContext (NodeContext_NodeContextRead, ncnc_nodecontext1, ncnc_nodecontext2, queryNodeContext_NodeContextTable)
+import Gargantext.Database.Schema.Context (ContextRead, ContextSearchRead, context_date, context_hyperdata, context_id, context_typename, cs_date, cs_hyperdata, cs_id, cs_name, cs_search, cs_typename, queryContextTable)
 import Gargantext.Database.Schema.ContextNodeNgrams (ContextNodeNgramsPoly(..))
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))
-import Gargantext.Database.Schema.Node
+import Gargantext.Database.Schema.Node (ns_hyperdata, ns_id, ns_search, ns_typename)
 import Gargantext.Prelude hiding (groupBy)
 import Opaleye hiding (Order)
 import Opaleye qualified as O hiding (Order)
@@ -95,9 +95,9 @@ searchDocInDatabase p t = runOpaQuery (queryDocInDatabase p t)
     queryDocInDatabase :: ParentId -> Text -> O.Select (Column SqlInt4, Column SqlJsonb)
     queryDocInDatabase _p q = proc () -> do
         row <- queryNodeSearchTable -< ()
-        restrict -< (_ns_search row)    @@ (sqlPlainToTSQuery (unpack q))
-        restrict -< (_ns_typename row) .== (sqlInt4 $ toDBid NodeDocument)
-        returnA  -< (_ns_id row, _ns_hyperdata row)
+        restrict -< (row ^. ns_search)    @@ (sqlPlainToTSQuery (unpack q))
+        restrict -< (row ^. ns_typename) .== (sqlInt4 $ toDBid NodeDocument)
+        returnA  -< (row ^. ns_id, row ^. ns_hyperdata)
 
 ------------------------------------------------------------------------
 -- | Search ngrams in documents, ranking them by TF-IDF. We narrow our
@@ -200,7 +200,7 @@ queryInCorpus :: HasDBid NodeType
 queryInCorpus cId t q = proc () -> do
   c <- queryContextSearchTable -< ()
   nc <- optionalRestrict queryNodeContextTable -<
-    \nc' -> (nc' ^. nc_context_id) .== _cs_id c
+    \nc' -> (nc' ^. nc_context_id) .== c ^. cs_id
   restrict -< (view nc_node_id <$> nc) .=== justFields (pgNodeId cId)
   restrict -< if t
                  then (view nc_category <$> nc) .=== justFields (sqlInt4 0)

src/Gargantext/Database/Admin/Types/Hyperdata/List.hs

@@ -30,7 +30,7 @@ import Control.Applicative
 import Gargantext.Prelude
 import Gargantext.Core.Viz.Types (Histo(..))
 import Gargantext.API.Ngrams.NgramsTree (NgramsTree)
-import Gargantext.API.Ngrams.Types (TabType)
+import Gargantext.Core.Ngrams.Types (TabType)
 import Gargantext.Database.Admin.Types.Hyperdata.Prelude
 import Gargantext.Database.Admin.Types.Metrics (ChartMetrics(..), Metrics)
 

src/Gargantext/Database/Query/Table/NodeContext.hs

@@ -11,8 +11,6 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# OPTIONS_GHC -fno-warn-orphans #-}
-
 {-# LANGUAGE Arrows                 #-}
 {-# LANGUAGE FunctionalDependencies #-}
 {-# LANGUAGE LambdaCase             #-}
@@ -52,11 +50,11 @@ import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
 import Gargantext.Core
 import Gargantext.Core.Types
-import Gargantext.Database.Admin.Types.Hyperdata
+import Gargantext.Database.Admin.Types.Hyperdata (Hyperdata, HyperdataDocument, hd_publication_date)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError, NodeError(..), nodeError)
-import Gargantext.Database.Prelude
-import Gargantext.Database.Schema.Context
-import Gargantext.Database.Schema.Node
+import Gargantext.Database.Prelude (DBCmd, execPGSQuery, mkCmd, restrictMaybe, runCountOpaQuery, runOpaQuery, runPGSQuery)
+import Gargantext.Database.Schema.Context (ContextRead, context_hyperdata, context_id, context_typename, queryContextTable)
+import Gargantext.Database.Schema.Node (NodeRead, node_id, node_typename, queryNodeTable)
 import Gargantext.Database.Schema.NodeContext
 import Gargantext.Prelude
 import Gargantext.Prelude.Crypto.Hash (Hash)

src/Gargantext/Database/Schema/NodeContext.hs

@@ -11,8 +11,6 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# OPTIONS_GHC -fno-warn-orphans #-}
-
 {-# LANGUAGE Arrows                 #-}
 {-# LANGUAGE FunctionalDependencies #-}
 {-# LANGUAGE QuasiQuotes            #-}

src/Gargantext/Utils/JohnSnowNLP.hs

@@ -14,7 +14,7 @@ Portability : POSIX
 module Gargantext.Utils.JohnSnowNLP where
 
 import Control.Lens
-import Data.Aeson (encode, ToJSON, toJSON, FromJSON, parseJSON, Value(..), (.:), (.:?))
+import Data.Aeson (encode, ToJSON(..), FromJSON(..), Value(..), (.:), (.:?))
 import Data.Aeson.TH (deriveJSON)
 import Data.Aeson.Types (prependFailure, typeMismatch)
 import Data.List.Safe qualified as LS

src/Gargantext/Utils/Servant.hs

@@ -15,7 +15,7 @@ import Data.Csv (defaultEncodeOptions, encodeByNameWith, encodeDefaultOrderedByN
 import qualified Data.Map.Strict as Map
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as TE
-import Gargantext.API.Ngrams.Types (mSetToList, NgramsRepoElement(..), NgramsTableMap, NgramsTerm(..), unNgramsTerm)
+import Gargantext.Core.Ngrams.Types (mSetToList, NgramsRepoElement(..), NgramsTableMap, NgramsTerm(..), unNgramsTerm)
 import Gargantext.Core.Types.Main (ListType(..))
 import Network.HTTP.Media ((//), (/:))
 import qualified Prelude

test/Test/API/UpdateList.hs

@@ -23,7 +23,7 @@ import Fmt
 import Gargantext.API.Admin.Auth.Types (Token)
 import Gargantext.API.Admin.Orchestrator.Types
 import Gargantext.API.Ngrams.List ( ngramsListFromCSVData )
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Types
 import Gargantext.Core.Types.Individu
 import Gargantext.Database.Action.User

test/Test/Database/Operations/NodeStory.hs

@@ -21,7 +21,7 @@ import Data.Set qualified as Set
 import Database.PostgreSQL.Simple qualified as PSQL
 import Database.PostgreSQL.Simple.SqlQQ
 import Gargantext.API.Ngrams (commitStatePatch, setListNgrams)
-import Gargantext.API.Ngrams.Types (MSet(..), NgramsPatch(..), NgramsRepoElement(..), NgramsTerm(..), Versioned(..), mkNgramsTablePatch, mSetFromList, nre_children, nre_list, nre_parent, nre_root)
+import Gargantext.Core.Ngrams.Types (MSet(..), NgramsPatch(..), NgramsRepoElement(..), NgramsTerm(..), Versioned(..), mkNgramsTablePatch, mSetFromList, nre_children, nre_list, nre_parent, nre_root)
 import Gargantext.API.Ngrams.Tools (getNodeStory)
 import Gargantext.Core.NodeStory (ArchiveList, a_state, a_version, currentVersion, initArchive)
 import Gargantext.Core.NodeStory.Utils (saveNodeStory)

test/Test/Graph/Clustering.hs

@@ -15,8 +15,8 @@ module Test.Graph.Clustering where
 import Data.HashMap.Strict (HashMap)
 import Data.HashMap.Strict qualified as HashMap
 import Data.List qualified as List
-import Gargantext.API.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Methods.Similarities (Similarity(..))
+import Gargantext.CoreAPI.Ngrams.Types (NgramsTerm(..))
 import Gargantext.Core.Viz.Graph.Tools (doSimilarityMap)
 import Gargantext.Core.Viz.Graph.Tools.IGraph (spinglass)
 import Gargantext.Core.Viz.Graph.Types

test/Test/Ngrams/Query.hs

@@ -10,7 +10,7 @@ import Data.Patch.Class qualified as Patch
 import Data.Text qualified as T
 import Data.Validity qualified as Validity
 import Gargantext.API.Ngrams
-import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Ngrams.Types
 import Gargantext.Core.Types.Main
 import Gargantext.Core.Types.Query
 import Gargantext.Prelude

test/Test/Ngrams/Query/PaginationCorpus.hs

@@ -17,7 +17,7 @@ import           Prelude
 
 import           Data.Aeson
 import           Data.Map.Strict (Map)
-import           Gargantext.API.Ngrams.Types
+import           Gargantext.Core.Ngrams.Types
 import           Gargantext.Core.Types.Main
 import           Gargantext.Database.Admin.Types.Node
 import           System.IO.Unsafe