Merge branch...

Merge branch '345-docs-tsv-export-import-check-and-maintain-consistency-between-export-and-import-formats' of https://gitlab.iscpif.fr/gargantext/haskell-gargantext into 345-docs-tsv-export-import-check-and-maintain-consistency-between-export-and-import-formats

.gitignore

@@ -19,6 +19,9 @@ TAGS
 *.swp
 .dir-locals.el
 
+# VSCode
+.vscode
+
 # UI
 gui
 purescript-gargantext
@@ -45,3 +48,7 @@ devops/docker/js-cache
 
 cabal.project.local
 gargantext_profile_out.dot
+
+dev.jwk
+.psc-ide-port
+logs/
\ No newline at end of file

bin/metrics/metrics

+#!/bin/bash
+
+
+for f in bin/metrics/histo/*sql ; do
+  ./bin/psql gargantext.ini < $f
+done
+

devops/nix/config

+#!/bin/bash
+
+mkdir -p ~/.config/nix/
+echo "experimental-features = nix-command flakes" > ~/.config/nix/nix.conf
+
+

gargantext.cabal

@@ -5,7 +5,7 @@ cabal-version: 3.4
 -- see: https://github.com/sol/hpack
 
 name:           gargantext
-version: 0.0.7.1.1
+version: 0.0.7.1.5.3
 synopsis:       Search, map, share
 description:    Please see README.md
 category:       Data
@@ -134,9 +134,29 @@ library
       Gargantext.API.Node.File
       Gargantext.API.Node.Share
       Gargantext.API.Node.ShareURL
+      Gargantext.API.Node.Types
       Gargantext.API.Node.Update
       Gargantext.API.Prelude
       Gargantext.API.Routes
+      Gargantext.API.Routes.Named
+      Gargantext.API.Routes.Named.Contact
+      Gargantext.API.Routes.Named.Context
+      Gargantext.API.Routes.Named.Corpus
+      Gargantext.API.Routes.Named.Count
+      Gargantext.API.Routes.Named.Document
+      Gargantext.API.Routes.Named.File
+      Gargantext.API.Routes.Named.FrameCalc
+      Gargantext.API.Routes.Named.List
+      Gargantext.API.Routes.Named.Metrics
+      Gargantext.API.Routes.Named.Node
+      Gargantext.API.Routes.Named.Private
+      Gargantext.API.Routes.Named.Public
+      Gargantext.API.Routes.Named.Search
+      Gargantext.API.Routes.Named.Share
+      Gargantext.API.Routes.Named.Table
+      Gargantext.API.Routes.Named.Tree
+      Gargantext.API.Routes.Named.Viz
+      Gargantext.API.Routes.Types
       Gargantext.Core
       Gargantext.Core.Mail.Types
       Gargantext.Core.Methods.Similarities
@@ -276,12 +296,13 @@ library
       Gargantext.API.Node.Corpus.Searx
       Gargantext.API.Node.Document.Export
       Gargantext.API.Node.Document.Export.Types
+      Gargantext.API.Node.Phylo.Export
+      Gargantext.API.Node.Phylo.Export.Types
       Gargantext.API.Node.DocumentUpload
       Gargantext.API.Node.DocumentsFromWriteNodes
       Gargantext.API.Node.FrameCalcUpload
       Gargantext.API.Node.Get
       Gargantext.API.Node.New
-      Gargantext.API.Node.Types
       Gargantext.API.Public
       Gargantext.API.Search
       Gargantext.API.Server

src/Gargantext/API/Count.hs

@@ -16,8 +16,22 @@ Count API part of Gargantext.
 {-# LANGUAGE TypeOperators      #-}
 {-# LANGUAGE DeriveAnyClass     #-}
 
-module Gargantext.API.Count
-      where
+module Gargantext.API.Count (
+    CountAPI
+  , Scraper(..)
+  , QueryBool(..)
+  , Query(..)
+  , Message(..)
+  , Code
+  , Error
+  , Errors
+  , Counts(..)
+  , Count(..)
+
+  -- * functions
+  , count
+  , scrapers
+  ) where
 
 import Data.Swagger ( ToSchema(..), genericDeclareNamedSchema )
 import Data.Text (pack)

src/Gargantext/API/Node/Corpus/Export.hs

+{-# LANGUAGE TypeOperators #-}
 {-|
 Module      : Gargantext.API.Node.Corpus.Export
 Description : Corpus export
@@ -41,7 +42,14 @@ import Gargantext.Database.Query.Table.NodeContext (selectDocNodes)
 import Gargantext.Database.Schema.Context (_context_id)
 import Gargantext.Prelude hiding (hash)
 import Gargantext.Prelude.Crypto.Hash (hash)
-import Servant (Headers, Header, addHeader)
+import Servant (Headers, Header, addHeader, Summary, (:>), JSON, Get, QueryParam)
+
+--------------------------------------------------
+type API = Summary "Corpus Export"
+            :> "export"
+            :> QueryParam "listId"     ListId
+            :> QueryParam "ngramsType" NgramsType
+            :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] Corpus)
 
 --------------------------------------------------
 -- | Hashes are ordered by Set

src/Gargantext/API/Node/Corpus/Export/Types.hs

@@ -18,8 +18,7 @@ import Data.Swagger ( ToSchema(..), genericDeclareNamedSchema, ToParamSchema(..)
 import Data.Text (Text)
 import GHC.Generics (Generic)
 import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
-import Gargantext.Core.Text.Ngrams (NgramsType(..))
-import Gargantext.Core.Types ( ListId, TODO )
+import Gargantext.Core.Types ( TODO )
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Servant
 
@@ -37,11 +36,4 @@ instance ToSchema Corpus where
 -------
 instance ToParamSchema Corpus where
   toParamSchema _ = toParamSchema (Proxy :: Proxy TODO)
---------------------------------------------------
-type API = Summary "Corpus Export"
-            :> "export"
-            :> QueryParam "listId"     ListId
-            :> QueryParam "ngramsType" NgramsType
-            :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] Corpus)
-
 $(deriveJSON (unPrefix "_c_") ''Corpus)

src/Gargantext/API/Node/Corpus/New.hs

@@ -22,10 +22,9 @@ module Gargantext.API.Node.Corpus.New
 
 import Conduit
 import Control.Lens ( view, non )
-import Data.Aeson ( genericParseJSON, genericToJSON )
 import Data.ByteString.Base64 qualified as BSB64
 import Data.Conduit.Internal (zipSources)
-import Data.Swagger ( ToSchema(..), genericDeclareNamedSchema )
+import Data.Swagger ( ToSchema(..) )
 import Data.Text qualified as T
 import Data.Text.Encoding qualified as TE
 import EPO.API.Client.Types qualified as EPO
@@ -34,17 +33,16 @@ import Gargantext.API.Admin.Types (HasSettings)
 import Gargantext.API.Ngrams (commitStatePatch, Versioned(..))
 import Gargantext.API.Node.Corpus.New.Types ( FileFormat(..), FileType(..) )
 import Gargantext.API.Node.Corpus.Searx ( triggerSearxSearch )
-import Gargantext.API.Node.Corpus.Types ( Database, Datafield(Web), database2origin )
+import Gargantext.API.Node.Corpus.Types ( Datafield(Web), database2origin )
 import Gargantext.API.Node.Corpus.Update (addLanguageToCorpus)
 import Gargantext.API.Node.Types
-import Gargantext.Core (Lang(..), withDefaultLanguage, defaultLanguage)
+import Gargantext.Core (withDefaultLanguage, defaultLanguage)
 import Gargantext.Core.NodeStory (HasNodeStoryImmediateSaver, HasNodeArchiveStoryImmediateSaver, currentVersion, NgramsStatePatch')
 import Gargantext.API.Admin.Orchestrator.Types qualified as API
 import Gargantext.Core.Text.Corpus.Query qualified as API
 import Gargantext.Core.Text.Corpus.Parsers qualified as Parser (FileType(..), parseFormatC, _ParseFormatError)
-import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Core.Utils.Prefix (unPrefix)
 import Gargantext.Database.Action.Flow (flowCorpus, getDataText, flowDataText, TermType(..){-, allDataOrigins-})
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Action.Mail (sendMail)
@@ -64,7 +62,6 @@ import Gargantext.Prelude.Config (gc_max_docs_parsers)
 import Gargantext.System.Logging ( logLocM, LogLevel(..) )
 import Gargantext.Utils.Jobs.Monad (JobHandle, MonadJobStatus(..))
 import Servant ( JSON, type (:>), FormUrlEncoded, Capture, Summary )
-import Servant.Job.Utils (jsonOptions)
 import Test.QuickCheck.Arbitrary (Arbitrary(..))
 
 ------------------------------------------------------------------------
@@ -136,41 +133,6 @@ info :: ApiInfo
 info = ApiInfo API.externalAPIs
 
 ------------------------------------------------------------------------
-------------------------------------------------------------------------
-data WithQuery = WithQuery
-  { _wq_query        :: !API.RawQuery
-  , _wq_databases    :: !Database
-  , _wq_datafield    :: !(Maybe Datafield)
-  , _wq_lang         :: !Lang
-  , _wq_node_id      :: !Int
-  , _wq_flowListWith :: !FlowSocialListWith
-  , _wq_pubmedAPIKey :: !(Maybe Text)
-  , _wq_epoAPIUser   :: !(Maybe Text)
-  , _wq_epoAPIToken  :: !(Maybe Text)
-  }
-  deriving (Show, Eq, Generic)
-
-makeLenses ''WithQuery
-instance FromJSON WithQuery where
-  parseJSON = genericParseJSON $ jsonOptions "_wq_"
-instance ToJSON WithQuery where
-  toJSON = genericToJSON $ jsonOptions "_wq_"
-instance ToSchema WithQuery where
-  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wq_")
-
-instance Arbitrary WithQuery where
-  arbitrary = WithQuery <$> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-                        <*> arbitrary
-
-------------------------------------------------------------------------
-
 type AddWithQuery = Summary "Add with Query to corpus endpoint"
    :> "corpus"
      :> Capture "corpus_id" CorpusId

src/Gargantext/API/Node/Phylo/Export.hs

+{-|
+Module      : Gargantext.API.Node.Phylo.Export
+Description : Phylo export
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
+module Gargantext.API.Node.Phylo.Export
+  where
+
+import Data.Aeson
+import Data.Text qualified as T
+import Gargantext.API.Node.Phylo.Export.Types
+import Gargantext.API.Prelude (GargNoServer, GargServer)
+import Gargantext.Core.Viz.Phylo.API.Tools
+import Gargantext.Core.Viz.Phylo.Example (phyloCleopatre)
+import Gargantext.Database.Admin.Types.Node (PhyloId, NodeId,)
+import Gargantext.Prelude
+import Servant
+
+api :: NodeId
+    -- ^ The ID of the target user
+    -> PhyloId
+    -> GargServer API
+api userNodeId dId = getPhyloJson userNodeId dId 
+                :<|> getPhyloDot userNodeId dId
+
+getPhyloJson  :: NodeId
+                -- ^ The ID of the target user
+              -> PhyloId
+              -> GargNoServer (Headers '[Header "Content-Disposition" T.Text] Value)
+getPhyloJson _ pId = do
+  maybePhyloData <- getPhyloData pId
+  let phyloData = fromMaybe phyloCleopatre maybePhyloData
+  phyloJson <- liftBase $ phylo2dot2json phyloData
+  pure $ addHeader (T.concat [ "attachment; filename="
+                                          , "GarganText_Phylo-"
+                                          , T.pack (show pId)
+                                          , ".json" ])
+                                          phyloJson
+
+
+getPhyloDot :: NodeId
+            -> PhyloId
+            -> GargNoServer (Headers '[Header "Content-Disposition" T.Text] T.Text)
+getPhyloDot _ pId = do
+  maybePhyloData <- getPhyloData pId
+  let phyloData = fromMaybe phyloCleopatre maybePhyloData
+  phyloDot <- liftBase $ phylo2dot phyloData
+  pure $ addHeader (T.concat [ "attachment; filename="
+                                          , "GarganText_Phylo-"
+                                          , T.pack (show pId)
+                                          , ".dot" ])
+                                          phyloDot
\ No newline at end of file

src/Gargantext/API/Node/Phylo/Export/Types.hs

+{-|
+Module      : Gargantext.API.Node.Phylo.Export.Types
+Description : Types for Gargantext.API.Node.Phylo.Export
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE TypeOperators   #-}
+
+module Gargantext.API.Node.Phylo.Export.Types where
+
+import Data.Aeson
+import Data.Aeson.TH (deriveJSON)
+-- import Data.Csv (DefaultOrdered(..), ToNamedRecord(..), (.=), header, namedRecord)
+import Data.Swagger ( genericDeclareNamedSchema, ToParamSchema(..), ToSchema(..) )
+-- import Data.Text qualified as T
+-- import Data.Text.Encoding qualified as TE
+import Gargantext.Core.Types ( Node, TODO )
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Database.Admin.Types.Hyperdata.Phylo ( HyperdataPhylo(..) )
+import Gargantext.Database.Admin.Types.Node (PhyloId)
+-- import Gargantext.Database.Schema.Node (NodePoly(..))
+-- import Gargantext.Utils.Servant (ZIP)
+-- import Gargantext.Utils.Zip (zipContentsPure)
+import Protolude
+import Servant ((:>), (:<|>), Get, Header, Headers(..), JSON, Summary) --, PlainText, MimeRender(..)
+
+
+-- | Phylo Export
+data PhyloExport =
+  PhyloExport { _pe_phylos    :: [Phylo]
+              , _pe_garg_version :: Text
+              } deriving (Generic)
+
+data PhyloExportDOT =
+  PhyloExportDOT  { _ped_dexp   :: PhyloExport
+                  , _ped_doc_id :: PhyloId 
+                  } deriving (Generic)
+
+data PhyloExportJSON = 
+  PhyloExportJSON { _pej_pexp     :: PhyloExport
+                  , _pej_phy_id   :: PhyloId
+                  } deriving (Generic)
+    
+    
+data Phylo =
+  Phylo { _p_phylo    :: Node HyperdataPhylo
+        , _p_hash     :: Hash
+        } deriving (Generic)
+
+--instance Read Phylo where
+--  read "" = panic "not implemented"
+-- instance DefaultOrdered Phylo where
+--   headerOrder _ = header ["Publication Day"
+--                          , "Publication Month"
+--                          , "Publication Year"
+--                          , "Authors"
+--                          , "Title"
+--                          , "Source"
+--                          , "Abstract"]
+-- instance ToNamedRecord Phylo where
+--   toNamedRecord (Phylo { _p_phylo = Node { .. }}) =
+--     namedRecord
+--     [ "Publication Day" .= _hd_publication_day _node_hyperdata
+--     , "Publication Month" .= _hd_publication_month _node_hyperdata
+--     , "Publication Year" .= _hd_publication_year _node_hyperdata
+--     , "Authors" .= _hd_authors _node_hyperdata
+--     , "Title" .= _hd_title _node_hyperdata
+--     , "Source" .= (TE.encodeUtf8 <$> _hd_source _node_hyperdata)
+--     , "Abstract" .= (TE.encodeUtf8 <$> _hd_abstract _node_hyperdata) ]
+
+
+type Hash = Text
+-------
+instance ToSchema PhyloExport where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_pe_")
+
+instance ToSchema PhyloExportDOT where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_ped_")
+  
+instance ToSchema Phylo where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_p_")
+
+
+-------
+instance ToParamSchema PhyloExport where
+  toParamSchema _ = toParamSchema (Proxy :: Proxy TODO)
+
+instance ToParamSchema PhyloExportDOT where
+  toParamSchema _ = toParamSchema (Proxy :: Proxy TODO)
+
+instance ToParamSchema Phylo where
+  toParamSchema _ = toParamSchema (Proxy :: Proxy TODO)
+
+--------------------------------------------------
+type API = Summary "Phylo Export"
+            :> "export"
+            :> ( "json"
+                 :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] Value)
+               :<|> "dot"
+                 :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] Text)
+                )
+
+-- type API = Summary "Phylo Export"
+--             :> "export"
+--                  :> Get '[JSON,DOT] (Headers '[Servant.Header "Content-Disposition" Text] Value)
+
+
+
+
+$(deriveJSON (unPrefix "_p_") ''Phylo)
+$(deriveJSON (unPrefix "_pe_") ''PhyloExport)
+
+
+------
+
+-- Needs to be here because of deriveJSON TH above
+
+-- pedFileName :: PhyloExportDOT -> Text
+-- pedFileName (PhyloExportDOT { .. }) = "GarganText_DocsList-" <> show _ped_doc_id <> ".dot"
+    
+-- instance MimeRender ZIP PhyloExportDOT where
+--   mimeRender _ dexpz@(PhyloExportDOT { .. }) =
+--     zipContentsPure (T.unpack $ pedFileName dexpz) (encode _ped_dexp)

src/Gargantext/API/Node/Types.hs

@@ -19,12 +19,15 @@ import Data.ByteString.Base64 qualified as BSB64
 import Data.Swagger
 import Data.Text qualified as T
 import Gargantext.API.Node.Corpus.New.Types (FileType, FileFormat)
+import Gargantext.API.Node.Corpus.Types
 import Gargantext.Core (Lang(..))
+import Gargantext.Core.Text.Corpus.Query qualified as API
 import Gargantext.Core.Text.List.Social (FlowSocialListWith)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Database.GargDB qualified as GargDB
 import Gargantext.Prelude
 import Servant.Job.Utils (jsonOptions)
+import Test.QuickCheck
 import Web.FormUrlEncoded          (FromForm, ToForm)
 
 -------------------------------------------------------
@@ -77,3 +80,36 @@ instance GargDB.SaveFile NewWithFile where
 
 --instance GargDB.ReadFile NewWithFile where
 --  readFile' = TIO.readFile
+
+data WithQuery = WithQuery
+  { _wq_query        :: !API.RawQuery
+  , _wq_databases    :: !Database
+  , _wq_datafield    :: !(Maybe Datafield)
+  , _wq_lang         :: !Lang
+  , _wq_node_id      :: !Int
+  , _wq_flowListWith :: !FlowSocialListWith
+  , _wq_pubmedAPIKey :: !(Maybe Text)
+  , _wq_epoAPIUser   :: !(Maybe Text)
+  , _wq_epoAPIToken  :: !(Maybe Text)
+  }
+  deriving (Show, Eq, Generic)
+
+makeLenses ''WithQuery
+instance FromJSON WithQuery where
+  parseJSON = genericParseJSON $ jsonOptions "_wq_"
+instance ToJSON WithQuery where
+  toJSON = genericToJSON $ jsonOptions "_wq_"
+instance ToSchema WithQuery where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wq_")
+
+instance Arbitrary WithQuery where
+  arbitrary = WithQuery <$> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+                        <*> arbitrary
+

src/Gargantext/API/Routes.hs

@@ -16,6 +16,7 @@ Portability : POSIX
 {-# LANGUAGE TypeOperators        #-}
 {-# LANGUAGE TypeApplications     #-}
 {-# LANGUAGE UndecidableInstances #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-} -- instance HasSwagger (WithCustomErrorScheme GargAPI)
 
 module Gargantext.API.Routes
       where
@@ -29,7 +30,6 @@ import Gargantext.API.Admin.FrontEnd (FrontEndAPI)
 import Gargantext.API.Auth.PolicyCheck
 import Gargantext.API.Context
 import Gargantext.API.Count  (CountAPI, count, Query)
-import Gargantext.API.Errors (GargErrorScheme (..))
 import Gargantext.API.Errors.Types
 import Gargantext.API.GraphQL qualified as GraphQL
 import Gargantext.API.Members (MembersAPI, members)
@@ -39,13 +39,15 @@ import Gargantext.API.Node
 import Gargantext.API.Node.Contact qualified as Contact
 import Gargantext.API.Node.Corpus.Annuaire qualified as Annuaire
 import Gargantext.API.Node.Corpus.Export qualified as CorpusExport
-import Gargantext.API.Node.Corpus.Export.Types qualified as CorpusExport
 import Gargantext.API.Node.Corpus.New qualified as New
 import Gargantext.API.Node.Document.Export qualified as DocumentExport
 import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
+import Gargantext.API.Node.Phylo.Export qualified as PhyloExport
+import Gargantext.API.Node.Phylo.Export.Types qualified as PhyloExport
 import Gargantext.API.Node.ShareURL qualified as ShareURL
 import Gargantext.API.Prelude
 import Gargantext.API.Public qualified as Public
+import Gargantext.API.Routes.Types
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Viz.Graph.API
 import Gargantext.Database.Admin.Types.Hyperdata
@@ -54,41 +56,17 @@ import Gargantext.Database.Prelude (HasConfig(..))
 import Gargantext.Prelude
 import Gargantext.Prelude.Config (gc_max_docs_scrapers)
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
-import Network.Wai (requestHeaders)
 import Servant
 import Servant.Auth as SA
 import Servant.Auth.Swagger ()
-import Servant.Ekg
-import Servant.Server.Internal.Delayed
-import Servant.Server.Internal.DelayedIO
 import Servant.Swagger
 import Servant.Swagger.UI
-import qualified Data.List as L
-
-data WithCustomErrorScheme a
-
-instance (HasServer subApi ctx) => HasServer (WithCustomErrorScheme subApi) ctx where
-  type ServerT (WithCustomErrorScheme subApi) m = GargErrorScheme -> ServerT subApi m
-  hoistServerWithContext _ pc nt s = hoistServerWithContext (Proxy :: Proxy subApi) pc nt . s
-  route Proxy ctx d = route (Proxy :: Proxy subApi) ctx (d `addHeaderCheck` getErrorScheme)
-    where
-      getErrorScheme :: DelayedIO GargErrorScheme
-      getErrorScheme = withRequest $ \rq -> do
-        let hdrs = requestHeaders rq
-            in case L.lookup "X-Garg-Error-Scheme" hdrs of
-                 Nothing     -> pure GES_old
-                 Just "new"  -> pure GES_new
-                 Just _      -> pure GES_old
+
+type GargAPI = MkGargAPI (GargAPIVersion GargAPI')
 
 instance HasSwagger (WithCustomErrorScheme GargAPI) where
     toSwagger _ = toSwagger (Proxy :: Proxy GargAPI)
 
-instance HasEndpoint sub => HasEndpoint (WithCustomErrorScheme sub) where
-  getEndpoint _ = getEndpoint (Proxy :: Proxy sub)
-  enumerateEndpoints _ = enumerateEndpoints (Proxy :: Proxy sub)
-
-type GargAPI = MkGargAPI (GargAPIVersion GargAPI')
-
 type MkGargAPI sub = "api" :> Summary "API " :> sub
 --- | TODO          :<|> Summary "Latest API" :> GargAPI'
 
@@ -178,9 +156,12 @@ type GargPrivateAPI' =
                            :> "ngrams"
                            :> TableNgramsApi
 
-           :<|> "texts" :> Capture "node_id" DocId
+           :<|> "texts"    :> Capture "node_id" DocId
                            :> DocumentExport.API
 
+           :<|> "phylo"     :> Capture "node_id" DocId
+                            :> PhyloExport.API
+
         -- :<|> "counts" :> Stream GET NewLineFraming '[JSON] Count :> CountAPI
             -- TODO-SECURITY
            :<|> "count"    :> Summary "Count endpoint"
@@ -196,6 +177,10 @@ type GargPrivateAPI' =
                            :> Capture "graph_id" NodeId
                            :> GraphAPI
 
+          --  :<|> "phylo"    :> Summary "Phylo endpoint"
+          --                  :> Capture "pylo_id" NodeId
+          --                  :> 
+
            -- TODO move to NodeAPI?
            -- Tree endpoint
            :<|> "tree"    :> Summary "Tree endpoint"
@@ -282,6 +267,8 @@ serverPrivateGargAPI' authenticatedUser@(AuthenticatedUser userNodeId userId)
 
      :<|> DocumentExport.api userNodeId
 
+     :<|> PhyloExport.api userNodeId
+
      :<|> count -- TODO: undefined
 
      -- :<|> withAccess (Proxy :: Proxy (Search.API Search.SearchResult)) Proxy uid

src/Gargantext/API/Routes/Named.hs

+
+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Auth (ForgotPasswordAsyncParams)
+import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Routes.Named.Public
+import Gargantext.API.Routes.Named.Private
+import Servant.API ((:>), (:-), JSON, ReqBody, Post, Get, QueryParam)
+import Servant.API.Description (Summary)
+import Servant.API.NamedRoutes
+
+type GargAPI = NamedRoutes (MkGargAPI (GargAPIVersion GargAPI'))
+
+
+data MkGargAPI sub mode = MkGargAPI
+  { mkGargAPI :: mode :- "api" :> Summary "API " :> NamedRoutes sub
+  } deriving Generic
+
+
+data GargAPIVersion sub mode = GargAPIVersion
+  { gargAPIVersion :: mode :- "v1.0" :> Summary "Garg API Version " :> NamedRoutes sub
+  } deriving Generic
+
+
+data GargAPI' mode = GargAPI'
+  { gargAuthAPI                :: mode :- NamedRoutes AuthAPI
+  , gargForgotPasswordAPI      :: mode :- "forgot-password" :> NamedRoutes ForgotPasswordAPI
+  , gargForgotPasswordAsyncAPI :: mode :- "async" :> "forgot-password" :> NamedRoutes ForgotPasswordAsyncAPI
+  , gargVersionAPI             :: mode :- NamedRoutes GargVersion
+  , gargPrivateAPI             :: mode :- NamedRoutes GargPrivateAPI
+  , gargPublicAPI              :: mode :- "public" :> NamedRoutes GargPublicAPI
+  } deriving Generic
+
+
+data AuthAPI mode = AuthAPI
+  { authEp :: mode :- "auth"  :> Summary "AUTH API"
+                              :> ReqBody '[JSON] AuthRequest
+                              :> Post    '[JSON] AuthResponse
+  } deriving Generic
+
+
+data ForgotPasswordAPI mode = ForgotPasswordAPI
+  { forgotPasswordPostEp :: mode :- Summary "Forgot password POST API"
+                                 :> ReqBody '[JSON] ForgotPasswordRequest
+                                 :> Post '[JSON] ForgotPasswordResponse
+  , forgotPasswordGetEp  :: mode :- Summary "Forgot password GET API"
+                                 :> QueryParam "uuid" Text
+                                 :> Get '[JSON] ForgotPasswordGet
+  } deriving Generic
+
+
+data ForgotPasswordAsyncAPI mode = ForgotPasswordAsyncAPI
+  { forgotPasswordAsyncEp :: mode :- Summary "Forgot password asnc"
+                                  :> AsyncJobs JobLog '[JSON] ForgotPasswordAsyncParams JobLog
+  } deriving Generic
+
+
+data GargVersion mode = GargVersion
+  { gargVersionEp :: "version" :> Summary "Backend version" :> Get '[JSON] Text
+  } deriving Generic

src/Gargantext/API/Routes/Named/Contact.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.Contact (
+  -- * Routes types
+    ContactAPI(..)
+  , ContactAsyncAPI(..)
+
+  -- * API types (appears in the routes)
+  , AddContactParams(..)
+  ) where
+
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Routes.Named.Node (NodeNodeAPI(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Contact
+import Gargantext.Database.Admin.Types.Node
+import Servant
+
+
+data ContactAPI mode = ContactAPI
+  { contactAsyncAPI :: mode :- "contact" :> Summary "Contact endpoint" :> NamedRoutes ContactAsyncAPI
+  , getContactEp    :: mode :- Capture "contact_id" NodeId :> NamedRoutes (NodeNodeAPI HyperdataContact)
+  } deriving Generic
+
+
+newtype ContactAsyncAPI mode = ContactAsyncAPI
+  { addContactAsyncEp :: mode :- AsyncJobs JobLog '[JSON] AddContactParams JobLog
+  } deriving Generic
+
+
+data AddContactParams = AddContactParams         { firstname :: !Text, lastname :: !Text }
+                      | AddContactParamsAdvanced { firstname :: !Text
+                                                 , lastname  :: !Text
+                                                 -- TODO add others fields
+                                                 }
+    deriving Generic

src/Gargantext/API/Routes/Named/Context.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.Context where
+
+import GHC.Generics
+import Gargantext.Database.Admin.Types.Node
+import Servant
+
+data ContextAPI mode a = ContextAPI
+  { getNodeEp :: mode :- Get '[JSON] (Node a)
+  } deriving Generic

src/Gargantext/API/Routes/Named/Corpus.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.Corpus (
+  -- * Routes types
+    CorpusExportAPI(..)
+  , AddWithForm(..)
+  , AddWithQuery(..)
+  ) where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Node.Corpus.Annuaire hiding (AddWithForm)
+import Gargantext.API.Node.Corpus.Export.Types
+import Gargantext.API.Node.Types
+import Gargantext.Core.Text.Ngrams (NgramsType(..))
+import Gargantext.Database.Admin.Types.Node
+import Servant
+
+--------------------------------------------------
+newtype CorpusExportAPI mode = CorpusExportAPI
+  { corpusExportEp :: mode :- Summary "Corpus Export"
+                           :> "export"
+                           :> QueryParam "listId"     ListId
+                           :> QueryParam "ngramsType" NgramsType
+                           :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] Corpus)
+  } deriving Generic
+
+newtype AddWithForm mode = AddWithForm
+  { addWithFormEp :: mode :- Summary "Add with FormUrlEncoded to annuaire endpoint"
+                          :> "annuaire"
+                          :> Capture "annuaire_id" AnnuaireId
+                          :> "add"
+                          :> "form"
+                          :> "async"
+                          :> AsyncJobs JobLog '[FormUrlEncoded] AnnuaireWithForm JobLog
+  } deriving Generic
+
+
+newtype AddWithQuery mode = AddWithQuery
+  { addWithQueryEp :: mode :- Summary "Add with Query to corpus endpoint"
+                           :> "corpus"
+                           :> Capture "corpus_id" CorpusId
+                           :> "query"
+                           :> AsyncJobs JobLog '[JSON] WithQuery JobLog
+  } deriving Generic

src/Gargantext/API/Routes/Named/Count.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.Count (
+  -- * Routes types
+    CountAPI(..)
+
+  -- * Re-exports
+  , module X
+  ) where
+
+import GHC.Generics
+import Servant
+import Gargantext.API.Count as X hiding (CountAPI)
+
+
+newtype CountAPI mode = CountAPI
+  { postCountsEp :: mode :- Post '[JSON] X.Counts
+  } deriving Generic

src/Gargantext/API/Routes/Named/Document.hs

+{-# LANGUAGE DeriveGeneric   #-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE TypeOperators   #-}
+module Gargantext.API.Routes.Named.Document (
+  -- * Routes types
+    DocumentsFromWriteNodesAPI(..)
+  , DocumentUploadAPI(..)
+  , DocumentExportAPI(..)
+
+  -- * API types
+  , Params(..)
+  , DocumentUpload(..)
+
+  -- * functions and lenses
+  , du_title
+  , du_sources
+  , du_language
+  , du_date
+  , du_authors
+  , du_abstract
+  ) where
+
+import Control.Lens
+import Data.Aeson
+import Data.Swagger hiding (fieldLabelModifier)
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Node.Document.Export.Types
+import Gargantext.Core
+import Gargantext.Core.Text.List.Social
+import Gargantext.Core.Utils.Prefix
+import Gargantext.Utils.Servant (ZIP)
+import Prelude
+import Servant
+
+newtype DocumentExportAPI mode = DocumentExportAPI
+  { documentExportAPI ::
+      mode :- Summary "Document Export"
+           :> "export"
+           :> ( "json"
+                :> Get '[JSON] (Headers '[Servant.Header "Content-Disposition" Text] DocumentExport)
+              :<|> "json.zip"
+                :> Get '[ZIP] (Headers '[Servant.Header "Content-Disposition" Text] DocumentExportZIP)
+              :<|> "csv"
+                :> Get '[PlainText] (Headers '[Servant.Header "Content-Disposition" Text] Text) )
+  } deriving Generic
+
+newtype DocumentsFromWriteNodesAPI mode = DocumentsFromWriteNodesAPI
+  { docFromWriteNodesEp :: mode :- Summary " Documents from Write nodes."
+                                :> AsyncJobs JobLog '[JSON] Params JobLog
+  } deriving Generic
+
+
+newtype DocumentUploadAPI mode = DocumentUploadAPI
+  { uploadDocAsyncEp :: mode :- Summary " Document upload"
+                             :> "document"
+                             :> "upload"
+                             :> "async"
+                             :> AsyncJobs JobLog '[JSON] DocumentUpload JobLog
+  } deriving Generic
+
+
+data Params = Params
+  { id         :: Int
+  , paragraphs :: Text
+  , lang       :: Lang
+  , selection  :: FlowSocialListWith
+  }
+  deriving (Generic, Show)
+
+
+data DocumentUpload = DocumentUpload
+  { _du_abstract :: Text
+  , _du_authors  :: Text
+  , _du_sources  :: Text
+  , _du_title    :: Text
+  , _du_date     :: Text
+  , _du_language :: Text
+  }
+  deriving Generic
+
+
+--
+-- instances
+--
+
+instance FromJSON Params where
+  parseJSON = genericParseJSON defaultOptions
+
+instance ToJSON Params where
+  toJSON = genericToJSON defaultOptions
+
+instance ToSchema Params
+
+$(makeLenses ''DocumentUpload)
+
+instance ToSchema DocumentUpload
+
+instance FromJSON DocumentUpload
+  where
+    parseJSON = genericParseJSON
+            ( defaultOptions { sumEncoding = ObjectWithSingleField
+                            , fieldLabelModifier = unCapitalize . dropPrefix "_du_"
+                            , omitNothingFields = True
+                            }
+            )
+
+instance ToJSON DocumentUpload
+  where
+    toJSON = genericToJSON
+           ( defaultOptions { sumEncoding = ObjectWithSingleField
+                            , fieldLabelModifier = unCapitalize . dropPrefix "_du_"
+                            , omitNothingFields = True
+                            }
+           )
+

src/Gargantext/API/Routes/Named/File.hs

+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE TypeOperators #-}
+module Gargantext.API.Routes.Named.File (
+  -- * Routes types
+    FileAPI(..)
+  , FileAsyncAPI(..)
+  ) where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Node.File
+import Gargantext.API.Node.Types
+import Servant
+
+data FileAPI mode = FileAPI
+  { fileDownloadEp :: mode :- Summary "File download"
+                           :> "download"
+                           :> Get '[RESPONSE] (Headers '[Servant.Header "Content-Type" Text] BSResponse)
+  } deriving Generic
+
+
+data FileAsyncAPI mode = FileAsyncAPI
+  { addFileAsyncEp :: mode :- Summary "File Async Api"
+                           :> "file"
+                           :> "add"
+                           :> AsyncJobs JobLog '[FormUrlEncoded] NewWithFile JobLog
+  } deriving Generic
+

src/Gargantext/API/Routes/Named/FrameCalc.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.FrameCalc where
+
+import Servant
+import GHC.Generics
+import Gargantext.API.Node.FrameCalcUpload (FrameCalcUpload)
+import Gargantext.API.Admin.Orchestrator.Types
+
+data FrameCalcAPI mode = FrameCalcAPI
+  { frameCalcUploadEp :: mode :- Summary " FrameCalc upload"
+                              :> "add"
+                              :> "framecalc"
+                              :> "async"
+                              :> AsyncJobs JobLog '[JSON] FrameCalcUpload JobLog
+  } deriving Generic
+

src/Gargantext/API/Routes/Named/List.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.List where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Ngrams.List.Types
+import Gargantext.API.Ngrams.Types
+import Gargantext.API.Types (HTML)
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Utils.Servant qualified as GUS
+import Servant
+
+newtype GETAPI mode = GETAPI
+  { getListEp :: mode :- Summary "Get List"
+                      :> "lists"
+                      :> Capture "listId" ListId
+                      :> ( "json"
+                           :> Get '[JSON, HTML] (Headers '[Header "Content-Disposition" Text] NgramsList)
+                         :<|>  "json.zip"
+                           :> Get '[GUS.ZIP] (Headers '[Header "Content-Disposition" Text] NgramsListZIP)
+                         :<|> "csv"
+                           :> Get '[GUS.CSV] (Headers '[Header "Content-Disposition" Text] NgramsTableMap) )
+  } deriving Generic
+
+
+newtype JSONAPI mode = JSONAPI
+  { updateListJSONEp :: mode :- Summary "Update List"
+                             :> "lists"
+                             :> Capture "listId" ListId
+                             :> "add"
+                             :> "form"
+                             :> "async"
+                             :> AsyncJobs JobLog '[FormUrlEncoded] WithJsonFile JobLog
+  } deriving Generic
+
+
+newtype CSVAPI mode = CSVAPI
+  { updateListCSVEp :: mode :- Summary "Update List (legacy v3 CSV)"
+                            :> "lists"
+                            :> Capture "listId" ListId
+                            :> "csv"
+                            :> "add"
+                            :> "form"
+                            :> "async"
+                            :> AsyncJobs JobLog '[FormUrlEncoded] WithTextFile JobLog
+  } deriving Generic

src/Gargantext/API/Routes/Named/Metrics.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.Metrics (
+  -- * Routes types
+
+    TreeAPI(..)
+  , ScatterAPI(..)
+  , ChartAPI(..)
+  , PieAPI(..)
+  ) where
+
+import Data.Text (Text)
+import Data.Time
+import Data.Vector
+import GHC.Generics
+import Gargantext.API.HashedResponse
+import Gargantext.API.Ngrams.NgramsTree
+import Gargantext.API.Ngrams.Types
+import Gargantext.Core.Types.Main
+import Gargantext.Core.Types.Query (Limit)
+import Gargantext.Core.Viz.Types
+import Gargantext.Database.Admin.Types.Metrics
+import Gargantext.Database.Admin.Types.Node
+import Servant
+
+
+data TreeAPI mode = TreeAPI
+  { treeChartEp       :: mode :- Summary " Tree API"
+                              :> QueryParam "from" UTCTime
+                              :> QueryParam "to"   UTCTime
+                              :> QueryParam  "list"       ListId
+                              :> QueryParamR "ngramsType" TabType
+                              :> QueryParamR "listType"   ListType
+                              :> Get '[JSON] (HashedResponse (ChartMetrics (Vector NgramsTree)))
+  , treeChartUpdateEp :: mode :- Summary "Tree Chart update"
+                              :> QueryParam  "list"       ListId
+                              :> QueryParamR "ngramsType" TabType
+                              :> QueryParamR "listType"   ListType
+                              :> Post '[JSON] ()
+  , treeHashEp        :: mode :- "hash"
+                              :> Summary "Tree Hash"
+                              :> QueryParam  "list"       ListId
+                              :> QueryParamR "ngramsType" TabType
+                              :> QueryParamR "listType"   ListType
+                              :> Get '[JSON] Text
+  } deriving Generic
+
+
+data ScatterAPI mode = ScatterAPI
+  { sepGenEp :: mode :- Summary "SepGen IncExc metrics"
+                     :> QueryParam  "list"       ListId
+                     :> QueryParamR "ngramsType" TabType
+                     :> QueryParam  "limit"      Limit
+                     :> Get '[JSON] (HashedResponse Metrics)
+  , scatterUpdateEp :: mode :- Summary "Scatter update"
+                            :> QueryParam  "list"       ListId
+                            :> QueryParamR "ngramsType" TabType
+                            :> QueryParam  "limit"      Limit
+                            :> Post '[JSON] ()
+  , scatterHashEp :: mode :- "hash"
+                          :> Summary "Scatter Hash"
+                          :> QueryParam  "list"       ListId
+                          :> QueryParamR "ngramsType" TabType
+                          :> Get '[JSON] Text
+  } deriving Generic
+
+
+data PieAPI mode = PieAPI
+  { getPieChartEp :: mode :- Summary "Pie Chart"
+                          :> QueryParam "from" UTCTime
+                          :> QueryParam "to"   UTCTime
+                          :> QueryParam  "list"       ListId
+                          :> QueryParamR "ngramsType" TabType
+                          :> Get '[JSON] (HashedResponse (ChartMetrics Histo))
+  , pieChartUpdateEp :: mode :- Summary "Pie Chart update"
+                             :> QueryParam  "list"       ListId
+                             :> QueryParamR "ngramsType" TabType
+                             :> QueryParam  "limit"      Limit
+                             :> Post '[JSON] ()
+  , pieHashEp :: mode :- "hash"
+                      :> Summary "Pie Hash"
+                      :> QueryParam  "list"       ListId
+                      :> QueryParamR "ngramsType" TabType
+                      :> Get '[JSON] Text
+  } deriving Generic
+
+
+data ChartAPI mode = ChartAPI
+  { getChartEp :: mode :- Summary " Chart API"
+                       :> QueryParam "from" UTCTime
+                       :> QueryParam "to"   UTCTime
+                       :> QueryParam  "list"       ListId
+                       :> QueryParamR "ngramsType" TabType
+                       :> Get '[JSON] (HashedResponse (ChartMetrics Histo))
+  , updateChartEp :: mode :- Summary "Chart update"
+                          :> QueryParam  "list"       ListId
+                          :> QueryParamR "ngramsType" TabType
+                          :> QueryParam  "limit"      Limit
+                          :> Post '[JSON] ()
+  , chartHashEp :: mode :- "hash"
+                        :> Summary "Chart Hash"
+                        :> QueryParam  "list"       ListId
+                        :> QueryParamR "ngramsType" TabType
+                        :> Get '[JSON] Text
+  } deriving Generic
+

src/Gargantext/API/Routes/Named/Private.hs

+
+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.Private (
+  -- * Routes types
+    GargPrivateAPI
+  , GargPrivateAPI'(..)
+  , GargAdminAPI(..)
+  , NodeEndpoint(..)
+  , MembersAPI(..)
+  ) where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Auth.PolicyCheck
+import Gargantext.API.Routes.Named.Contact
+import Gargantext.API.Routes.Named.Context
+import Gargantext.API.Routes.Named.Corpus
+import Gargantext.API.Routes.Named.Count
+import Gargantext.API.Routes.Named.Document
+import Gargantext.API.Routes.Named.Metrics
+import Gargantext.API.Routes.Named.Node
+import Gargantext.API.Routes.Named.List qualified as List
+import Gargantext.API.Routes.Named.Share
+import Gargantext.API.Routes.Named.Tree
+import Gargantext.API.Routes.Named.Table
+import Gargantext.API.Routes.Named.Viz
+import Gargantext.Database.Admin.Types.Hyperdata.Any
+import Gargantext.Database.Admin.Types.Hyperdata.Corpus
+import Gargantext.Database.Admin.Types.Node
+import Servant.API
+import Servant.Auth qualified as SA
+
+
+type MkProtectedAPI private = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser :> private
+
+
+newtype GargPrivateAPI mode = GargPrivateAPI
+  { mkPrivateAPI :: mode :- MkProtectedAPI (NamedRoutes GargPrivateAPI')
+  } deriving Generic
+
+
+data GargPrivateAPI' mode = GargPrivateAPI'
+  { gargAdminAPI :: mode :- NamedRoutes GargAdminAPI
+  , nodeEp       :: mode :- NamedRoutes NodeEndpoint
+  , contextEp    :: mode :- "context"  :> Summary "Node endpoint"
+                                       :> Capture "node_id" ContextId
+                                       :> NamedRoutes (ContextAPI HyperdataAny)
+  , corpusNodeAPI :: mode :- "corpus"  :> Summary "Corpus endpoint"
+                                       :> Capture "corpus_id" CorpusId
+                                       :> NamedRoutes (NodeAPI HyperdataCorpus)
+  , corpusNodeNodeAPI :: mode :- "corpus" :> Summary "Corpus endpoint"
+                                          :> Capture "node1_id" NodeId
+                                          :> "document"
+                                          :> Capture "node2_id" NodeId
+                                          :> NamedRoutes (NodeNodeAPI HyperdataAny)
+  , corpusExportAPI :: mode :- "corpus" :> Capture "node_id" CorpusId
+                                        :> NamedRoutes CorpusExportAPI
+  , annuaireEp :: mode :- "annuaire" :> Summary "Annuaire endpoint"
+                                     :> Capture "annuaire_id" AnnuaireId
+                                     :> NamedRoutes (NodeAPI HyperdataAnnuaire)
+  , contactAPI :: mode :- "annuaire" :> Summary "Contact endpoint"
+                                     :> Capture "annuaire_id" NodeId
+                                     :> NamedRoutes ContactAPI
+  , tableNgramsAPI :: mode :- "document" :> Summary "Document endpoint"
+                                         :> Capture "doc_id" DocId
+                                         :> "ngrams"
+                                         :> NamedRoutes TableNgramsAPI
+  , documentExportAPI :: mode :- "texts" :> Capture "node_id" DocId
+                                         :> NamedRoutes DocumentExportAPI
+  , countAPI :: mode :- "count" :> Summary "Count endpoint"
+                                :> ReqBody '[JSON] Query
+                                :> NamedRoutes CountAPI
+  , graphAPI :: mode :- "graph" :> Summary "Graph endpoint"
+                                :> Capture "graph_id" NodeId
+                                :> NamedRoutes GraphAPI
+  , treeAPI :: mode :- "tree" :> Summary "Tree endpoint"
+                              :> Capture "tree_id" NodeId
+                              :> PolicyChecked (NamedRoutes TreeAPI)
+  , treeFlatAPI :: mode :- "treeflat" :> Summary "Flat tree endpoint"
+                                      :> Capture "tree_id" NodeId
+                                      :> NamedRoutes TreeFlatAPI
+  , membersAPI     :: mode :- "members" :> Summary "Team node members" :> NamedRoutes MembersAPI
+  , addWithFormEp  :: mode :- NamedRoutes AddWithForm
+  , addWithQueryEp :: mode :- NamedRoutes AddWithQuery
+  , listGetAPI     :: mode :- NamedRoutes List.GETAPI
+  , listJsonAPI    :: mode :- NamedRoutes List.JSONAPI
+  , listCsvAPI     :: mode :- NamedRoutes List.CSVAPI
+  , shareUrlEp     :: mode :- "shareurl" :> NamedRoutes ShareURL
+  } deriving Generic
+
+
+data GargAdminAPI mode = GargAdminAPI
+  { rootsEp       :: mode :- "user"  :> Summary "First user endpoint" :> NamedRoutes Roots
+  , adminNodesAPI :: mode :- "nodes" :> Summary "Nodes endpoint"
+                                     :> ReqBody '[JSON] [NodeId]
+                                     :> NamedRoutes NodesAPI
+  } deriving Generic
+
+
+data NodeEndpoint mode = NodeEndpoint
+  { nodeEndpointAPI :: mode :- "node" :> Summary "Node endpoint"
+                                      :> Capture "node_id" NodeId
+                                      :> NamedRoutes (NodeAPI HyperdataAny)
+  } deriving Generic
+
+
+data MembersAPI mode = MembersAPI
+  { getMembersEp :: mode :- Get '[JSON] [Text]
+  }
+

src/Gargantext/API/Routes/Named/Public.hs

+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE TypeOperators #-}
+module Gargantext.API.Routes.Named.Public where
+
+import GHC.Generics
+import Gargantext.API.Public qualified as Public
+import Gargantext.API.Routes.Named.File
+import Gargantext.Database.Admin.Types.Node ( NodeId )
+import Servant.API
+
+data GargPublicAPI mode = GargPublicAPI
+  { publicHomeAPI :: mode :- NamedRoutes HomeAPI
+  , publicNodeAPI :: mode :- NamedRoutes NodeAPI
+  } deriving Generic
+
+data HomeAPI mode = HomeAPI
+  { homeEp :: mode :- Summary "Public Home API" :> Get '[JSON] [Public.PublicData]
+  } deriving Generic
+
+data NodeAPI mode = NodeAPI
+  { nodeEp :: mode :- Summary "Public Node API" :> Capture "node" NodeId :> "file" :> NamedRoutes FileAPI
+  } deriving Generic

src/Gargantext/API/Routes/Named/Search.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+module Gargantext.API.Routes.Named.Search (
+  -- * Routes types
+    SearchAPI(..)
+
+  -- * API types (appears in the routes)
+  , SearchType(..)
+  , SearchQuery(..)
+  , SearchResult(..)
+  , SearchResultTypes(..)
+  ) where
+
+
+import Data.Aeson
+import Data.Swagger
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.Core.Text.Corpus.Query (RawQuery (..))
+import Gargantext.Core.Types.Query (Limit, Offset)
+import Gargantext.Core.Types.Search
+import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
+import Gargantext.Database.Query.Facet
+import Prelude
+import Servant
+import Test.QuickCheck
+
+
+-- TODO-ACCESS: CanSearch? or is it part of CanGetNode
+-- TODO-EVENTS: No event, this is a read-only query.
+data SearchAPI results mode = SearchAPI
+  { searchEp :: mode :- Summary "Search endpoint"
+                     :> ReqBody '[JSON] SearchQuery
+                     :> QueryParam "offset" Offset
+                     :> QueryParam "limit"  Limit
+                     :> QueryParam "order"  OrderBy
+                     :> Post '[JSON] results
+  } deriving Generic
+
+
+data SearchType = SearchDoc | SearchContact | SearchDocWithNgrams
+  deriving Generic
+
+
+data SearchQuery = SearchQuery
+  { query    :: !RawQuery
+  , expected :: !SearchType
+  } deriving Generic
+
+
+newtype SearchResult =
+  SearchResult { result :: SearchResultTypes }
+    deriving Generic
+
+
+data SearchResultTypes =
+    SearchResultDoc { docs     :: ![Row] }
+  | SearchResultContact  { contacts :: ![Row] }
+  | SearchNoResult      { message  :: !Text }
+  deriving Generic
+
+
+--
+-- instances
+--
+
+instance FromJSON SearchResult where
+  parseJSON = genericParseJSON defaultOptions
+
+instance ToJSON SearchResult where
+  toJSON = genericToJSON defaultOptions
+
+instance ToSchema SearchResult
+
+instance Arbitrary SearchResult where
+  arbitrary = SearchResult <$> arbitrary
+
+instance FromJSON SearchResultTypes where
+  parseJSON = genericParseJSON (defaultOptions { sumEncoding = defaultTaggedObject })
+
+instance ToJSON SearchResultTypes where
+  toJSON = genericToJSON (defaultOptions { sumEncoding = defaultTaggedObject })
+
+instance Arbitrary SearchResultTypes where
+  arbitrary = do
+    srd <- SearchResultDoc     <$> arbitrary
+    src <- SearchResultContact <$> arbitrary
+    srn <- pure $ SearchNoResult "No result because.."
+    elements [srd, src, srn]
+
+instance ToSchema SearchResultTypes where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "")
+
+instance FromJSON SearchQuery where
+  parseJSON = genericParseJSON defaultOptions
+
+instance ToJSON SearchQuery where
+  toJSON = genericToJSON defaultOptions
+
+instance ToSchema SearchQuery
+
+instance Arbitrary SearchQuery where
+  arbitrary = elements [SearchQuery (RawQuery "electrodes") SearchDoc]
+
+instance FromJSON SearchType where
+  parseJSON = genericParseJSON (defaultOptions { sumEncoding = ObjectWithSingleField })
+
+instance ToJSON SearchType where
+  toJSON = genericToJSON (defaultOptions { sumEncoding = ObjectWithSingleField })
+
+instance ToSchema SearchType
+
+instance Arbitrary SearchType where
+  arbitrary = elements [SearchDoc, SearchContact]

src/Gargantext/API/Routes/Named/Share.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.Share (
+  -- * Routes types
+    ShareNode(..)
+  , Unpublish(..)
+  , ShareURL(..)
+
+  -- * API types (which appears in the routes)
+  , ShareNodeParams(..)
+  ) where
+
+import Data.Aeson
+import Data.Swagger
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Utils.Aeson qualified as GUA
+import Prelude
+import Servant
+import Test.QuickCheck
+
+newtype ShareURL mode = ShareURL
+  { shareUrlEp :: mode :- Summary "Fetch URL for sharing a node"
+                       :> QueryParam "type" NodeType
+                       :> QueryParam "id" NodeId
+                       :> Get '[JSON] Text
+  } deriving Generic
+
+
+newtype ShareNode mode = ShareNode
+  { shareNodeEp :: mode :- Summary " Share Node with username"
+                        :> ReqBody '[JSON] ShareNodeParams
+                        :> Post    '[JSON] Int
+  } deriving Generic
+
+
+newtype Unpublish mode = Unpublish
+  { unpublishEp :: mode :- Summary " Unpublish Node" :> Capture "node_id" NodeId :> Put '[JSON] Int
+  } deriving Generic
+
+--
+-- API Types
+--
+data ShareNodeParams = ShareTeamParams   { username :: Text  }
+                     | SharePublicParams { node_id  :: NodeId }
+  deriving (Generic)
+
+--
+-- Instances
+--
+
+-- TODO unPrefix "pn_" FromJSON, ToJSON, ToSchema, adapt frontend.
+instance FromJSON  ShareNodeParams where
+  parseJSON = genericParseJSON (defaultOptions { sumEncoding = GUA.defaultTaggedObject })
+instance ToJSON    ShareNodeParams where
+  toJSON = genericToJSON (defaultOptions { sumEncoding = GUA.defaultTaggedObject })
+instance ToSchema  ShareNodeParams
+instance Arbitrary ShareNodeParams where
+  arbitrary = elements [ ShareTeamParams "user1"
+                       , SharePublicParams (UnsafeMkNodeId 1)
+                       ]

src/Gargantext/API/Routes/Named/Table.hs

+{-# LANGUAGE DeriveGeneric   #-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE TypeOperators   #-}
+
+module Gargantext.API.Routes.Named.Table (
+  -- * Routes types
+    TableAPI(..)
+  , TableNgramsAPI(..)
+  , TableNgramsApiGet(..)
+  , TableNgramsApiPut(..)
+  , RecomputeScoresNgramsApiGet(..)
+  , TableNgramsApiGetVersion(..)
+  , TableNgramsAsyncAPI(..)
+
+  -- * API types (appears in the routes)
+
+  , TableQuery(..)
+  , FacetTableResult
+  ) where
+
+import Data.Aeson.TH
+import Data.Swagger
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.HashedResponse
+import Gargantext.API.Ngrams.Types (TabType(..), UpdateTableNgramsCharts, Version, QueryParamR, Versioned, VersionedWithCount, NgramsTable, NgramsTablePatch)
+import Gargantext.Core.Text.Corpus.Query (RawQuery)
+import Gargantext.Core.Types (TableResult(..))
+import Gargantext.Core.Types.Main (ListType)
+import Gargantext.Core.Types.Query
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Database.Admin.Types.Node (ListId)
+import Gargantext.Database.Query.Facet.Types
+import Prelude
+import Servant
+import Test.QuickCheck
+
+
+data TableAPI mode = TableAPI
+  { getTableEp   :: mode :- Summary "Table API"
+                         :> QueryParam "tabType" TabType
+                         :> QueryParam "limit" Limit
+                         :> QueryParam "offset" Offset
+                         :> QueryParam "orderBy" OrderBy
+                         :> QueryParam "query" RawQuery
+                         :> QueryParam "year" Text
+                         :> Get    '[JSON] (HashedResponse FacetTableResult)
+  , postTableEp :: mode :- Summary "Table API (POST)"
+                        :> ReqBody '[JSON] TableQuery
+                        :> Post    '[JSON] FacetTableResult
+  , hashTableEp :: mode :- "hash"
+                        :> Summary "Hash Table"
+                        :> QueryParam "tabType" TabType
+                        :> Get '[JSON] Text
+  } deriving Generic
+
+
+data TableNgramsAPI mode = TableNgramsAPI
+  { tableNgramsGetAPI       :: mode :- NamedRoutes TableNgramsApiGet
+  , tableNgramsPutAPI       :: mode :- NamedRoutes TableNgramsApiPut
+  , recomputeScoresEp       :: mode :- NamedRoutes RecomputeScoresNgramsApiGet
+  , tableNgramsGetVersionEp :: mode :- "version" :> NamedRoutes TableNgramsApiGetVersion
+  , tableNgramsAsyncAPI     :: mode :- NamedRoutes TableNgramsAsyncAPI
+  } deriving Generic
+
+
+data TableNgramsApiGet mode = TableNgramsApiGet
+  { getNgramsTableEp :: mode :- Summary " Table Ngrams API Get"
+                             :> QueryParamR "ngramsType"  TabType
+                             :> QueryParamR "list"        ListId
+                             :> QueryParamR "limit"       Limit
+                             :> QueryParam  "offset"      Offset
+                             :> QueryParam  "listType"    ListType
+                             :> QueryParam  "minTermSize" MinSize
+                             :> QueryParam  "maxTermSize" MaxSize
+                             :> QueryParam  "orderBy"     OrderBy
+                             :> QueryParam  "search"      Text
+                             :> Get    '[JSON] (VersionedWithCount NgramsTable)
+  } deriving Generic
+
+
+data TableNgramsApiPut mode = TableNgramsApiPut
+  { putNgramsTableEp :: mode :- Summary " Table Ngrams API Change"
+                             :> QueryParamR "ngramsType" TabType
+                             :> QueryParamR "list"       ListId
+                             :> ReqBody '[JSON] (Versioned NgramsTablePatch)
+                             :> Put     '[JSON] (Versioned NgramsTablePatch)
+  } deriving Generic
+
+
+data RecomputeScoresNgramsApiGet mode = RecomputeScoresNgramsApiGet
+  { recomputeNgramsEp :: mode :- Summary " Recompute scores for ngrams table"
+                              :> QueryParamR "ngramsType"  TabType
+                              :> QueryParamR "list"        ListId
+                              :> "recompute" :> Post '[JSON] Int
+  } deriving Generic
+
+
+data TableNgramsApiGetVersion mode = TableNgramsApiGetVersion
+  { getTableNgramsVersion :: mode :- Summary " Table Ngrams API Get Version"
+                                  :> QueryParamR "ngramsType"  TabType
+                                  :> QueryParamR "list"        ListId
+                                  :> Get    '[JSON] Version
+  } deriving Generic
+
+
+data TableNgramsAsyncAPI mode = TableNgramsAsyncAPI
+  { updateTableNgramsChartsEp :: mode :- Summary "Table Ngrams Async API"
+                                      :> "async"
+                                      :> "charts"
+                                      :> "update"
+                                      :> AsyncJobs JobLog '[JSON] UpdateTableNgramsCharts JobLog
+  } deriving Generic
+
+
+data TableQuery = TableQuery
+  { tq_offset       :: Offset
+  , tq_limit        :: Limit
+  , tq_orderBy      :: OrderBy
+  , tq_view         :: TabType
+  , tq_query        :: RawQuery
+  } deriving Generic
+
+type FacetTableResult = TableResult FacetDoc
+
+--
+-- instances
+--
+
+$(deriveJSON (unPrefix "tq_") ''TableQuery)
+
+instance ToSchema TableQuery where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "tq_")
+
+instance Arbitrary TableQuery where
+  arbitrary = elements [TableQuery { tq_offset = 0
+                                   , tq_limit = 10
+                                   , tq_orderBy = DateAsc
+                                   , tq_view = Docs
+                                   , tq_query = "electrodes" }]

src/Gargantext/API/Routes/Named/Tree.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.Tree (
+  -- * Routes types
+    NodeTreeAPI(..)
+  , TreeFlatAPI(..)
+  ) where
+
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.Core.Types.Main
+import Gargantext.Database.Admin.Types.Node
+import Servant
+
+data NodeTreeAPI mode = NodeTreeAPI
+  { nodeTreeEp   :: mode :- QueryParams "type" NodeType :> Get '[JSON] (Tree NodeTree)
+  , firstLevelEp :: mode :- "first-level" :> QueryParams "type" NodeType :> Get '[JSON] (Tree NodeTree)
+  } deriving Generic
+
+
+data TreeFlatAPI mode = TreeFlatAPI
+  { getNodesEp :: mode :- QueryParams "type" NodeType :> QueryParam "query" Text :> Get '[JSON] [NodeTree]
+  } deriving Generic

src/Gargantext/API/Routes/Named/Viz.hs

+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Gargantext.API.Routes.Named.Viz (
+  -- * Routes types
+    PhyloAPI(..)
+  , GetPhylo(..)
+  , PostPhylo(..)
+  , GraphAPI(..)
+  , GraphAsyncAPI(..)
+  , GraphVersionsAPI(..)
+
+  -- * API types (appears in the routes)
+  , PhyloData(..)
+  , GraphVersions(..)
+  ) where
+
+import Data.Aeson
+import Data.Swagger
+import Data.Text (Text)
+import GHC.Generics
+import Gargantext.API.Admin.Orchestrator.Types ( JobLog )
+import Gargantext.Core.Types
+import Gargantext.Core.Types.Phylo
+import Gargantext.Core.Viz.Graph.Types
+import Gargantext.Core.Viz.LegacyPhylo (Level)
+import Gargantext.Core.Viz.Phylo
+import Gargantext.Core.Viz.Phylo.Legacy.LegacyMain (MinSizeBranch)
+import Prelude
+import Servant
+import Servant.Job.Async (AsyncJobsAPI)
+import Servant.XML.Conduit (XML)
+import Test.QuickCheck
+
+
+data PhyloAPI mode = PhyloAPI
+  { getPhyloEp  :: mode :- Summary "Phylo API" :> NamedRoutes GetPhylo
+  , postPhyloEp :: mode :- NamedRoutes PostPhylo
+  } deriving Generic
+
+
+newtype GetPhylo mode = GetPhylo
+  { getPhyloDataEp :: mode :- QueryParam "listId"      ListId
+                           :> QueryParam "level"       Level
+                           :> QueryParam "minSizeBranch" MinSizeBranch
+                           :> Get '[JSON] PhyloData
+  } deriving Generic
+
+
+newtype PostPhylo mode = PostPhylo
+  { postPhyloByListIdEp :: mode :- QueryParam "listId" ListId :> (Post '[JSON] NodeId)
+  } deriving Generic
+
+
+data GraphAPI mode = GraphAPI
+  { getGraphEp       :: mode :- Get '[JSON] HyperdataGraphAPI
+  , getGraphAsyncEp  :: mode :- "async" :> NamedRoutes GraphAsyncAPI
+  , cloneGraphEp     :: mode :- "clone" :> ReqBody '[JSON] HyperdataGraphAPI :> Post '[JSON] NodeId
+  , gexfEp           :: mode :- "gexf" :> Get '[XML] (Headers '[Servant.Header "Content-Disposition" Text] Graph)
+  , graphVersionsAPI :: mode :- "versions" :> NamedRoutes GraphVersionsAPI
+  } deriving Generic
+
+
+newtype GraphAsyncAPI mode = GraphAsyncAPI
+  { recomputeGraphEp :: mode :- Summary "Recompute graph"
+                             :> "recompute"
+                             :> AsyncJobsAPI JobLog () JobLog
+  } deriving Generic
+
+
+data GraphVersionsAPI mode = GraphVersionsAPI
+  { getGraphVersionsEp :: mode :- Summary "Graph versions" :> Get '[JSON] GraphVersions
+  , recomputeGraphVersionEp :: mode :- Summary "Recompute graph version" :> Post '[JSON] Graph
+  } deriving Generic
+
+
+data PhyloData = PhyloData { pd_corpusId :: NodeId
+                           , pd_listId   :: NodeId
+                           , pd_data     :: GraphData
+                           , pd_config   :: PhyloConfig
+                           }
+  deriving (Generic, Show, Eq)
+
+
+data GraphVersions = GraphVersions
+  { gv_graph :: Maybe Int
+  , gv_repo  :: Int
+  } deriving (Show, Generic)
+
+
+--
+-- instances
+--
+
+instance ToJSON PhyloData where
+  toJSON PhyloData{..} =
+    object [
+      "pd_corpusId" .= toJSON pd_corpusId
+    , "pd_listId"   .= toJSON pd_listId
+    , "pd_data"     .= toJSON pd_data
+    , "pd_config"   .= toJSON pd_config
+    ]
+
+instance FromJSON PhyloData where
+  parseJSON = withObject "PhyloData" $ \o -> do
+    pd_corpusId <- o .: "pd_corpusId"
+    pd_listId   <- o .: "pd_listId"
+    pd_data     <- o .: "pd_data"
+    pd_config   <- o .: "pd_config"
+    pure $ PhyloData{..}
+
+instance Arbitrary PhyloData where
+  arbitrary = PhyloData <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
+
+instance ToSchema PhyloData
+
+instance FromJSON GraphVersions
+
+instance ToJSON GraphVersions
+
+instance ToSchema GraphVersions
+

src/Gargantext/API/Routes/Types.hs

+{-# LANGUAGE TypeFamilies         #-}
+{-# LANGUAGE ScopedTypeVariables  #-}
+module Gargantext.API.Routes.Types where
+
+import Data.List qualified as L
+import Data.Proxy
+import Gargantext.API.Errors
+import Prelude
+import Servant.Ekg
+import Servant.Server
+import Servant.Server.Internal.DelayedIO
+import Servant.Server.Internal.Delayed
+import Network.Wai
+
+data WithCustomErrorScheme a
+
+instance (HasServer subApi ctx) => HasServer (WithCustomErrorScheme subApi) ctx where
+  type ServerT (WithCustomErrorScheme subApi) m = GargErrorScheme -> ServerT subApi m
+  hoistServerWithContext _ pc nt s = hoistServerWithContext (Proxy :: Proxy subApi) pc nt . s
+  route Proxy ctx d = route (Proxy :: Proxy subApi) ctx (d `addHeaderCheck` getErrorScheme)
+    where
+      getErrorScheme :: DelayedIO GargErrorScheme
+      getErrorScheme = withRequest $ \rq -> do
+        let hdrs = requestHeaders rq
+            in case L.lookup "X-Garg-Error-Scheme" hdrs of
+                 Nothing     -> pure GES_old
+                 Just "new"  -> pure GES_new
+                 Just _      -> pure GES_old
+
+instance HasEndpoint sub => HasEndpoint (WithCustomErrorScheme sub) where
+  getEndpoint _ = getEndpoint (Proxy :: Proxy sub)
+  enumerateEndpoints _ = enumerateEndpoints (Proxy :: Proxy sub)

src/Gargantext/Core/Methods/Similarities/Accelerate/Distributional.hs

@@ -204,7 +204,8 @@ logDistributional' n m' = trace ("logDistributional'") result
     -- m_{i,j} = 0 if n_{i,j} = 0 or i = j, 
     -- m_{i,j} = log(to * n_{i,j} / s_{i,j}) otherwise.
     mi = (.*) (matrixEye n)
-              (map (lift1 (\x -> let x' = x * to in cond (x' < 1) 0 (log x'))) ((./) m ss))
+              (map (lift1 (\x -> let x' = x * to in cond (x' < 0.5) 0 (log x'))) ((./) m ss))
+              -- (map (lift1 (\x -> let x' = x * to in cond (x' < 1) 0 (log x'))) ((./) m ss))
     -- mi_nnz :: Int
     -- mi_nnz = flip indexArray Z . run $
     --   foldAll (+) 0 $ map (\a -> ifThenElse (abs a < 10^(-6 :: Exp Int)) 0 1) mi

src/Gargantext/Core/Text.hs

@@ -66,7 +66,7 @@ class Collage sup inf where
 
 instance Collage Texte Paragraphe where
   dec (Texte t) = map Paragraphe $ DT.splitOn "\n" t
-  inc           = Texte . DT.intercalate "\n" . map (\(Paragraphe t) -> t)
+  inc           = Texte . DT.unlines . map (\(Paragraphe t) -> t)
 
 instance Collage Paragraphe Phrase where
   dec (Paragraphe t) = map Phrase $ sentences t
@@ -78,7 +78,7 @@ instance Collage Phrase MultiTerme where
 
 instance Collage MultiTerme Mot where
   dec (MultiTerme mt) = map Mot $ DT.words mt
-  inc                 = MultiTerme . DT.intercalate " " . map (\(Mot m) -> m)
+  inc                 = MultiTerme . DT.unwords . map (\(Mot m) -> m)
 
 -------------------------------------------------------------------
 -- Contexts of text
@@ -92,7 +92,7 @@ isCharStop :: Char -> Bool
 isCharStop c = c `elem` ['.','?','!']
 
 unsentences :: [Text] -> Text
-unsentences txts = DT.intercalate " " txts
+unsentences txts = DT.unwords txts
 
 -- | Ngrams size
 size :: Text -> Int

src/Gargantext/Core/Text/Corpus/API/Isidore.hs

@@ -71,7 +71,7 @@ isidoreToDoc lang (IsidoreDoc t a d u s as) = do
     langText :: LangText -> Text
     langText (LangText _l t1) = t1
     langText (OnlyText t2   ) = t2
-    langText (ArrayText ts  ) = Text.intercalate " " $ map langText ts
+    langText (ArrayText ts  ) = Text.unwords $ map langText ts
 
   let mDateS = maybe (Just $ Text.pack $ show Defaults.year) Just d
   let (utcTime, (pub_year, pub_month, pub_day)) = Date.mDateSplit mDateS

src/Gargantext/Core/Text/Corpus/Parsers/FrameWrite.hs

@@ -247,7 +247,7 @@ text2titleParagraphs n = catMaybes
     n' = n + (round $ (fromIntegral n) / (2 :: Double))
 
 doTitle :: [Text] -> Maybe (Text, Text)
-doTitle (t:ts) = Just (t, DT.intercalate " " ts)
+doTitle (t:ts) = Just (t, DT.unwords ts)
 doTitle [] = Nothing
 
 

src/Gargantext/Core/Text/List.hs

@@ -58,6 +58,11 @@ isStopTerm (StopSize n) x = Text.length x < n || any isStopChar (Text.unpack x)
 -}
 
 
+-- | Good value from users' requests and anthropological analysis
+goodMapListSize :: Int
+goodMapListSize = 350
+    
+
 -- | TODO improve grouping functions of Authors, Sources, Institutes..
 buildNgramsLists :: ( HasNodeStory env err m
                     , HasNLPServer env
@@ -71,7 +76,7 @@ buildNgramsLists :: ( HasNodeStory env err m
                  -> GroupParams
                  -> m (Map NgramsType [NgramsElement])
 buildNgramsLists user uCid mCid mfslw gp = do
-  ngTerms     <- buildNgramsTermsList user uCid mCid mfslw gp (NgramsTerms, MapListSize 350)
+  ngTerms     <- buildNgramsTermsList user uCid mCid mfslw gp (NgramsTerms, MapListSize goodMapListSize)
   othersTerms <- mapM (buildNgramsOthersList user uCid mfslw GroupIdentity)
                       [ (Authors   , MapListSize 9, MaxListSize 1000)
                       , (Sources   , MapListSize 9, MaxListSize 1000)
@@ -179,23 +184,24 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
                                     )
   -- printDebug "[buildNgramsTermsList: Flow Social List / end]" nt
 
-  let !ngramsKeys = HashSet.fromList
-                  $ List.take mapListSize
-                  $ HashSet.toList
-                  $ HashMap.keysSet allTerms
+  let !allKeys = HashMap.keysSet allTerms
 
   -- printDebug "[buildNgramsTermsList: ngramsKeys]" (HashSet.size ngramsKeys)
 
-  !groupParams' <- getGroupParams groupParams (HashSet.map (text2ngrams . unNgramsTerm) ngramsKeys)
+  !groupParams' <- getGroupParams groupParams (HashSet.map (text2ngrams . unNgramsTerm) allKeys)
 
   let
-    !socialLists_Stemmed = addScoreStem groupParams' ngramsKeys socialLists
+    !socialLists_Stemmed = addScoreStem groupParams' allKeys socialLists
     !groupedWithList = toGroupedTree socialLists_Stemmed allTerms
     !(stopTerms, candidateTerms) = HashMap.partition ((== Just StopTerm) . viewListType)
+                                 $ HashMap.fromList
+                                 $ List.take mapListSize
+                                 $ HashMap.toList
                                  $ HashMap.filter (\g -> view gts'_score g > 1)
                                  $ view flc_scores groupedWithList
 
-    !(groupedMono, groupedMult)  = HashMap.partitionWithKey (\(NgramsTerm t) _v -> size t < 2) candidateTerms
+    -- | Split candidateTerms into mono-terms and multi-terms.
+    !(groupedMono, groupedMult) = HashMap.partitionWithKey (\(NgramsTerm t) _v -> size t < 2) candidateTerms
 
   -- void $ panicTrace $ "groupedWithList: " <> show groupedWithList
 
@@ -211,6 +217,7 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
     !monoSize = 0.4  :: Double
     !multSize = 1 - monoSize
 
+    -- | Splits given hashmap into 2 pieces, based on score
     splitAt' n' ns = both (HashMap.fromListWith (<>))
                       $ List.splitAt (round $ n' * listSizeGlobal)
                       $ List.sortOn (viewScore . snd)
@@ -254,8 +261,7 @@ buildNgramsTermsList user uCid mCid mfslw groupParams (nt, MapListSize mapListSi
                            ]
           where
             mapStemNodeIds = HashMap.toList
-                           $ HashMap.map viewScores
-                           $ groupedTreeScores_SetNodeId
+                           $ HashMap.map viewScores groupedTreeScores_SetNodeId
   let
     -- computing scores
     mapScores f = HashMap.fromList

src/Gargantext/Core/Text/List/Group/WithStem.hs

@@ -69,7 +69,7 @@ groupWith :: GroupParams
 groupWith GroupIdentity  t = identity t
 groupWith (GroupParams { unGroupParams_lang = l }) t =
                     NgramsTerm
-                  $ Text.intercalate " "
+                  $ Text.unwords
                   $ map (stem l PorterAlgorithm)
                   -- . take n
                   $ List.sort

src/Gargantext/Core/Text/Metrics/TFICF.hs

@@ -14,14 +14,15 @@ TFICF is a generalization of [TFIDF](https://en.wikipedia.org/wiki/Tf%E2%80%93id
 -}
 
 
-module Gargantext.Core.Text.Metrics.TFICF ( TFICF
-                                     , TficfContext(..)
-                                     , Total(..)
-                                     , Count(..)
-                                     , tficf
-                                     , sortTficf
-                                     )
-  where
+module Gargantext.Core.Text.Metrics.TFICF
+ ( TFICF
+ , TficfContext(..)
+ , Total(..)
+ , Count(..)
+ , tficf
+ , sortTficf
+ )
+where
 
 import Data.List qualified as List
 import Data.Map.Strict (toList)
@@ -34,12 +35,19 @@ path = "[G.T.Metrics.TFICF]"
 
 type TFICF = Double
 
+-- https://www.researchgate.net/publication/221226686_TF-ICF_A_New_Term_Weighting_Scheme_for_Clustering_Dynamic_Data_Streams
+-- TficfSupra n m
+--  - m is the total number of documents in the corpus
+--  - n is the number of documents, where given term occured more than once
+-- TficfInfra n m
+--  - 
+    
 data TficfContext n m = TficfInfra n m
                       | TficfSupra n m
   deriving (Show)
 
-data Total = Total {unTotal :: !Double}
-data Count = Count {unCount :: !Double}
+newtype Total = Total { unTotal :: Double }
+newtype Count = Count { unCount :: Double }
 
 tficf :: TficfContext Count Total
       -> TficfContext Count Total
@@ -50,7 +58,11 @@ tficf (TficfInfra (Count ic) (Total it) )
             | otherwise                        = panicTrace
                                                $ "[ERR]"
                                                <> path
-                                               <> " Frequency impossible"
+                                               <> " Frequency impossible: "
+                                               <> "ic = " <> show ic
+                                               <> ", it = " <> show it
+                                               <> ", sc = " <> show sc
+                                               <> ", st = " <> show st
 tficf _ _ = panicTrace $ "[ERR]" <> path <> "Undefined for these contexts"
 
 

src/Gargantext/Core/Text/Ngrams.hs

@@ -92,7 +92,8 @@ instance Hashable Ngrams
 
 makeLenses ''Ngrams
 instance PGS.ToRow Ngrams where
-  toRow (UnsafeNgrams t s) = [toField t, toField s]
+  toRow (UnsafeNgrams { .. }) = [ toField _ngramsTerms
+                                , toField _ngramsSize ]
 
 ------------------------------------------------------------------------
 -------------------------------------------------------------------------

src/Gargantext/Core/Text/Prepare.hs

@@ -50,7 +50,7 @@ data Paragraph = Uniform Grain | AuthorLike
 -- Grain: number of Sentences by block of Text
 -- Step : overlap of sentence between connex block of Text
 groupUniform :: Grain -> [Text] -> [Text]
-groupUniform g ts = map (Text.intercalate " ")
+groupUniform g ts = map Text.unwords
                   $ chunkAlong g g
                   $ sentences
                   $ Text.concat ts

src/Gargantext/Core/Text/Terms.hs

@@ -52,7 +52,7 @@ import Gargantext.Core.Text.Terms.Mono  (monoTerms)
 import Gargantext.Core.Text.Terms.Mono.Stem (stem, StemmingAlgorithm(..))
 import Gargantext.Core.Text.Terms.Mono.Token.En (tokenize)
 import Gargantext.Core.Text.Terms.Multi (multiterms)
-import Gargantext.Core.Types ( TermsCount, POS, Terms(Terms), TermsWithCount )
+import Gargantext.Core.Types ( TermsCount, POS, Terms(..), TermsWithCount )
 import Gargantext.Core.Utils (groupWithCounts)
 import Gargantext.Database.Prelude (DBCmd)
 import Gargantext.Database.Query.Table.Ngrams (insertNgrams)
@@ -60,6 +60,7 @@ import Gargantext.Database.Query.Table.NgramsPostag (NgramsPostag(..), insertNgr
 import Gargantext.Database.Schema.Ngrams (text2ngrams, NgramsId)
 import Gargantext.Prelude
 
+
 data TermType lang
   = Mono      { _tt_lang :: !lang }
   | Multi     { _tt_lang :: !lang }
@@ -86,7 +87,7 @@ extractTerms ncs (Unsupervised {..}) xs = mapM (terms ncs (Unsupervised { _tt_mo
   where
     m' = case _tt_model of
       Just m''-> m''
-      Nothing -> newTries _tt_windowSize (Text.intercalate " " xs)
+      Nothing -> newTries _tt_windowSize (Text.unwords xs)
 extractTerms ncs termTypeLang xs = mapM (terms ncs termTypeLang) xs
 
 
@@ -124,15 +125,15 @@ class ExtractNgramsT h
                    -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount))
 ------------------------------------------------------------------------
 enrichedTerms :: Lang -> PosTagAlgo -> POS -> Terms -> NgramsPostag
-enrichedTerms l pa po (Terms ng1 ng2) =
+enrichedTerms l pa po (Terms { .. }) =
   NgramsPostag { _np_lang = l
                , _np_algo = pa
                , _np_postag = po
                , _np_form = form
                , _np_lem = lem }
     where
-      form = text2ngrams $ Text.intercalate " " ng1
-      lem  = text2ngrams $ Text.intercalate " " $ Set.toList ng2
+      form = text2ngrams $ Text.unwords _terms_label
+      lem  = text2ngrams $ Text.unwords $ Set.toList _terms_stem
 
 ------------------------------------------------------------------------
 cleanNgrams :: Int -> Ngrams -> Ngrams

src/Gargantext/Core/Text/Terms/WithList.hs

@@ -114,7 +114,7 @@ extractTermsWithList' pats = map (concat . map concat . replaceTerms KeepAll pat
 
 --------------------------------------------------------------------------
 addSpaces :: Text -> Text
-addSpaces = (Text.intercalate " ") . (Text.chunksOf 1)
+addSpaces = Text.unwords . (Text.chunksOf 1)
 
 
 --------------------------------------------------------------------------

src/Gargantext/Core/Types.hs

@@ -68,7 +68,7 @@ data Terms = Terms { _terms_label :: Label
                    , _terms_stem  :: Stems
                    } deriving (Ord, Show)
 instance Eq Terms where
-  (==) (Terms _ s1) (Terms _ s2) = s1 == s2
+  (==) (Terms { _terms_stem = s1 }) (Terms { _terms_stem = s2 }) = s1 == s2
 
 type TermsCount = Int
 

src/Gargantext/Core/Types/Search.hs

@@ -151,6 +151,6 @@ instance ToHyperdataRow HyperdataContact where
   toHyperdataRow (HyperdataContact { _hc_who = Just (ContactWho _ fn ln _ _ _), _hc_where = ou} ) =
     HyperdataRowContact (fromMaybe "FirstName" fn) (fromMaybe "LastName" ln) ou'
       where
-        ou' = maybe "CNRS" (Text.intercalate " " . _cw_organization) (head ou)
+        ou' = maybe "CNRS" (Text.unwords . _cw_organization) (head ou)
   toHyperdataRow (HyperdataContact {}) =
     HyperdataRowContact "FirstName" "LastName" "Labs"

src/Gargantext/Core/Viz/Graph/Tools.hs

@@ -118,14 +118,14 @@ cooc2graphWith' _doPartitions _bridgenessMethod multi similarity threshold stren
 
   partitions <- if (Map.size distanceMap > 0)
       then recursiveClustering' (spinglass' 1) distanceMap
-      else panic $ Text.intercalate " " [ "I can not compute the graph you request"
-                                        , "because either the quantity of documents"
-                                        , "or the quantity of terms"
-                                        , "are lacking."
-                                        , "Solution: add more either Documents or Map Terms to your analysis."
-                                        , "Follow the available tutorials on the Training EcoSystems."
-                                        , "Ask your co-users of GarganText how to have access to it."
-                                        ]
+      else panic $ Text.unwords [ "I can not compute the graph you request"
+                                , "because either the quantity of documents"
+                                , "or the quantity of terms"
+                                , "are lacking."
+                                , "Solution: add more either Documents or Map Terms to your analysis."
+                                , "Follow the available tutorials on the Training EcoSystems."
+                                , "Ask your co-users of GarganText how to have access to it."
+                                ]
   length partitions `seq` pure ()
 
   let

src/Gargantext/Core/Viz/Phylo/API/Tools.hs

@@ -76,7 +76,8 @@ phylo2dot2json phylo = do
         fileDot  = dirPath </> "phylo.dot"
         fileToJson = dirPath </> "output.json"
 
-    dotToFile fileFrom (toPhyloExport phylo)
+    phyloContent <- phylo2dot phylo
+    writeFile fileFrom phyloContent
 
     -- parsing a file can be done with:
     -- runParser' (Data.GraphViz.Parsing.parse :: Parse (Data.GraphViz.DotGraph Text)) $ TL.fromStrict f
@@ -91,6 +92,20 @@ phylo2dot2json phylo = do
       Just v  -> pure v
 
 
+phylo2dot :: Phylo -> IO Text
+phylo2dot phylo = do
+  withTempDirectory "/tmp" "phylo" $ \dirPath -> do
+    let fileFrom = dirPath </> "phyloFrom.dot"
+
+    dotToFile fileFrom (toPhyloExport phylo)
+
+    value <- readFile fileFrom
+
+    case value of
+      ""  -> panic "[G.C.V.Phylo.API.phylo2dot] Error no file"
+      _   -> pure value
+
+
 flowPhyloAPI :: (HasNodeStory env err m, HasNodeError err, MonadLogger m)
              => PhyloConfig -> CorpusId -> m Phylo
 flowPhyloAPI config cId = do

src/Gargantext/Database/Action/Metrics/NgramsByContext.hs

@@ -185,12 +185,13 @@ selectNgramsOccurrencesOnlyByContextUser_withSample cId int nt tms =
                 ( int
                 , toDBid NodeDocument
                 , cId
-                , Values fields ((DPS.Only . unNgramsTerm) <$> (List.take 10000 tms))
+                -- , Values fields ((DPS.Only . unNgramsTerm) <$> (List.take 10000 tms))
+                , DPS.In (unNgramsTerm <$> (List.take 10000 tms))
                 , cId
                 , toDBid nt
                 )
-    where
-      fields = [QualifiedIdentifier Nothing "text"]
+    -- where
+    --   fields = [QualifiedIdentifier Nothing "text"]
 
 queryNgramsOccurrencesOnlyByContextUser_withSample :: DPS.Query
 queryNgramsOccurrencesOnlyByContextUser_withSample = [sql|
@@ -198,18 +199,42 @@ queryNgramsOccurrencesOnlyByContextUser_withSample = [sql|
                           JOIN nodes_contexts nn ON n.id = nn.context_id
                             WHERE n.typename  = ?
                             AND nn.node_id = ?),
-       input_rows(terms) AS (?)
-  SELECT ng.terms, COUNT(cng.context_id) FROM context_node_ngrams cng
-    JOIN ngrams ng      ON cng.ngrams_id = ng.id
-    JOIN input_rows  ir ON ir.terms      = ng.terms
+       input_rows AS (
+           SELECT id, terms
+             FROM ngrams
+             WHERE terms IN ?
+        )
+  SELECT ir.terms, COUNT(cng.context_id) FROM context_node_ngrams cng
+    JOIN input_rows  ir ON cng.ngrams_id = ir.id
     JOIN nodes_contexts nn ON nn.context_id   = cng.context_id
     JOIN nodes_sample n ON nn.context_id   = n.id
     WHERE nn.node_id      = ? -- CorpusId
       AND cng.ngrams_type = ? -- NgramsTypeId
       AND nn.category     > 0
-      GROUP BY cng.node_id, ng.terms
+      GROUP BY cng.node_id, ir.terms
   |]
 
+  
+-- queryNgramsOccurrencesOnlyByContextUser_withSample :: DPS.Query
+-- queryNgramsOccurrencesOnlyByContextUser_withSample = [sql|
+--   WITH nodes_sample AS (SELECT c.id FROM contexts c TABLESAMPLE SYSTEM_ROWS (?)
+--                           JOIN nodes_contexts nc ON c.id = nc.context_id
+--                             WHERE c.typename  = ?
+--                             AND nc.node_id = ?),
+--        input_rows(terms) AS (?)
+--   SELECT ng.terms, COUNT(cng.context_id) FROM context_node_ngrams cng
+--     JOIN ngrams ng      ON cng.ngrams_id = ng.id
+--     JOIN input_rows  ir ON ir.terms      = ng.terms
+--     JOIN nodes_contexts nc ON nc.context_id   = cng.context_id
+--     JOIN nodes_sample ns ON nc.context_id   = ns.id
+--     WHERE nc.node_id      = ? -- CorpusId
+--       AND cng.ngrams_type = ? -- NgramsTypeId
+--       AND nc.category     > 0
+--       -- AND nc.context_id IN (SELECT id FROM nodes_sample)
+--       GROUP BY cng.node_id, ng.terms
+--   |]
+
+
 selectNgramsOccurrencesOnlyByContextUser_withSample' :: HasDBid NodeType
                                                      => CorpusId
                                                      -> Int

src/Gargantext/Database/Action/Metrics/TFICF.hs

@@ -70,6 +70,9 @@ getTficf_withSample cId mId nt = do
                      <$> getOccByNgramsOnlyFast_withSample mId countGlobal nt
                             (HM.keys mapTextDoubleLocal)
 
+  printDebug "[getTficf_withSample] mapTextDoubleLocal: " mapTextDoubleLocal
+  printDebug "[getTficf_withSample] mapTextDoubleGlobal: " mapTextDoubleGlobal
+
   --printDebug "getTficf_withSample" (mapTextDoubleLocal, mapTextDoubleGlobal, countLocal, countGlobal)
   pure $ HM.mapWithKey (\t n ->
       tficf (TficfInfra (Count n                                               )

src/Gargantext/Database/Prelude.hs

@@ -207,9 +207,9 @@ fromField' field mb = do
           valueToHyperdata v = case fromJSON v of
              Success a  -> pure a
              Error _err -> returnError ConversionFailed field
-                         $ DL.intercalate " " [ "cannot parse hyperdata for JSON: "
-                                              , show v
-                                              ]
+                         $ DL.unwords [ "cannot parse hyperdata for JSON: "
+                                      , show v
+                                      ]
 
 printSqlOpa :: Default Unpackspec a a => Select a -> IO ()
 printSqlOpa = putStrLn . maybe "Empty query" identity . showSql

src/Gargantext/Database/Query/Table/Ngrams.hs

@@ -28,7 +28,7 @@ import Data.HashMap.Strict qualified as HashMap
 import Data.List qualified as List
 import Data.Map.Strict qualified as Map
 import Database.PostgreSQL.Simple qualified as PGS
-import Gargantext.Core.Text.Ngrams (Ngrams, NgramsType)
+import Gargantext.Core.Text.Ngrams (Ngrams(..), NgramsType)
 import Gargantext.Database.Admin.Types.Node ( pgNodeId, CorpusId, ListId, DocId )
 import Gargantext.Database.Prelude (runOpaQuery, formatPGSQuery, runPGSQuery, DBCmd)
 import Gargantext.Database.Query.Join (leftJoin3)
@@ -79,14 +79,15 @@ insertNgrams ns =
 
 -- TODO-ACCESS: access must not be checked here but when insertNgrams' is called.
 insertNgrams' :: [Ngrams] -> DBCmd err [Indexed Int Text]
-insertNgrams' ns = runPGSQuery queryInsertNgrams (PGS.Only $ Values fields ns)
+insertNgrams' ns = runPGSQuery queryInsertNgrams (PGS.Only $ Values fields ns')
   where
-    fields = map (\t -> QualifiedIdentifier Nothing t) ["text", "int4"]
+    ns' = (\n -> (_ngramsTerms n, _ngramsSize n)) <$> ns
+    fields = map (QualifiedIdentifier Nothing) ["text", "int4"]
 
 _insertNgrams_Debug :: [(Text, Size)] -> DBCmd err ByteString
 _insertNgrams_Debug ns = formatPGSQuery queryInsertNgrams (PGS.Only $ Values fields ns)
   where
-    fields = map (\t -> QualifiedIdentifier Nothing t) ["text", "int4"]
+    fields = map (QualifiedIdentifier Nothing) ["text", "int4"]
 
 ----------------------
 queryInsertNgrams :: PGS.Query

src/Gargantext/Database/Query/Table/NgramsPostag.hs

@@ -53,14 +53,14 @@ type NgramsPostagInsert = ( Int
                           )
 
 toInsert :: NgramsPostag -> NgramsPostagInsert
-toInsert (NgramsPostag l a p form lem) =
-  ( toDBid l
-  , toDBid a
-  , show p
-  , view ngramsTerms form
-  , view ngramsSize  form
-  , view ngramsTerms lem
-  , view ngramsSize  lem
+toInsert (NgramsPostag { .. }) =
+  ( toDBid _np_lang
+  , toDBid _np_algo
+  , show _np_postag
+  , view ngramsTerms _np_form
+  , view ngramsSize  _np_form
+  , view ngramsTerms _np_lem
+  , view ngramsSize  _np_lem
   )
 
 insertNgramsPostag :: [NgramsPostag] -> DBCmd err (HashMap Text NgramsId)
@@ -154,17 +154,18 @@ SELECT terms,id FROM ins_form_ret
 -- TODO add lang and postag algo
 -- TODO remove when form == lem in insert
 selectLems :: Lang -> NLPServerConfig -> [Ngrams] -> DBCmd err [(Form, Lem)]
-selectLems l (NLPServerConfig { server }) ns = runPGSQuery querySelectLems (PGS.In (map _ngramsTerms ns), toDBid l, toDBid server)
+selectLems l (NLPServerConfig { server }) ns =
+  runPGSQuery querySelectLems (PGS.In (_ngramsTerms <$> ns), toDBid l, toDBid server)
 ----------------------
 querySelectLems :: PGS.Query
 querySelectLems = [sql|
   WITH
      trms
-       AS (SELECT id, terms, n
+       AS (SELECT id, terms
             FROM ngrams
             WHERE terms IN ?)
-   , input_rows(lang_id, algo_id, terms,n)
-    AS (SELECT ? as lang_id, ? as algo_id, terms, n, id
+   , input_rows
+    AS (SELECT ? as lang_id, ? as algo_id, terms, id
          FROM trms)
     , lems AS ( select ir.terms as t1, n2.terms as t2, sum(np.score) as score from input_rows ir
     JOIN ngrams_postag np ON np.ngrams_id = ir.id
@@ -179,29 +180,29 @@ querySelectLems = [sql|
   |]
 
 -- | This is the same as 'selectLems', but slower.
-selectLems' :: Lang -> NLPServerConfig -> [Ngrams] -> DBCmd err [(Form, Lem)]
-selectLems' l (NLPServerConfig { server }) ns = runPGSQuery querySelectLems' (PGS.Only $ Values fields datas)
-  where
-    fields = map (QualifiedIdentifier Nothing) ["int4","int4","text", "int4"]
-    datas  = map (\d -> [toField $ toDBid l, toField $ toDBid server] <> toRow d) ns
+-- selectLems' :: Lang -> NLPServerConfig -> [Ngrams] -> DBCmd err [(Form, Lem)]
+-- selectLems' l (NLPServerConfig { server }) ns = runPGSQuery querySelectLems' (PGS.Only $ Values fields datas)
+--   where
+--     fields = map (QualifiedIdentifier Nothing) ["int4","int4","text", "int4"]
+--     datas  = map (\d -> [toField $ toDBid l, toField $ toDBid server] <> toRow d) ns
 
     
-querySelectLems' :: PGS.Query
-querySelectLems' = [sql|
-  WITH input_rows(lang_id, algo_id, terms,n)
-    AS (?) -- ((VALUES ('automata' :: "text")))
-    , lems AS ( select n1.terms as t1 ,n2.terms as t2 ,sum(np.score) as score from input_rows ir
-    JOIN ngrams        n1 ON ir.terms = n1.terms
-    JOIN ngrams_postag np ON np.ngrams_id = n1.id
-    JOIN ngrams        n2 ON n2.id    = np.lemm_id
-    WHERE np.lang_id = ir.lang_id
-      AND np.algo_id = ir.algo_id
-    GROUP BY n1.terms, n2.terms
-    ORDER BY score DESC
-  )
-
-  SELECT t1,t2 from lems
-  |]
+-- querySelectLems' :: PGS.Query
+-- querySelectLems' = [sql|
+--   WITH input_rows(lang_id, algo_id, terms,n)
+--     AS (?) -- ((VALUES ('automata' :: "text")))
+--     , lems AS ( select n1.terms as t1 ,n2.terms as t2 ,sum(np.score) as score from input_rows ir
+--     JOIN ngrams        n1 ON ir.terms = n1.terms
+--     JOIN ngrams_postag np ON np.ngrams_id = n1.id
+--     JOIN ngrams        n2 ON n2.id    = np.lemm_id
+--     WHERE np.lang_id = ir.lang_id
+--       AND np.algo_id = ir.algo_id
+--     GROUP BY n1.terms, n2.terms
+--     ORDER BY score DESC
+--   )
+
+--   SELECT t1,t2 from lems
+--   |]
 
 -- | Insert Table
 createTable_NgramsPostag :: DBCmd err [Int]

src/Gargantext/Database/Query/Table/NodeContext.hs

@@ -317,6 +317,15 @@ nodeContextsScore inputData = map (\(PGS.Only a) -> a)
 
 
 ------------------------------------------------------------------------
+-- | Counts the number of documents in a corpus.
+--   Also applies filter for category to be at least 1 (i.e. not in trash).
+--     select count(*)
+--     from contexts c 
+--     join nodes_contexts nc on c.id = nc.context_id 
+--     where 
+--     nc.node_id = 88
+--     and nc.category >= 1
+--     and c.typename = 4
 selectCountDocs :: HasDBid NodeType => CorpusId -> DBCmd err Int
 selectCountDocs cId = runCountOpaQuery (queryCountDocs cId)
   where

src/Gargantext/Database/Schema/Ngrams.hs

@@ -39,6 +39,10 @@ import Gargantext.Prelude
 type NgramsId  = Int
 type Size      = Int
 
+-- | Ngrams table
+-- 'n' is the size, see G.D.Q.T.Ngrams -> insertNgrams'
+-- function. I.e. ngrams with 1 term are of size 1, ngrams with 2
+-- terms are of size 2 etc.
 data NgramsPoly id terms n = NgramsDB { _ngrams_id    :: !id
                                       , _ngrams_terms :: !terms
                                       , _ngrams_n     :: !n
@@ -90,7 +94,8 @@ instance PGS.ToRow Text where
   toRow t = [toField t]
 
 text2ngrams :: Text -> Ngrams
-text2ngrams txt = UnsafeNgrams txt' $ length $ splitOn " " txt'
+text2ngrams txt = UnsafeNgrams { _ngramsTerms = txt'
+                               , _ngramsSize = length $ splitOn " " txt' }
   where
     txt' = strip txt
 

src/Gargantext/Database/Types.hs

@@ -21,7 +21,9 @@ import Gargantext.Database.Schema.Prelude
 import Gargantext.Prelude
 
 
--- | Index memory of any type in Gargantext
+-- | Index memory of any type in Gargantext.
+-- I.e. given entity 'a', we use this type to mark that it has a DB id of type 'i'.
+-- An un-indexed entity 'a' might not have been INSERT-ed yet to the DB.
 data Indexed i a =
   Indexed { _index     :: !i
           , _unIndex   :: !a

test/Test/Offline/JSON.hs

@@ -7,22 +7,22 @@
 module Test.Offline.JSON (tests) where
 
 import Data.Aeson
+import Data.ByteString qualified as B
+import Data.ByteString.Lazy.Char8 qualified as C8
 import Data.Either
 import Gargantext.API.Errors
 import Gargantext.API.Node.Corpus.New
 import Gargantext.API.Node.Corpus.Types
+import Gargantext.API.Node.Types
 import Gargantext.Core.Types.Phylo
 import Gargantext.Core.Viz.Phylo.API
+import Gargantext.Database.Admin.Types.Node
+import Paths_gargantext
 import Prelude
 import Test.Tasty
 import Test.Tasty.HUnit
 import Test.Tasty.QuickCheck
 import Text.RawString.QQ
-import qualified Data.ByteString as B
-import qualified Data.ByteString.Lazy.Char8 as C8
-
-import Paths_gargantext
-import Gargantext.Database.Admin.Types.Node
 
 jsonRoundtrip :: (Show a, FromJSON a, ToJSON a, Eq a) => a -> Property
 jsonRoundtrip a =