[flow] more massaging of Flow.hs file, refactorings

bin/gargantext-import/Main.hs

@@ -23,9 +23,10 @@ import Gargantext.API.Admin.EnvTypes (DevEnv(..), DevJobHandle(..))
 import Gargantext.API.Errors.Types
 import Gargantext.API.Node () -- instances
 import Gargantext.Core (Lang(..))
+import Gargantext.Core.Text.Terms (TermType(..))
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Types.Query (Limit)
-import Gargantext.Database.Action.Flow (flowCorpusFile, flowAnnuaire, TermType(..))
+import Gargantext.Database.Action.Flow (flowCorpusFile, flowAnnuaire)
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Admin.Types.Node (CorpusId)
 import Gargantext.Prelude

bin/gargantext-init/Main.hs

@@ -15,11 +15,11 @@ Import a corpus binary.
 
 module Main where
 
+import Data.List.NonEmpty qualified as NE
 import Gargantext.API.Dev (withDevEnv, runCmdDev)
 import Gargantext.API.Errors.Types
 import Gargantext.API.Node () -- instances only
 import Gargantext.Core.Types.Individu (User(..), arbitraryNewUsers, NewUser(..), arbitraryUsername, GargPassword(..))
-import Gargantext.Database.Action.Flow (getOrMkRoot, getOrMk_RootWithCorpus)
 import Gargantext.Database.Admin.Config (userMaster, corpusMasterName)
 import Gargantext.Database.Admin.Trigger.Init (initFirstTriggers, initLastTriggers)
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataCorpus)
@@ -27,9 +27,9 @@ import Gargantext.Database.Admin.Types.Node
 import Gargantext.Database.Prelude (Cmd, DBCmd)
 import Gargantext.Database.Query.Table.Node (getOrMkList)
 import Gargantext.Database.Query.Table.User (insertNewUsers, )
+import Gargantext.Database.Query.Tree.Root (getOrMkRoot, getOrMk_RootWithCorpus)
 import Gargantext.Prelude
 import Gargantext.Prelude.Config (GargConfig(..), readConfig)
-import qualified Data.List.NonEmpty as NE
 
 
 main :: IO ()

src/Gargantext/API/Node/Corpus/New.hs

@@ -42,10 +42,11 @@ import Gargantext.Core (Lang(..), withDefaultLanguage, defaultLanguage)
 import Gargantext.Core.NodeStory (HasNodeStoryImmediateSaver, HasNodeArchiveStoryImmediateSaver, currentVersion, NgramsStatePatch')
 import Gargantext.Core.Text.Corpus.API qualified as API
 import Gargantext.Core.Text.Corpus.Parsers qualified as Parser (FileType(..), parseFormatC)
+import Gargantext.Core.Text.Terms (TermType(..))
 import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Action.Flow (flowCorpus, getDataText, flowDataText, TermType(..){-, allDataOrigins-})
+import Gargantext.Database.Action.Flow (flowCorpus, getDataText, flowDataText)
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Action.Mail (sendMail)
 import Gargantext.Database.Action.Node (mkNodeWithParent)

src/Gargantext/API/Node/Corpus/Types.hs

@@ -21,7 +21,7 @@ import Data.Swagger
 import Data.Text qualified as T
 import Gargantext.API.Admin.Orchestrator.Types qualified as Types
 import Gargantext.Core.Utils.Prefix (unPrefix)
-import Gargantext.Database.Action.Flow (DataOrigin(..))
+import Gargantext.Database.Action.Flow.Types (DataOrigin(..))
 import Gargantext.Prelude
 import Test.QuickCheck
 

src/Gargantext/API/Node/DocumentsFromWriteNodes.hs

@@ -36,8 +36,8 @@ import Gargantext.Core.Text.Corpus.Parsers.FrameWrite
 import Gargantext.Core.Text.List.Social (FlowSocialListWith)
 import Gargantext.Core.Text.Terms (TermType(..))
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Database.Action.Flow (flowDataText, DataText(..))
-import Gargantext.Database.Action.Flow.Types (FlowCmdM)
+import Gargantext.Database.Action.Flow (flowDataText)
+import Gargantext.Database.Action.Flow.Types (DataText(..), FlowCmdM)
 import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
 import Gargantext.Database.Admin.Types.Hyperdata.Frame
 import Gargantext.Database.Admin.Types.Node

src/Gargantext/Database/Action/Flow/Types.hs

@@ -18,17 +18,19 @@ Portability : POSIX
 module Gargantext.Database.Action.Flow.Types
     where
 
-import Conduit (ConduitT)
+import Conduit (ConduitT, (.|))
 import Control.Lens (makeLenses)
 import Data.Aeson (ToJSON)
 import Data.Aeson.TH (deriveJSON)
+import Data.Conduit qualified as C
+import Data.Conduit.List qualified as CL
 import Data.HashMap.Strict (HashMap)
 import Data.Swagger (ToSchema(..), genericDeclareNamedSchema)
-import Gargantext.Core.Flow.Types
-import Gargantext.Core.NodeStory
-import Gargantext.Core.Text
+import Gargantext.Core.Flow.Types (UniqId)
+import Gargantext.Core.NodeStory (HasNodeStory)
+import Gargantext.Core.Text (HasText)
 import Gargantext.Core.Text.Corpus.API qualified as API
-import Gargantext.Core.Text.Terms
+import Gargantext.Core.Text.Terms (ExtractNgramsT)
 import Gargantext.Core.Types (HasValidationError, TermsCount)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument)
@@ -40,7 +42,7 @@ import Gargantext.Database.Query.Tree.Error (HasTreeError)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))
 import Gargantext.Database.Types (Indexed)
 import Gargantext.Prelude
-import Gargantext.System.Logging
+import Gargantext.System.Logging (MonadLogger)
 
 
 type FlowCmdM env err m =
@@ -88,6 +90,27 @@ deriveJSON (unPrefix "_do_") ''DataOrigin
 instance ToSchema DataOrigin where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_do_")
 
+
+
+
+
 data DataText = DataOld ![NodeId]
               | DataNew !(Maybe Integer, ConduitT () HyperdataDocument IO ())
               --- | DataNew ![[HyperdataDocument]]
+
+
+-- Show instance is not possible because of IO
+printDataText :: DataText -> IO ()
+printDataText (DataOld xs) = putText $ show xs
+printDataText (DataNew (maybeInt, conduitData)) = do
+  res <- C.runConduit (conduitData .| CL.consume)
+  putText $ show (maybeInt, res)
+
+
+
+------------------------------------------------------------------------
+-- Unused functions
+
+-- allDataOrigins :: [DataOrigin]
+-- allDataOrigins = map InternalOrigin API.externalAPIs <> map ExternalOrigin API.externalAPIs
+

src/Gargantext/Database/Action/User/New.hs

@@ -27,20 +27,20 @@ module Gargantext.Database.Action.User.New
 
 import Control.Lens (view)
 import Control.Monad.Random
+import Data.List.NonEmpty qualified as NE
 import Data.Text (splitOn)
 import Data.Text qualified as Text
 import Gargantext.Core.Mail
 import Gargantext.Core.Mail.Types (HasMail, mailSettings)
 import Gargantext.Core.Types.Individu
-import Gargantext.Database.Action.Flow (getOrMkRoot)
 import Gargantext.Database.Admin.Types.Node
 import Gargantext.Database.Prelude (Cmd, DBCmd, CmdM)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError(..), nodeError, NodeError(..))
 import Gargantext.Database.Query.Table.User
+import Gargantext.Database.Query.Tree.Root (getOrMkRoot)
 import Gargantext.Prelude
 import Gargantext.Prelude.Crypto.Pass.User (gargPass)
 import Gargantext.Prelude.Mail.Types (MailConfig)
-import qualified Data.List.NonEmpty as NE
 
 ------------------------------------------------------------------------
 -- | Creates a new 'User' from the input 'EmailAddress', which needs to

test/Test/API/Setup.hs

@@ -16,32 +16,31 @@ import Gargantext.API.Prelude
 import Gargantext.Core.NLP
 import Gargantext.Core.NodeStory
 import Gargantext.Core.Types.Individu
-import Gargantext.Database.Action.Flow
 import Gargantext.Database.Action.User.New
 import Gargantext.Database.Admin.Config (userMaster, corpusMasterName)
 import Gargantext.Database.Admin.Trigger.Init
 import Gargantext.Database.Admin.Types.Hyperdata
 import Gargantext.Database.Prelude
 import Gargantext.Database.Query.Table.Node (getOrMkList)
--- import Gargantext.Prelude (printDebug)
+import Gargantext.Database.Query.Tree.Root (getOrMk_RootWithCorpus)
 import Gargantext.Prelude.Config
+import Gargantext.Prelude.Mail qualified as Mail
+import Gargantext.Prelude.NLP qualified as NLP
 import Gargantext.System.Logging
+import Gargantext.Utils.Jobs qualified as Jobs
+import Gargantext.Utils.Jobs.Monad qualified as Jobs
+import Gargantext.Utils.Jobs.Queue qualified as Jobs
+import Gargantext.Utils.Jobs.Settings qualified as Jobs
 import Network.HTTP.Client.TLS (newTlsManager)
 import Network.Wai (Application)
+import Network.Wai.Handler.Warp qualified as Wai
+import Network.Wai.Handler.Warp qualified as Warp
 import Prelude
 import Servant.Auth.Client ()
 import Servant.Client
+import Servant.Job.Async qualified as ServantAsync
 import Test.Database.Setup (withTestDB, fakeIniPath, testEnvToPgConnectionInfo)
 import Test.Database.Types
-import qualified Gargantext.Prelude.Mail as Mail
-import qualified Gargantext.Prelude.NLP as NLP
-import qualified Gargantext.Utils.Jobs as Jobs
-import qualified Gargantext.Utils.Jobs.Monad as Jobs
-import qualified Gargantext.Utils.Jobs.Queue as Jobs
-import qualified Gargantext.Utils.Jobs.Settings as Jobs
-import qualified Network.Wai.Handler.Warp         as Warp
-import qualified Network.Wai.Handler.Warp as Wai
-import qualified Servant.Job.Async as ServantAsync
 
 
 newTestEnv :: TestEnv -> Logger (GargM Env BackendInternalError) -> Warp.Port -> IO Env

test/Test/Database/Operations/DocumentSearch.hs

@@ -13,36 +13,35 @@ Portability : POSIX
 
 module Test.Database.Operations.DocumentSearch where
 
-import Prelude
+
 
 import Control.Lens (view)
 import Control.Monad.Reader
 import Data.Aeson.QQ.Simple
 import Data.Aeson.Types
--- import Gargantext.API.Node.Update (updateDocs)
+import Data.Text qualified as T
 import Gargantext.Core
 import Gargantext.Core.NLP (nlpServerGet)
+import Gargantext.Core.Text.Corpus.Query qualified as API
+import Gargantext.Core.Text.Terms (TermType(..))
+import Gargantext.Core.Text.Terms.Mono.Stem.En
 import Gargantext.Core.Types.Individu
-import Gargantext.Database.Action.Flow
+import Gargantext.Database.Action.Flow (addDocumentsToHyperCorpus)
 import Gargantext.Database.Action.Search
+import Gargantext.Database.Admin.Config (userMaster)
 import Gargantext.Database.Admin.Types.Hyperdata.Document
+import Gargantext.Database.Query.Facet
 import Gargantext.Database.Query.Table.Node
 import Gargantext.Database.Query.Tree.Root
 import Gargantext.Database.Schema.Node (NodePoly(..))
--- import Network.URI (parseURI)
-
+import Gargantext.Prelude
 import Test.Database.Types
 import Test.Hspec.Expectations
 import Test.Tasty.HUnit
-import Gargantext.Core.Text.Terms.Mono.Stem.En
-import Gargantext.Database.Admin.Config (userMaster)
-import qualified Data.Text as T
-import qualified Gargantext.Core.Text.Corpus.Query as API
-import Gargantext.Database.Query.Facet
 
 
 exampleDocument_01 :: HyperdataDocument
-exampleDocument_01 = either error id $ parseEither parseJSON $ [aesonQQ|
+exampleDocument_01 = either errorTrace identity $ parseEither parseJSON $ [aesonQQ|
   { "doi":"01"
   , "publication_day":6
   , "language_iso2":"EN"
@@ -63,7 +62,7 @@ exampleDocument_01 = either error id $ parseEither parseJSON $ [aesonQQ|
 |]
 
 exampleDocument_02 :: HyperdataDocument
-exampleDocument_02 = either error id $ parseEither parseJSON $ [aesonQQ|
+exampleDocument_02 = either errorTrace identity $ parseEither parseJSON $ [aesonQQ|
   { "doi":""
   , "uniqId": "1405.3072v3"
   , "bdd": "Arxiv"
@@ -83,7 +82,7 @@ exampleDocument_02 = either error id $ parseEither parseJSON $ [aesonQQ|
 |]
 
 exampleDocument_03 :: HyperdataDocument
-exampleDocument_03 = either error id $ parseEither parseJSON $ [aesonQQ|
+exampleDocument_03 = either errorTrace identity $ parseEither parseJSON $ [aesonQQ|
 {
     "bdd": "Arxiv"
   , "doi": ""
@@ -101,7 +100,7 @@ exampleDocument_03 = either error id $ parseEither parseJSON $ [aesonQQ|
 |]
 
 exampleDocument_04 :: HyperdataDocument
-exampleDocument_04 = either error id $ parseEither parseJSON $ [aesonQQ|
+exampleDocument_04 = either errorTrace identity $ parseEither parseJSON $ [aesonQQ|
 {
     "bdd": "Arxiv"
   , "doi": ""
@@ -141,7 +140,7 @@ stemmingTest _env = do
   stemIt "PyPlasm:" `shouldBe` "PyPlasm:"
 
 mkQ :: T.Text -> API.Query
-mkQ txt = either (\e -> error $ "(query) = " <> T.unpack txt <> ": " <> e) id . API.parseQuery . API.RawQuery $ txt
+mkQ txt = either (\e -> errorTrace $ "(query) = " <> T.unpack txt <> ": " <> e) identity . API.parseQuery . API.RawQuery $ txt
 
 corpusSearch01 :: TestEnv -> Assertion
 corpusSearch01 env = do