[API Query] api connection ok, needs RootId fix

bin/gargantext-import/Main.hs

@@ -27,13 +27,12 @@ import Gargantext.API.Admin.Settings (withDevEnv, runCmdDev, DevEnv)
 import Gargantext.API.Admin.Types (GargError)
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpusFile, flowAnnuaire)
+import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpusFile, flowAnnuaire, TermType(..))
 import Gargantext.Database.Action.Query.User (insertUsersDemo)
 import Gargantext.Database.Admin.Types.Node (CorpusId, toHyperdataDocument)
 import Gargantext.Database.Admin.Utils (Cmd, )
 import Gargantext.Prelude
 import Gargantext.Text.Corpus.Parsers (FileFormat(..))
-import Gargantext.Text.Terms (TermType(..))
 import Prelude (read)
 import System.Environment (getArgs)
 import qualified Data.Text as Text

src/Gargantext/API/Corpus/New.hs

@@ -35,10 +35,10 @@ import Data.Text (Text)
 import GHC.Generics (Generic)
 import Gargantext.API.Admin.Orchestrator.Types
 import Gargantext.API.Corpus.New.File
-import Gargantext.Core (Lang(..))
+import Gargantext.Core (Lang(..), allLangs)
 import Gargantext.Core.Types.Individu (UserId, User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, getDataText, flowDataText, TermType(..), DataOrigin(..))
+import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpus, getDataText, flowDataText, TermType(..), DataOrigin(..), allDataOrigins)
 import Gargantext.Database.Admin.Types.Node (CorpusId, ToHyperdataDocument(..))
 import Gargantext.Prelude
 import Servant
@@ -55,20 +55,22 @@ import qualified Gargantext.Text.Corpus.Parsers as Parser (FileFormat(..), parse
 
 ------------------------------------------------------------------------
 data Query = Query { query_query      :: Text
-                   , query_corpus_id  :: Int
-                   , query_databases  :: [API.ExternalAPIs]
+                   , query_node_id    :: Int
+                   , query_lang       :: Lang
+                   , query_databases  :: [DataOrigin]
                    }
-                   deriving (Eq, Show, Generic)
+                   deriving (Eq, Generic)
 
 deriveJSON (unPrefix "query_") 'Query
 
 instance Arbitrary Query where
-    arbitrary = elements [ Query q n fs
+    arbitrary = elements [ Query q n la fs
                          | q <- ["honeybee* AND collapse"
                                 ,"covid 19"
                                 ]
                          , n <- [0..10]
-                         , fs <- take 3 $ repeat API.externalAPIs
+                         , la <- allLangs
+                         , fs <- take 3 $ repeat allDataOrigins
                          ]
 
 instance ToSchema Query where
@@ -204,7 +206,7 @@ addToCorpusWithQuery u cid (WithQuery q dbs l) logStatus = do
   -- TODO if cid is folder -> create Corpus
   --      if cid is corpus -> add to corpus
   --      if cid is root   -> create corpus in Private
-  txts <- mapM (\db  -> getDataText db (fromMaybe (Multi EN) l) q (Just 10000)) dbs
+  txts <- mapM (\db  -> getDataText db     (fromMaybe (Multi EN) l) q (Just 10000)) dbs
   cids <- mapM (\txt -> flowDataText u txt (fromMaybe (Multi EN) l) cid) txts
   printDebug "corpus id" cids
   -- TODO ...

src/Gargantext/Database/Action/Flow.hs

@@ -41,6 +41,13 @@ module Gargantext.Database.Action.Flow -- (flowDatabase, ngrams2list)
   , getOrMk_RootWithCorpus
   , TermType(..)
   , DataOrigin(..)
+  , allDataOrigins
+
+-- To remove maybe
+  , tt_lang
+  , tt_ngramsSize
+  , tt_windowSize
+  , do_api
   )
     where
 
@@ -52,7 +59,7 @@ import Data.Map (Map, lookup)
 import Data.Maybe (Maybe(..), catMaybes)
 import Data.Monoid
 import Data.Swagger
-import Data.Text (Text, splitOn, intercalate)
+import Data.Text (splitOn, intercalate)
 import Data.Traversable (traverse)
 import Data.Tuple.Extra (first, second)
 import Debug.Trace (trace)
@@ -96,9 +103,20 @@ import qualified Gargantext.Text.Corpus.API as API
 
 ------------------------------------------------------------------------
 -- TODO use internal with API name (could be old data)
-data DataOrigin = Internal Gargantext
-                | External API.ExternalAPIs
+data DataOrigin = Internal { _do_api :: API.ExternalAPIs }
+                | External { _do_api :: API.ExternalAPIs }
                -- TODO Web
+  deriving (Generic, Eq)
+
+makeLenses ''DataOrigin
+deriveJSON (unPrefix "_do_") ''DataOrigin
+instance ToSchema DataOrigin where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_do_")
+
+allDataOrigins :: [DataOrigin]
+allDataOrigins = map Internal API.externalAPIs <> map External API.externalAPIs
+
+---------------
 
 data DataText = DataOld ![NodeId]
               | DataNew ![[HyperdataDocument]]
@@ -114,7 +132,7 @@ getDataText :: FlowCmdM env err m
 getDataText (External api) la q li = liftBase $ DataNew
                                   <$> splitEvery 500
                                   <$> API.get api (_tt_lang la) q li
-getDataText Gargantext     la q li = do
+getDataText (Internal _) _la q _li = do
   (_masterUserId, _masterRootId, cId) <- getOrMk_RootWithCorpus
                                            (UserName userMaster)
                                            (Left "")
@@ -146,7 +164,7 @@ makeLenses ''TermType
 deriveJSON (unPrefix "_tt_") ''TermType
 
 instance (ToSchema a) => ToSchema (TermType a) where
-  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_tta_")
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_tt_")
 
 
 flowDataText :: FlowCmdM env err m

src/Gargantext/Text/Terms.hs

@@ -37,13 +37,10 @@ module Gargantext.Text.Terms
   where
 
 import Control.Lens
-import Data.Aeson.TH (deriveJSON)
-import Data.Swagger
 import Data.Text (Text)
 import Data.Traversable
 import GHC.Base (String)
 import GHC.Generics (Generic)
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Core
 import Gargantext.Core.Types
 import Gargantext.Prelude