[conduit] some refactoring

src/Gargantext/API/Node/Corpus/New.hs

@@ -52,7 +52,7 @@ import qualified Gargantext.Core.Text.Corpus.API as API
 import qualified Gargantext.Core.Text.Corpus.Parsers as Parser (FileFormat(..), parseFormat)
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Action.Flow (flowCorpus, getDataText, flowDataText, DataText(..), TermType(..){-, allDataOrigins-})
+import Gargantext.Database.Action.Flow (flowCorpus, getDataText, flowDataText, TermType(..){-, allDataOrigins-})
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Action.Mail (sendMail)
 import Gargantext.Database.Action.Node (mkNodeWithParent)
@@ -221,18 +221,13 @@ addToCorpusWithQuery user cid (WithQuery { _wq_query = q
         [] -> do
           let txts = rights eTxts
           -- TODO Sum lenghts of each txt elements
-          let jl = JobLog { _scst_succeeded = Just 2
+          logStatus $ JobLog { _scst_succeeded = Just 2
                              , _scst_failed    = Just 0
                              , _scst_remaining = Just $ 1 + length txts
                              , _scst_events    = Just []
                              }
-          logStatus jl
 
           cids <- mapM (\txt -> do
-                           let id = case txt of
-                                 (DataNew (i, _)) -> i
-                                 _ -> (Just 0)
-                           logStatus $ addEvent "INFO: doc id" (T.pack $ show id) jl
                            flowDataText user txt (Multi l) cid Nothing logStatus) txts
           printDebug "corpus id" cids
           printDebug "sending email" ("xxxxxxxxxxxxxxxxxxxxx" :: Text)

src/Gargantext/Core/Text/Corpus/API/Pubmed.hs

@@ -42,12 +42,13 @@ get q l = do
   --      <$> PubMed.getMetadataWithC q l
 
 toDoc :: Lang -> PubMedDoc.PubMed -> HyperdataDocument
-toDoc l (PubMedDoc.PubMed (PubMedDoc.PubMedArticle t j as aus)
-                    (PubMedDoc.PubMedDate a y m d)
+toDoc l (PubMedDoc.PubMed { pubmed_id
+                          , pubmed_article = PubMedDoc.PubMedArticle t j as aus
+                          , pubmed_date = PubMedDoc.PubMedDate a y m d }
           ) = HyperdataDocument { _hd_bdd = Just "PubMed"
                                 , _hd_doi = Nothing
                                 , _hd_url = Nothing
-                                , _hd_uniqId = Nothing
+                                , _hd_uniqId = Just $ Text.pack $ show pubmed_id
                                 , _hd_uniqIdBdd = Nothing
                                 , _hd_page = Nothing
                                 , _hd_title = t

src/Gargantext/Database/Action/Flow.hs

@@ -168,7 +168,7 @@ flowDataText u (DataOld ids) tt cid mfslw _ = flowCorpusUser (_tt_lang tt) u (Ri
   where
     corpusType = (Nothing :: Maybe HyperdataCorpus)
 flowDataText u (DataNew (mLen, txtC)) tt cid mfslw logStatus =
-  flowCorpus u (Right [cid]) tt mfslw (mLen, transPipe liftBase txtC) logStatus
+  flowCorpus u (Right [cid]) tt mfslw (mLen, (transPipe liftBase txtC)) logStatus
 
 ------------------------------------------------------------------------
 -- TODO use proxy
@@ -252,7 +252,7 @@ flow c u cn la mfslw (mLength, docsC) logStatus = do
       id <- insertMasterDocs c la [doc]
       case mLength of
         Nothing -> pure ()
-        Just len ->
+        Just len -> do
           logStatus JobLog { _scst_succeeded = Just $ fromIntegral $ 1 + idx
                            , _scst_failed    = Just 0
                            , _scst_remaining = Just $ fromIntegral $ len - idx
@@ -531,5 +531,3 @@ extractInsert docs = do
                     documentsWithId
   _ <- insertExtractedNgrams $ HashMap.keys mapNgramsDocs'
   pure ()
-
-