[temp file] fix tests

src/Gargantext/API/Node/Corpus/New.hs

@@ -23,11 +23,9 @@ module Gargantext.API.Node.Corpus.New
 import Conduit ((.|), yieldMany, mapMC, mapC, transPipe)
 import Control.Exception.Safe (MonadMask)
 import Control.Lens ( view, non )
-import Data.ByteString.Lazy qualified as BSL
 import Data.Conduit.Internal (zipSources)
 import Data.Swagger ( ToSchema(..) )
 import Data.Text qualified as T
-import Data.Text.Encoding qualified as TE
 import Database.PostgreSQL.Simple.LargeObjects qualified as PSQL
 import Gargantext.API.Admin.Orchestrator.Types qualified as API
 import Gargantext.API.Ngrams (commitStatePatch, Versioned(..))
@@ -53,7 +51,7 @@ import Gargantext.Database.Admin.Types.Hyperdata.Document ( ToHyperdataDocument(
 import Gargantext.Database.Admin.Types.Hyperdata.File ( HyperdataFile(..) )
 import Gargantext.Database.Admin.Types.Node (CorpusId, NodeType(..), ParentId)
 import Gargantext.Database.GargDB qualified as GargDB
-import Gargantext.Database.Prelude (readLargeObjectViaTempFile)
+import Gargantext.Database.Prelude (readLargeObject)
 import Gargantext.Database.Query.Table.Node (getNodeWith, getOrMkList)
 import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
 import Gargantext.Database.Query.Tree.Root (MkCorpusUser(MkCorpusUserNormalCorpusIds))
@@ -248,10 +246,9 @@ addToCorpusWithTempFile user cid nwtf jobHandle = do
 
   -- TODO granularity of the logStatus
   let oId = PSQL.Oid $ fromIntegral $ nwtf ^. wtf_file_oid
-  (data'', size) <- readLargeObjectViaTempFile oId
-  let data' = BSL.toStrict data''
-  $(logLocM) DEBUG $ "[addToCorpusWithTempFile] size: " <> show size
-  $(logLocM) DEBUG $ "[addToCorpusWithTempFile] data': " <> TE.decodeUtf8 data'
+  data' <- readLargeObject oId
+  -- $(logLocM) DEBUG $ "[addToCorpusWithTempFile] size: " <> show size
+  -- $(logLocM) DEBUG $ "[addToCorpusWithTempFile] data': " <> TE.decodeUtf8 data'
   eDocsC <- liftBase $ parseC (nwtf ^. wtf_fileformat) data'
   case eDocsC of
     Right (count, docsC) -> do
@@ -297,7 +294,7 @@ addToCorpusWithTempFile user cid nwtf jobHandle = do
 
       markComplete jobHandle
     Left parseErr -> do
-      $(logLocM) ERROR $ "[addToCorpusWithTempFile] parse error: " <> (Parser._ParseFormatError parseErr)
+      $(logLocM) ERROR $ "[addToCorpusWithTempFile] parse error: " <> Parser._ParseFormatError parseErr
       markFailed (Just parseErr) jobHandle
 
 {-

src/Gargantext/API/Routes.hs

@@ -75,7 +75,6 @@ addCorpusWithQuery user =
 -- | Uses temporary file stored in postgres to add that file to a corpus
 addWithTempFileApi :: AuthenticatedUser
                    -> Named.AddWithTempFile (AsServerT (GargM Env BackendInternalError))
-                   -- -> WorkerAPI '[FormUrlEncoded] NewWithForm (AsServerT m)
 addWithTempFileApi authenticatedUser =
   Named.AddWithTempFile {
     addWithTempFileEp = \cId -> 

src/Gargantext/Database/Prelude.hs

@@ -9,6 +9,7 @@ Portability : POSIX
 
 -}
 
+{-# LANGUAGE BangPatterns        #-}
 {-# LANGUAGE ConstraintKinds     #-}
 {-# LANGUAGE LambdaCase          #-}
 {-# LANGUAGE ScopedTypeVariables #-}
@@ -285,31 +286,34 @@ createLargeObject bs = mkCmd $ \c -> PGS.withTransaction c $ do
   PSQL.loClose c loFd
   pure oId
 
-readLargeObject :: PSQL.Oid -> DBCmd err (BSL.ByteString, Int)
+readLargeObject :: PSQL.Oid -> DBCmd err BS.ByteString
 readLargeObject oId = mkCmd $ \c -> PGS.withTransaction c $ do
   loFd <- PSQL.loOpen c oId PSQL.ReadMode
   let chunkSize = 1024
   let readChunks tell = do
         c' <- PSQL.loRead c loFd chunkSize
         tell' <- PSQL.loTell c loFd
-        putText $ "[readLargeObject] tell': " <> show tell'
         if tell == tell' then
           pure ([c'], tell)
         else do
           (cs', tell'') <- readChunks tell'
           pure (c':cs', tell'')
-  (chunks, size) <- readChunks 0
-  pure (BSL.fromChunks chunks, size)
+  (chunks, _size) <- readChunks 0
+  let s = force BSL.toStrict $ BSL.fromChunks chunks
+  PSQL.loClose c loFd
+  pure s
 
 readLargeObjectViaTempFile :: (CES.MonadMask m, IsDBCmd env err m)
-                           => PSQL.Oid -> m (BSL.ByteString, Int)
-readLargeObjectViaTempFile oId =
+                           => PSQL.Oid -> m BS.ByteString
+readLargeObjectViaTempFile oId = do
   CES.bracket (liftBase $ emptySystemTempFile "large-object")
     (liftBase . removeFile)
     (\fp -> do
-      mkCmd $ \c -> PSQL.loExport c oId fp
-      c <- liftBase $ BSL.readFile fp
-      pure (c, fromIntegral $ BSL.length c))
+      mkCmd $ \c -> withTransaction c $ \_ -> PSQL.loExport c oId fp
+      !contents <- liftBase $ BS.readFile fp
+      pure contents)
+  where
+    withTransaction c = CES.bracket (PGS.begin c) (\_ -> PGS.rollback c)
 
 removeLargeObject :: Int -> DBCmd err ()
 removeLargeObject oId = mkCmd $ \c -> do