Commit e4b72739 authored by Przemyslaw Kaminski's avatar Przemyslaw Kaminski

[zip upload] some more work on zip file

parent 52c7f251
Pipeline #1949 failed with stage
in 32 minutes and 32 seconds
...@@ -37,7 +37,7 @@ import Test.QuickCheck.Arbitrary ...@@ -37,7 +37,7 @@ import Test.QuickCheck.Arbitrary
import Gargantext.Prelude import Gargantext.Prelude
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs) import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs, ScraperEvent(..), scst_events)
import Gargantext.API.Admin.Types (HasSettings) import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Job (jobLogSuccess, jobLogFailTotal) import Gargantext.API.Job (jobLogSuccess, jobLogFailTotal)
import Gargantext.API.Node.Corpus.New.File import Gargantext.API.Node.Corpus.New.File
...@@ -258,7 +258,7 @@ addToCorpusWithForm user cid (NewWithForm ft d l _n) logStatus jobLog = do ...@@ -258,7 +258,7 @@ addToCorpusWithForm user cid (NewWithForm ft d l _n) logStatus jobLog = do
WOS -> Parser.parseFormat Parser.WOS WOS -> Parser.parseFormat Parser.WOS
PresseRIS -> Parser.parseFormat Parser.RisPresse PresseRIS -> Parser.parseFormat Parser.RisPresse
ZIP -> Parser.parseFormat Parser.ZIP ZIP -> Parser.parseFormat Parser.ZIP
-- TODO granularity of the logStatus -- TODO granularity of the logStatus
eDocs <- liftBase $ parse $ cs d eDocs <- liftBase $ parse $ cs d
case eDocs of case eDocs of
...@@ -283,9 +283,13 @@ addToCorpusWithForm user cid (NewWithForm ft d l _n) logStatus jobLog = do ...@@ -283,9 +283,13 @@ addToCorpusWithForm user cid (NewWithForm ft d l _n) logStatus jobLog = do
logStatus jobLog3 logStatus jobLog3
pure $ jobLog3 pure $ jobLog3
Left e -> do Left e -> do
printDebug "Error" e printDebug "[addToCorpusWithForm] parse error" e
let evt = ScraperEvent { _scev_message = Just $ T.pack e
, _scev_level = Just "ERROR"
, _scev_date = Nothing }
logStatus jobLogE logStatus $ over (scst_events . _Just) (\evt' -> evt' <> [evt]) jobLogE
pure jobLogE pure jobLogE
where where
jobLog2 = jobLogSuccess jobLog jobLog2 = jobLogSuccess jobLog
......
...@@ -43,6 +43,7 @@ import qualified Data.ByteString.Lazy as DBL ...@@ -43,6 +43,7 @@ import qualified Data.ByteString.Lazy as DBL
import qualified Data.Map as DM import qualified Data.Map as DM
import qualified Data.Text as DT import qualified Data.Text as DT
import qualified Prelude as Prelude import qualified Prelude as Prelude
import System.IO.Temp (emptySystemTempFile)
import Gargantext.Core (Lang(..)) import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..)) import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
...@@ -95,9 +96,12 @@ parseFormat WOS bs = do ...@@ -95,9 +96,12 @@ parseFormat WOS bs = do
$ partitionEithers $ partitionEithers
$ [runParser' WOS bs] $ [runParser' WOS bs]
pure $ Right docs pure $ Right docs
parseFormat ZIP _bs = do parseFormat ZIP bs = do
printDebug "[parseFormat]" ZIP path <- emptySystemTempFile "parsed-zip"
pure $ Left "Not implemented for ZIP" DB.writeFile path bs
parsedZip <- withArchive path $ do
DM.keys <$> getEntries
pure $ Left $ "Not implemented for ZIP, parsedZip" <> show parsedZip
parseFormat _ _ = undefined parseFormat _ _ = undefined
-- | Parse file into documents -- | Parse file into documents
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment