Commit 12c2beae authored by Przemyslaw Kaminski's avatar Przemyslaw Kaminski

[corpus] attempt to implement zip as base-64 encoded file

parent 3f8a8c0e
Pipeline #1958 failed with stage
in 9 minutes and 29 seconds
......@@ -21,6 +21,7 @@ module Gargantext.API.Node.Corpus.New
import Control.Lens hiding (elements, Empty)
import Data.Aeson
import Data.Aeson.TH (deriveJSON)
import qualified Data.ByteString.Base64 as BSB64
import Data.Either
import Data.Maybe (fromMaybe)
import Data.Swagger
......@@ -32,6 +33,7 @@ import Protolude (readFile)
import Servant
import Servant.Job.Utils (jsonOptions)
-- import Servant.Multipart
import qualified Data.Text.Encoding as TE
-- import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
......@@ -260,7 +262,12 @@ addToCorpusWithForm user cid (NewWithForm ft d l _n) logStatus jobLog = do
ZIP -> Parser.parseFormat Parser.ZIP
-- TODO granularity of the logStatus
eDocs <- liftBase $ parse $ cs d
let data' = case ft of
ZIP -> case BSB64.decode $ TE.encodeUtf8 d of
Left err -> panic $ T.pack "[addToCorpusWithForm] error decoding base64" err
Right decoded -> decoded
_ -> cs d
eDocs <- liftBase $ parse data'
case eDocs of
Right docs' -> do
let docs = splitEvery 500 $ take 1000000 docs'
......
......@@ -24,7 +24,7 @@ import Gargantext.API.Node.Corpus.New.File (FileType)
-------------------------------------------------------
data NewWithForm = NewWithForm
{ _wf_filetype :: !FileType
, _wf_data :: !Text
, _wf_data :: !Text -- NOTE for binary files, this represents base-64 data
, _wf_lang :: !(Maybe Lang)
, _wf_name :: !Text
} deriving (Eq, Show, Generic)
......
......@@ -10,7 +10,7 @@ packages:
#- 'deps/accelerate-utility'
docker:
enable: false
#enable: false
#enable: true
repo: 'cgenie/stack-build:lts-18.12-garg'
run-args:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment