diff --git a/src/Gargantext/API/Node/Corpus/Export/Utils.hs b/src/Gargantext/API/Node/Corpus/Export/Utils.hs index b3c2eab9d87646f8aa4b99353c428e7f3ece6bc6..4b7e749f1801bd467637ba96d71e55c64aafadb3 100644 --- a/src/Gargantext/API/Node/Corpus/Export/Utils.hs +++ b/src/Gargantext/API/Node/Corpus/Export/Utils.hs @@ -178,7 +178,7 @@ readCorpusSQLite :: ( CES.MonadMask m => CorpusSQLite -> m (Either Text CorpusSQLiteData) readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname, fpath) -> liftBase $ do - (info, corpusData, listData) <- S.withConnection fpath $ \conn -> do + (info, corpusData, listData, documents) <- S.withConnection fpath $ \conn -> do [S.Only version] <- S.query_ conn "SELECT value FROM info WHERE key = 'gargVersion'" [S.Only cId] <- S.query_ conn "SELECT value FROM info WHERE key = 'corpusId'" [S.Only lId] <- S.query_ conn "SELECT value FROM info WHERE key = 'listId'" @@ -188,18 +188,27 @@ readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname, [corpusData] <- S.query_ conn "SELECT name, hash, parent_id, hyperdata FROM corpus" [listData] <- S.query_ conn "SELECT name, parent_id, hyperdata FROM lists" -- [ngrams] <- S.query_ conn "SELECT context_id, terms, type_ FROM ngrams" - -- [documents] <- S.query_ conn "SELECT context_id, name, date, hyperdata FROM documents" + documents <- S.query_ conn "SELECT context_id, name, date, hyperdata FROM documents" - pure (info, corpusData, listData) --, ngrams, documents) + pure (info, corpusData, listData, documents) let (version, cId, lId, created) = info let (_csd_corpus_name, _csd_corpus_hash, corpusParent, corpusHyperdata) = corpusData let (_csd_list_name, listParent, listHyperdata) = listData + + let parseCtx (ctxId, name, date, hd) = + case ( iso8601ParseM date, Aeson.decode hd ) of + ( Just d, Just h ) -> Right ( UnsafeMkNodeId ctxId, name, d, h ) + _ -> Left ("Context " <> show ctxId <> " parse error" :: Text) + + let (context_errors, _csd_contexts) = partitionEithers (parseCtx <$> documents) + case ( readP_to_S parseVersion version , iso8601ParseM created , Aeson.decode corpusHyperdata - , Aeson.decode listHyperdata ) of - ([(_csd_version, _)], Just _csd_created, Just _csd_corpus_hyperdata, Just _csd_list_hyperdata) -> do + , Aeson.decode listHyperdata + , context_errors ) of + ([(_csd_version, _)], Just _csd_created, Just _csd_corpus_hyperdata, Just _csd_list_hyperdata, []) -> do let _csd_cId = UnsafeMkNodeId cId let _csd_lId = UnsafeMkNodeId lId @@ -208,7 +217,6 @@ readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname, let _csd_list_parent = UnsafeMkNodeId <$> listParent -- TODO - let _csd_contexts = [] let _csd_map_context_ngrams = Map.empty let _csd_stop_context_ngrams = Map.empty let _csd_candidate_context_ngrams = Map.empty diff --git a/test/Test/API/Export.hs b/test/Test/API/Export.hs index 9526842865806b2a48e1c6f7aff3833e94476ef1..c808b86cd7cb101a32763940fcd8be590ead5d2a 100644 --- a/test/Test/API/Export.hs +++ b/test/Test/API/Export.hs @@ -31,7 +31,7 @@ import Test.API.Prelude (checkEither) import Test.API.Routes (get_corpus_sqlite_export) import Test.API.Setup (withTestDBAndPort, dbEnvSetup, SpecContext (..)) import Test.API.UpdateList (createFortranDocsList) -import Test.Database.Operations.DocumentSearch (exampleDocument_01) +import Test.Database.Operations.DocumentSearch (exampleDocument_01, exampleDocument_02) import Test.Database.Types (runTestMonad) import Test.Hspec import Test.Hspec.Wai.Internal (withApplication) @@ -39,7 +39,7 @@ import Test.Utils (withValidLogin) tests :: Spec -tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do +tests = sequential $ around withTestDBAndPort $ beforeWith dbEnvSetup $ do describe "Export API" $ do describe "Check CorpusSQLiteData creation" $ do it "correctly creates CorpusSQLiteData" $ \ctx -> do @@ -52,7 +52,7 @@ tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do aliceListId <- getOrMkList corpusId aliceUserId corpus <- getNodeWith corpusId (Proxy @HyperdataCorpus) - let docs = [ exampleDocument_01 ] + let docs = [ exampleDocument_01, exampleDocument_02 ] let lang = EN nlpServer <- view (nlpServerGet lang) @@ -64,7 +64,7 @@ tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do _csd_version `shouldBe` PG.version _csd_cId `shouldBe` corpusId _csd_lId `shouldBe` aliceListId - length _csd_contexts `shouldBe` 1 + length _csd_contexts `shouldBe` 2 describe "GET /api/v1.0/corpus/cId/sqlite" $ do it "returns correct SQLite db" $ \ctx -> do