[sqlite] add contexts fetching to CorpusSQLiteData

src/Gargantext/API/Node/Corpus/Export/Utils.hs

@@ -178,7 +178,7 @@ readCorpusSQLite :: ( CES.MonadMask m
                  => CorpusSQLite
                  -> m (Either Text CorpusSQLiteData)
 readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname, fpath) -> liftBase $ do
-  (info, corpusData, listData) <- S.withConnection fpath $ \conn -> do
+  (info, corpusData, listData, documents) <- S.withConnection fpath $ \conn -> do
     [S.Only version] <- S.query_ conn "SELECT value FROM info WHERE key = 'gargVersion'"
     [S.Only cId]     <- S.query_ conn "SELECT value FROM info WHERE key = 'corpusId'"
     [S.Only lId]     <- S.query_ conn "SELECT value FROM info WHERE key = 'listId'"
@@ -188,18 +188,27 @@ readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname,
     [corpusData] <- S.query_ conn "SELECT name, hash, parent_id, hyperdata FROM corpus"
     [listData]   <- S.query_ conn "SELECT name, parent_id, hyperdata FROM lists"
     -- [ngrams]     <- S.query_ conn "SELECT context_id, terms, type_ FROM ngrams"
-    -- [documents]  <- S.query_ conn "SELECT context_id, name, date, hyperdata FROM documents"
+    documents    <- S.query_ conn "SELECT context_id, name, date, hyperdata FROM documents"
 
-    pure (info, corpusData, listData)  --, ngrams, documents)
+    pure (info, corpusData, listData, documents)
 
   let (version, cId, lId, created) = info
   let (_csd_corpus_name, _csd_corpus_hash, corpusParent, corpusHyperdata) = corpusData
   let (_csd_list_name, listParent, listHyperdata) = listData
+
+  let parseCtx (ctxId, name, date, hd) =
+        case ( iso8601ParseM date, Aeson.decode hd ) of
+          ( Just d, Just h ) -> Right ( UnsafeMkNodeId ctxId, name, d, h )
+          _                  -> Left ("Context " <> show ctxId <> " parse error" :: Text)
+
+  let (context_errors, _csd_contexts) = partitionEithers (parseCtx <$> documents)
+
   case ( readP_to_S parseVersion version
        , iso8601ParseM created
        , Aeson.decode corpusHyperdata
-       , Aeson.decode listHyperdata ) of
-    ([(_csd_version, _)], Just _csd_created, Just _csd_corpus_hyperdata, Just _csd_list_hyperdata) -> do
+       , Aeson.decode listHyperdata
+       , context_errors ) of
+    ([(_csd_version, _)], Just _csd_created, Just _csd_corpus_hyperdata, Just _csd_list_hyperdata, []) -> do
       let _csd_cId = UnsafeMkNodeId cId
       let _csd_lId = UnsafeMkNodeId lId
 
@@ -208,7 +217,6 @@ readCorpusSQLite (CorpusSQLite { _cs_bs }) = withTempSQLiteDir $ \(_fp, _fname,
       let _csd_list_parent = UnsafeMkNodeId <$> listParent
 
       -- TODO
-      let _csd_contexts = []
       let _csd_map_context_ngrams = Map.empty
       let _csd_stop_context_ngrams = Map.empty
       let _csd_candidate_context_ngrams = Map.empty

test/Test/API/Export.hs

@@ -31,7 +31,7 @@ import Test.API.Prelude (checkEither)
 import Test.API.Routes (get_corpus_sqlite_export)
 import Test.API.Setup (withTestDBAndPort, dbEnvSetup, SpecContext (..))
 import Test.API.UpdateList (createFortranDocsList)
-import Test.Database.Operations.DocumentSearch (exampleDocument_01)
+import Test.Database.Operations.DocumentSearch (exampleDocument_01, exampleDocument_02)
 import Test.Database.Types (runTestMonad)
 import Test.Hspec
 import Test.Hspec.Wai.Internal (withApplication)
@@ -39,7 +39,7 @@ import Test.Utils (withValidLogin)
 
 
 tests :: Spec
-tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do
+tests = sequential $ around withTestDBAndPort $ beforeWith dbEnvSetup $ do
   describe "Export API" $ do
     describe "Check CorpusSQLiteData creation" $ do
       it "correctly creates CorpusSQLiteData" $ \ctx -> do
@@ -52,7 +52,7 @@ tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do
           aliceListId    <- getOrMkList corpusId aliceUserId
           corpus <- getNodeWith corpusId (Proxy @HyperdataCorpus)
   
-          let docs = [ exampleDocument_01 ]
+          let docs = [ exampleDocument_01, exampleDocument_02 ]
           
           let lang = EN
           nlpServer <- view (nlpServerGet lang)
@@ -64,7 +64,7 @@ tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do
             _csd_version `shouldBe` PG.version
             _csd_cId `shouldBe` corpusId
             _csd_lId `shouldBe` aliceListId
-            length _csd_contexts `shouldBe` 1
+            length _csd_contexts `shouldBe` 2
     
     describe "GET /api/v1.0/corpus/cId/sqlite" $ do
       it "returns correct SQLite db" $ \ctx -> do