Make tests play nice with the changes

Not mergeable: the tests fail for now. I suspect the problem comes from
the tests for `testCorrectFile`, where the function `checkFromLine` tries
to look up a wrong line number. Some more investigation would be needed
to figure out exactly what happens (probably should show the TSV file
being tested).

Anyway, here are the changes in this commit:
- Encapsulation of the TSV module in an `Internal` module, so we can export
  just the interface, but still expose the internals to the tests
- Correction of type errors in the tests to have something that typechecks
  with the changes introduced before

gargantext.cabal

@@ -197,6 +197,7 @@ library
       Gargantext.Core.Text.Corpus.Parsers
       Gargantext.Core.Text.Corpus.Parsers.Date
       Gargantext.Core.Text.Corpus.Parsers.TSV
+      Gargantext.Core.Text.Corpus.Parsers.TSV.Internal
       Gargantext.Core.Text.Corpus.Query
       Gargantext.Core.Text.List
       Gargantext.Core.Text.List.Formats.TSV

test/Test/Core/Text/Corpus/TSV.hs

@@ -2,19 +2,16 @@
 
 module Test.Core.Text.Corpus.TSV (tests) where
 
-import Gargantext.Core.Text.Corpus.Parsers.TSV
-import Test.QuickCheck
-import Test.QuickCheck.Instances ()
-import Data.ByteString.Lazy.UTF8 as BLU
+import Gargantext.Core.Text.Corpus.Parsers.TSV.Internal
 import Data.ByteString.Lazy as BL
-import Data.Char ( ord )
+import Data.ByteString.Lazy.UTF8 as BLU
+import Data.Char (ord)
 import Data.Text as DT (Text, pack, null, elem)
 import Data.Text.Encoding as DT
-
 import Prelude
-
+import Test.QuickCheck
+import Test.QuickCheck.Instances ()
 import Test.Tasty
-import Test.Tasty.HUnit
 import Test.Tasty.QuickCheck hiding (Positive, Negative)
 
 tests :: TestTree
@@ -30,12 +27,12 @@ tests = testGroup "TSV Parser" [
 
 
 
-delimiterBS :: Delimiter -> BL.ByteString
-delimiterBS Tab   = BLU.fromString "\t"
-delimiterBS Comma = BLU.fromString ","
-delimiterBS Line = BLU.fromString "\n"
+delimiterBS :: ColumnDelimiter -> BL.ByteString
+delimiterBS Tab       = BLU.fromString "\t"
+delimiterBS Comma     = BLU.fromString ","
+delimiterBS Semicolon = BLU.fromString ";"
 
-data File = File { fDelimiter :: Delimiter
+data File = File { fDelimiter :: ColumnDelimiter
                  , allCorpus  :: [RandomCorpus]
                  } 
                  deriving (Show)
@@ -56,19 +53,19 @@ instance Arbitrary File where
 
 arbitrarySizedFile :: Int -> Gen File
 arbitrarySizedFile m = do
-  del <- elements [Tab, Comma]
+  del  <- elements [Tab, Comma, Semicolon]
   corp <- vectorOf m (generateRandomCorpus)
   return (File del corp)
 
-delimiterToText :: Delimiter -> Text
-delimiterToText Tab   = DT.pack "\t"
-delimiterToText Comma = DT.pack ","
-delimiterToText Line  = DT.pack "\n"
+delimiterToText :: ColumnDelimiter -> Text
+delimiterToText Tab       = DT.pack "\t"
+delimiterToText Comma     = DT.pack ","
+delimiterToText Semicolon = DT.pack ";"
 
-delimiterToString :: Delimiter -> Char
-delimiterToString Tab   = '\t'
-delimiterToString Comma = ','
-delimiterToString Line  = '\n'  
+delimiterToString :: ColumnDelimiter -> Char
+delimiterToString Tab       = '\t'
+delimiterToString Comma     = ','
+delimiterToString Semicolon = ';'  
 
 textToBL :: Text -> BL.ByteString
 textToBL b = BL.fromChunks . return . DT.encodeUtf8 $ b
@@ -85,7 +82,7 @@ generateRandomCorpus = RandomCorpus
 
 generateFileDelimiter :: Gen File
 generateFileDelimiter = do
-  del <- elements [Tab, Comma]
+  del <- elements [Tab, Comma, Semicolon]
   m <- choose (1,5)
   corp <- vectorOf m (generateRandomCorpus)
   return (File del corp)
@@ -106,7 +103,7 @@ randomHeaderList = frequency [
   ]
 
 --TODO add delimiter
-createLineFromCorpus :: RandomCorpus -> Delimiter -> BL.ByteString
+createLineFromCorpus :: RandomCorpus -> ColumnDelimiter -> BL.ByteString
 createLineFromCorpus corpus delD = do
     let aut = (DT.pack "\"") <> (authors corpus) <> (DT.pack "\"")
     let tit = (DT.pack "\"") <> (title corpus) <> (DT.pack "\"")
@@ -118,7 +115,7 @@ createLineFromCorpus corpus delD = do
     let del = delimiterToText delD
     textToBL(pDay <> del <> pMonth <> del <> pYears <> del <> aut <> del <> tit <> del <> sou <> del <> abt)
 
-createLineFromCorpusWithNewLine :: RandomCorpus -> Delimiter -> BL.ByteString
+createLineFromCorpusWithNewLine :: RandomCorpus -> ColumnDelimiter -> BL.ByteString
 createLineFromCorpusWithNewLine corpus delD = do
     let aut = (DT.pack "\"") <> (authors corpus) <> (DT.pack "\"")
     let tit = (DT.pack "\"") <> (title corpus) <> (DT.pack "\"")
@@ -143,9 +140,9 @@ createFileWithNewLine file = do
     let allLines = BL.intercalate (BLU.fromString "\n") $ Prelude.map (\x -> createLineFromCorpus x (fDelimiter file)) (allCorpus file)
     headers <> (BLU.fromString "\n") <> allLines
 
-validRandomCorpus :: RandomCorpus -> Delimiter -> Bool
+validRandomCorpus :: RandomCorpus -> ColumnDelimiter -> Bool
 validRandomCorpus tsv del
-    | BL.length (BL.filter (==delimiter del) (createLineFromCorpus tsv del)) > 3= True
+    | BL.length (BL.filter (== toWord8 del) (createLineFromCorpus tsv del)) > 3= True
     | DT.null $ abstract tsv                                                    = True
     | DT.null $ title tsv                                                       = True
     | DT.null $ authors tsv                                                     = True
@@ -161,7 +158,7 @@ testValidNumber :: Property
 testValidNumber = forAll generateNumber (\s -> do
     let nbText = DT.pack $ show s
     let bl = textToBL nbText
-    case validNumber bl nbText 1 [] of
+    case checkNumber bl nbText 1 of
         Right _                 -> True
         Left  _ | BL.empty == bl -> True
                 | s < 1 -> True
@@ -171,7 +168,7 @@ testValidNumber = forAll generateNumber (\s -> do
 testValidText :: Property
 testValidText = forAll generateString (\s -> 
     let bl = textToBL s in
-        case validTextField bl s 1 [] of
+        case checkTextField bl s 1 of
             Right _                 -> True
             Left  _ | BL.empty == bl -> True
                     | (fromIntegral $ ord '\"') `BL.elem` bl -> True
@@ -184,15 +181,11 @@ testTestErrorPerLine = forAll generateRandomCorpus (\tsv -> do
     let del = Tab
     let line = createLineFromCorpus tsv del
     let headers = Prelude.map DT.pack ["Publication Day", "Publication Month", "Publication Year", "Authors", "Title", "Source", "Abstract"]
-    let splitLine = BL.splitWith (==delimiter del) line
-    case testErrorPerLine splitLine del headers 1 [] of
+    let splitLine = BL.splitWith (== toWord8 del) line
+    case checkRow splitLine del headers 1 of
         Right _ -> True
         Left  _ -> validRandomCorpus tsv del)
 
---check : 
--- True Del
--- False Error
-
 -- Test if a file is OK
 testTestCorrectFile :: Property
 testTestCorrectFile = forAll generateFile (\file -> do
@@ -202,8 +195,8 @@ testTestCorrectFile = forAll generateFile (\file -> do
         Left _ -> Prelude.all (\x -> do 
             let del = fDelimiter file
             let headers = Prelude.map DT.pack ["Publication Day", "Publication Month", "Publication Year", "Authors", "Title", "Source", "Abstract"]
-            let splitLine = BL.splitWith (==delimiter del) $ createLineFromCorpus x del
-            case testErrorPerLine splitLine del headers 1 [] of
+            let splitLine = BL.splitWith (== toWord8 del) $ createLineFromCorpus x del
+            case checkRow splitLine del headers 1 of
                 Right _ -> True
                 Left  _ -> validRandomCorpus x del) (allCorpus file))
 
@@ -217,20 +210,20 @@ testTestCorrectFileWithNewLine = forAll generateFile (\file -> do
         Left _ -> Prelude.all (\x -> do 
             let del = fDelimiter file
             let headers = Prelude.map DT.pack ["Publication Day", "Publication Month", "Publication Year", "Authors", "Title", "Source", "Abstract"]
-            let splitLine = BL.splitWith (==delimiter del) $ createLineFromCorpus x del
-            case testErrorPerLine splitLine del headers 1 [] of
+            let splitLine = BL.splitWith (== toWord8 del) $ createLineFromCorpus x del
+            case checkRow splitLine del headers 1 of
                 Right _ -> True
                 Left  _ -> validRandomCorpus x del) (allCorpus file))
 
 testFindDelimiter :: Property
 testFindDelimiter = forAll generateFileDelimiter (\file -> do
     let tsv = createFile file
-    case findDelimiter tsv of
+    case findColumnDelimiter tsv of
         Right _ -> True
         Left _ -> do
             let line = Prelude.head $ allCorpus file
             let del = delimiterToString $ fDelimiter file
-            let delLine = delimiterToString Line
+            let delLine = '\n'
             del `DT.elem` (abstract line) || del `DT.elem` (authors line) || del `DT.elem` (title line) || del `DT.elem` (source line) || delLine `DT.elem` (abstract line) || delLine `DT.elem` (authors line) || delLine `DT.elem` (title line) || delLine `DT.elem` (source line))
 
 testGetHeader :: Property