[TEST] fix tests (WIP)

package.yaml

@@ -6,15 +6,15 @@ category: Data
 author: Gargantext Team
 maintainer: team@gargantext.org
 copyright:
-    - ! 'Copyright: (c) 2017-2018: see git logs and README'
-license: BSD3
+    - ! 'Copyright: (c) 2017-Present: see git logs and README'
+license: AGPL-3
 homepage: https://gargantext.org
 ghc-options: -Wall
 extra-libraries:
-- gfortran
+  - gfortran
 dependencies:
-- extra
-- text
+  - extra
+  - text
 default-extensions:
   - DataKinds
   - DeriveGeneric
@@ -57,6 +57,7 @@ library:
   - Gargantext.Text
   - Gargantext.Text.Context
   - Gargantext.Text.Corpus.Parsers
+  - Gargantext.Text.Corpus.Parsers.Date.Parsec
   - Gargantext.Text.Corpus.API
   - Gargantext.Text.Corpus.Parsers.CSV
   - Gargantext.Text.Examples
@@ -362,36 +363,58 @@ executables:
 
 
 tests:
-#   garg-test:
-#     main: Main.hs
-#     source-dirs: src-test
-#     ghc-options:
-#     - -threaded
-#     - -rtsopts
-#     - -with-rtsopts=-N
-#     dependencies:
-#     - base
-#     - gargantext
-#     - hspec
-#     - QuickCheck
-#     - quickcheck-instances 
-#     - time
-#     - parsec
-#     - duckling
-#     - text
-  garg-doctest:
-    main: Main.hs
-    source-dirs: src-doctest
-    ghc-options:
-    - -O2
-    - -Wcompat
-    - -Wmissing-signatures
-    - -rtsopts
-    - -threaded
-    - -with-rtsopts=-N
-    dependencies:
-    - doctest
-    - Glob
-    - QuickCheck
-    - base
-    - gargantext
+   garg-test:
+     main: Main.hs
+     source-dirs: src-test
+     default-extensions:
+     - DataKinds
+     - DeriveGeneric
+     - FlexibleContexts
+     - FlexibleInstances
+     - GeneralizedNewtypeDeriving
+     - MultiParamTypeClasses
+     - NoImplicitPrelude
+     - OverloadedStrings
+     - RankNTypes
+     ghc-options:
+     - -threaded
+     - -rtsopts
+     - -with-rtsopts=-N
+     dependencies:
+     - base
+     - gargantext
+     - hspec
+     - QuickCheck
+     - quickcheck-instances
+     - time
+     - parsec
+     - duckling
+     - text
+#  garg-doctest:
+#    main: Main.hs
+#    source-dirs: src-doctest
+#    ghc-options:
+#    - -O2
+#    - -Wcompat
+#    - -Wmissing-signatures
+#    - -rtsopts
+#    - -threaded
+#    - -with-rtsopts=-N
+#    dependencies:
+#    - doctest
+#    - Glob
+#    - QuickCheck
+#    - base
+#    - gargantext
+#    default-extensions:
+#    - DataKinds
+#    - DeriveGeneric
+#    - FlexibleContexts
+#    - FlexibleInstances
+#    - GeneralizedNewtypeDeriving
+#    - MultiParamTypeClasses
+#    - NoImplicitPrelude
+#    - OverloadedStrings
+#    - RankNTypes
+#
+

src-doctest/Main.hs

 import System.FilePath.Glob
 import Test.DocTest
+import Gargantext.Prelude
 
 main :: IO ()
 main = glob "src/Gargantext/" >>= doctest

src-test/Main.hs

@@ -12,15 +12,15 @@ Portability : POSIX
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
 --import qualified Ngrams.Lang.Fr as Fr
-import qualified Ngrams.Lang as Lang
+--import qualified Ngrams.Lang as Lang
 import qualified Ngrams.Lang.Occurrences as Occ
 import qualified Ngrams.Metrics as Metrics
 import qualified Parsers.Date as PD
 
 main :: IO ()
 main = do
-    Occ.parsersTest
-    Lang.ngramsExtractionTest FR
-    Lang.ngramsExtractionTest EN
-    Metrics.main
+--    Occ.parsersTest
+--    Lang.ngramsExtractionTest FR
+--    Lang.ngramsExtractionTest EN
+--    Metrics.main
     PD.testFromRFC3339

src-test/Ngrams/Lang.hs

@@ -15,6 +15,7 @@ commentary with @some markup@.
 
 module Ngrams.Lang where
 
+{-
 import Gargantext.Prelude (IO())
 
 import Gargantext.Core (Lang(..))
@@ -24,4 +25,4 @@ import qualified Ngrams.Lang.En as En
 ngramsExtractionTest :: Lang -> IO ()
 ngramsExtractionTest FR = Fr.ngramsExtractionTest
 ngramsExtractionTest EN = En.ngramsExtractionTest
-
+-}

src-test/Ngrams/Lang/En.hs

@@ -15,6 +15,7 @@ commentary with @some markup@.
 
 module Ngrams.Lang.En where
 
+{-
 import Data.List ((!!))
 import Data.Text (Text)
 
@@ -22,8 +23,11 @@ import Test.Hspec
 
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
-import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
 
+-- TODO this import is not used anymore
+import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
+-- use instead
+-- import Gargantext.Text.Terms (extractNgramsT)
 
 ngramsExtractionTest :: IO ()
 ngramsExtractionTest = hspec $ do
@@ -43,4 +47,4 @@ ngramsExtractionTest = hspec $ do
             t2 <- map (selectNgrams EN) <$> extractNgrams EN t
             t2 `shouldBe` [[("Donald Trump","NNP","PERSON"),("president of the United-States of America","NN","LOCATION")]]
 
-
+-}

src-test/Ngrams/Lang/Fr.hs

@@ -15,12 +15,15 @@ commentary with @some markup@.
 
 module Ngrams.Lang.Fr where
 
+{-
 import Test.Hspec
 
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
+-- TODO this import is not used anymore
 import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
-
+-- use instead
+-
 ngramsExtractionTest :: IO ()
 ngramsExtractionTest = hspec $ do
     describe "Behavioral tests: ngrams extraction in French Language" $ do
@@ -61,4 +64,4 @@ ngramsExtractionTest = hspec $ do
             let textFr1 = "L'heure d'arrivée des coureurs dépend de la météo du jour."
             testFr1 <- map (selectNgrams FR) <$> (extractNgrams FR) textFr1
             testFr1 `shouldBe` [[("heure d' arrivée des coureurs","NC","O"),("météo du jour","NC","O")]]
-
+-}

src-test/Ngrams/Lang/Occurrences.hs

@@ -15,6 +15,7 @@ commentary with @some markup@.
 
 module Ngrams.Lang.Occurrences where
 
+{-
 import Test.Hspec
 
 import Data.Either (Either(Right))
@@ -59,4 +60,4 @@ parsersTest = hspec $ do
 --  describe "Parser for nodes" $ do
 --    it "returns the result of one parsing after space" $ do
 --      occOfCorpus 249509 "sciences" `shouldReturn` 7
-
+-}

src-test/Ngrams/Metrics.hs

@@ -15,8 +15,10 @@ commentary with @some markup@.
 
 {-# LANGUAGE CPP                  #-}
 
-module Ngrams.Metrics (main) where
+--module Ngrams.Metrics (main) where
+module Ngrams.Metrics  where
 
+{-
 import           Data.Text (Text)
 import qualified Data.Text as T
 import           Data.Ratio
@@ -139,3 +141,5 @@ testPair :: (Eq a, Show a)
   -> SpecWith ()
 testPair f a b r = it ("‘" <> T.unpack a <> "’ and ‘" <> T.unpack b <> "’") $
   f a b `shouldBe` r
+
+-}

src-test/Parsers/Date.hs

@@ -28,7 +28,7 @@ import Duckling.Time.Types (toRFC3339)
 
 -----------------------------------------------------------
 import Gargantext.Prelude
-import Gargantext.Text.Parsers.Date (fromRFC3339)
+import Gargantext.Text.Corpus.Parsers.Date.Parsec (fromRFC3339)
 import Parsers.Types
 -----------------------------------------------------------
 

src/Gargantext/API/Node/Update.hs

@@ -81,7 +81,7 @@ updateNode uId nId (UpdateNodeParamsGraph metric) logStatus = do
 
   logStatus JobLog { _scst_succeeded = Just 1
                    , _scst_failed    = Just 0
-                   , _scst_remaining = Just 2
+                   , _scst_remaining = Just 1
                    , _scst_events    = Just []
                    }
 

src/Gargantext/Viz/Graph/API.hs

@@ -150,8 +150,9 @@ computeGraph cId d nt repo = do
   lIds <- selectNodesWithUsername NodeList userMaster
   let ngs = filterListWithRoot MapTerm $ mapTermListRoot [lId] nt repo
 
+  -- TODO split diagonal
   myCooc <- Map.filter (>1)
-         <$> getCoocByNgrams (Diagonal True)
+         <$> getCoocByNgrams (Diagonal False)
          <$> groupNodesByNgrams ngs
          <$> getNodesByNgramsOnlyUser cId (lIds <> [lId]) nt (Map.keys ngs)
 

src/Gargantext/Viz/Graph/Distances.hs

@@ -27,6 +27,7 @@ import Test.QuickCheck.Arbitrary
 
 ------------------------------------------------------------------------
 data Distance = Conditional | Distributional
+  deriving (Show)
 
 measure :: Distance -> Matrix Int -> Matrix Double
 measure Conditional    = measureConditional

src/Gargantext/Viz/Graph/Distances/Matrice.hs

@@ -88,13 +88,19 @@ dim m = n
 
 -- | Sum of a Matrix by Column
 --
--- >>> run $ matSum 3 (use $ matrix 3 [1..])
+-- >>> run $ matSumCol 3 (use $ matrix 3 [1..])
 -- Matrix (Z :. 3 :. 3)
 --   [ 12.0, 15.0, 18.0,
 --     12.0, 15.0, 18.0,
 --     12.0, 15.0, 18.0]
-matSum :: Dim -> Acc (Matrix Double) -> Acc (Matrix Double)
-matSum r mat = replicate (constant (Z :. (r :: Int) :. All)) $ sum $ transpose mat
+matSumCol :: Dim -> Acc (Matrix Double) -> Acc (Matrix Double)
+matSumCol r mat = replicate (constant (Z :. (r :: Int) :. All)) $ sum $ transpose mat
+
+matSumCol' :: Matrix Double -> Matrix Double
+matSumCol' m = run $ matSumCol n m'
+  where
+    n  = dim m
+    m' = use m
 
 
 -- | Proba computes de probability matrix: all cells divided by thee sum of its column
@@ -106,7 +112,7 @@ matSum r mat = replicate (constant (Z :. (r :: Int) :. All)) $ sum $ transpose m
 --       0.3333333333333333,  0.3333333333333333,  0.3333333333333333,
 --       0.5833333333333334,  0.5333333333333333,                 0.5]
 matProba :: Dim -> Acc (Matrix Double) -> Acc (Matrix Double)
-matProba r mat = zipWith (/) mat (matSum r mat)
+matProba r mat = zipWith (/) mat (matSumCol r mat)
 
 -- | Diagonal of the matrix
 --
@@ -164,7 +170,9 @@ matFilter t m = map (\x -> ifThenElse (x > (constant t)) x 0) (transpose m)
 -- interactions of 2 terms in the corpus.
 measureConditional :: Matrix Int -> Matrix Double
 --measureConditional m = run (matMiniMax $ matProba (dim m) $ map fromIntegral $ use m)
-measureConditional m = run (matProba (dim m) $ map fromIntegral $ use m)
+measureConditional m = run $ matProba (dim m)
+                           $ map fromIntegral
+                           $ use m
 
 
 -- *** Conditional distance (advanced)
@@ -196,9 +204,9 @@ conditional' m = ( run $ ie $ map fromIntegral $ use m
     r = dim m
 
     xs :: Acc (Matrix Double) -> Acc (Matrix Double)
-    xs mat = zipWith (-) (matSum r $ matProba r mat) (matProba r mat)
+    xs mat = zipWith (-) (matSumCol r $ matProba r mat) (matProba r mat)
     ys :: Acc (Matrix Double) -> Acc (Matrix Double)
-    ys mat = zipWith (-) (matSum r $ transpose $ matProba r mat) (matProba r mat)
+    ys mat = zipWith (-) (matSumCol r $ transpose $ matProba r mat) (matProba r mat)
 
 -----------------------------------------------------------------------
 -- ** Distributional Distance
@@ -206,11 +214,11 @@ conditional' m = ( run $ ie $ map fromIntegral $ use m
 -- | Distributional Distance Measure
 --
 -- Distributional measure is a relative measure which depends on the
--- selected list, it represents structural equivalence.
+-- selected list, it represents structural equivalence of mutual information.
 --
 -- The distributional measure P(c) of @i@ and @j@ terms is: \[
 -- S_{MI} = \frac {\sum_{k \neq i,j ; MI_{ik} >0}^{} \min(MI_{ik},
--- MI_{jk})}{\sum_{k \neq i,j ; MI_{ik}}^{}} \]
+-- MI_{jk})}{\sum_{k \neq i,j ; MI_{ik}>0}^{}} \]
 --
 -- Mutual information
 -- \[S_{MI}({i},{j}) = \log(\frac{C{ij}}{E{ij}})\]
@@ -228,26 +236,43 @@ conditional' m = ( run $ ie $ map fromIntegral $ use m
 --            \[N_{m} = \sum_{i,i \neq i}^{m} \sum_{j, j \neq j}^{m} S_{ij}\]
 --
 distributional :: Matrix Int -> Matrix Double
-distributional m = run $ matMiniMax $ ri (map fromIntegral $ use m)
+distributional m = run $ matMiniMax
+                       $ ri
+                       $ map fromIntegral  -- ^ from Int to Double
+                       $ use m             -- ^ push matrix in Accelerate type
   where
-    
     -- filter  m = zipWith (\a b -> max a b) m (transpose m)
-    
+
+    ri :: Acc (Matrix Double) -> Acc (Matrix Double)
     ri mat = zipWith (/) mat1 mat2
       where
-        mat1 = matSum n $ zipWith min (s_mi mat) (s_mi $ transpose mat)
-        mat2 = matSum n mat
-    
-    s_mi    m'  = zipWith (\a b -> log (a/b))  m'
-              $ zipWith (/) (crossProduct m') (total m')
+        mat1 = matSumCol n $ zipWith min (s_mi mat) (s_mi $ transpose mat)
+        mat2 = matSumCol n mat
+
+    s_mi :: Acc (Matrix Double) -> Acc (Matrix Double)
+    s_mi m' = zipWith (\a b -> log (a/b))  m'
+            $ zipWith (/) (crossProduct m') (total m')
 
-    total m'' = replicate (constant (Z :. n :. n)) $ fold (+) 0 $ fold (+) 0 m''
+    total :: Acc (Matrix Double) -> Acc (Matrix Double)
+    total = replicate (constant (Z :. n :. n)) . sum . sum
 
     n :: Dim
     n = dim m
-    
+
+    crossProduct :: Acc (Matrix Double) -> Acc (Matrix Double)
     crossProduct m''' = zipWith (*) (cross m'''  ) (cross (transpose m'''))
-    cross mat         = zipWith (-) (matSum n mat) (mat)
+    cross :: Acc (Matrix Double) -> Acc (Matrix Double)
+    cross mat         = zipWith (-) (matSumCol n mat) (mat)
+
+-- | cross
+{-
+cross :: Matrix Double -> Matrix Double
+cross mat = run $ zipWith (-) (matSumCol n mat') (mat')
+  where
+    mat' = use mat
+    n = dim mat
+-}
+
 
 -----------------------------------------------------------------------
 -----------------------------------------------------------------------

src/Gargantext/Viz/Graph/Tools.hs

@@ -56,6 +56,7 @@ cooc2graph :: Distance
            -> (Map (Text, Text) Int)
            -> IO Graph
 cooc2graph distance threshold myCooc = do
+  printDebug "cooc2graph" distance
   let
     (ti, _) = createIndices myCooc
     myCooc' = toIndex ti myCooc