[test] fixes to decodeUtf8 and ne_nre rewrite

src/Gargantext/API/Ngrams/Types.hs

@@ -22,6 +22,7 @@ import Control.Category ((>>>))
 import Control.Lens (makeLenses, makePrisms, Iso', iso, from, (.~), (.=), (?=), (#), to, folded, {-withIndex, ifolded,-} view, use, (^.), (^?), (%~), (.~), (%=), at, _Just, Each(..), itraverse_, both, forOf_, (?~), over)
 import Control.Monad.State
 import Data.Aeson hiding ((.=))
+import Data.Aeson qualified as Aeson
 import Data.Aeson.TH (deriveJSON)
 import Data.Foldable
 import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap
@@ -175,10 +176,34 @@ data NgramsElement =
                    , _ne_nre         :: NgramsRepoElement
                    }
       deriving (Ord, Eq, Show, Generic)
-
-deriveJSON (unPrefix "_ne_") ''NgramsElement
+    
+-- deriveJSON (unPrefix "_ne_") ''NgramsElement
 makeLenses ''NgramsElement
 
+instance ToJSON NgramsElement where
+  toJSON ne = object [
+      "ngrams" Aeson..= toJSON (ne ^. ne_ngrams)
+    , "occurrences" Aeson..= toJSON (ne ^. ne_occurrences)
+    , "size" Aeson..= toJSON (ne ^. ne_nre . nre_size)
+    , "list" Aeson..= toJSON (ne ^. ne_nre . nre_list)
+    , "root" Aeson..= toJSON (ne ^. ne_nre . nre_root)
+    , "parent" Aeson..= toJSON (ne ^. ne_nre . nre_parent)
+    , "children" Aeson..= toJSON (ne ^. ne_nre . nre_children)
+    ]
+instance FromJSON NgramsElement where
+  parseJSON = withObject "NgramsElement" $ \ne -> do
+    _ne_ngrams <- ne .: "ngrams"
+    _ne_occurrences <- ne .: "occurrences"
+    _nre_size <- ne .: "size"
+    _nre_list <- ne .: "list"
+    _nre_root <- ne .:? "root"
+    _nre_parent <- ne .:? "parent"
+    _nre_children <- ne .: "children"
+
+    let _ne_nre = NgramsRepoElement { .. }
+    
+    pure NgramsElement { .. }
+
 mkNgramsElement :: NgramsTerm
                 -> ListType
                 -> Maybe RootParent

test/Test/API/UpdateList.hs

@@ -83,7 +83,7 @@ pollUntilFinished :: HasCallStack
 pollUntilFinished tkn port mkUrlPiece = go 60
   where
    go :: Int -> JobPollHandle -> WaiSession () JobPollHandle
-   go 0 h  = panicTrace $ "pollUntilFinished exhausted attempts. Last found JobPollHandle: " <> decodeUtf8 (JSON.encode h)
+   go 0 h  = panicTrace $ "pollUntilFinished exhausted attempts. Last found JobPollHandle: " <> (T.decodeUtf8 $ BL.toStrict $ JSON.encode h)
    go n h = case _jph_status h == "IsPending" || _jph_status h == "IsRunning" of
     True -> do
       liftIO $ threadDelay 1_000_000
@@ -91,7 +91,7 @@ pollUntilFinished tkn port mkUrlPiece = go 60
       go (n-1) h'
     False
      | _jph_status h == "IsFailure"
-     -> panicTrace $ "JobPollHandle contains a failure: " <> decodeUtf8 (JSON.encode h)
+     -> panicTrace $ "JobPollHandle contains a failure: " <> T.decodeUtf8 (BL.toStrict $ JSON.encode h)
      | otherwise
      -> pure h
 
@@ -132,7 +132,8 @@ tests = sequential $ aroundAll withTestDBAndPort $ do
                                                     "size": 2,
                                                     "list": "MapTerm",
                                                     "occurrences": [],
-                                                    "children": []
+                                                    "children": [],
+                                                    "root": null
                                                 }
                                             ]
                                         } |]
@@ -173,7 +174,9 @@ tests = sequential $ aroundAll withTestDBAndPort $ do
                                            {"ngrams":"abelian group"
                                            ,"size":1
                                            ,"list":"MapTerm"
-                                           ,"occurrences":[],"children":[]}
+                                           ,"occurrences":[]
+                                           ,"children":[]
+                                           ,"root": null}
                                            ]
                                          } |]
             let getStopUrl = "/node/" +| listId |+ "/ngrams?ngramsType=Terms&listType=StopTerm&list="+| listId |+"&limit=50"

test/Test/Ngrams/Query.hs

@@ -2,6 +2,7 @@
 {-# LANGUAGE TypeApplications #-}
 module Test.Ngrams.Query (tests) where
 
+import Control.Lens
 import Control.Monad
 import Data.Coerce
 import Data.Map.Strict qualified as Map
@@ -269,7 +270,8 @@ test_paginationQuantum = do
   let elems = coerce @NgramsTable @[NgramsElement] $ _vc_data res
   length elems @?= 10
   forM_ elems $ \term ->
-    assertBool ("found " <> show (_ne_list term) <> " in: " <> show elems) (_ne_list term == MapTerm)
+    assertBool ("found " <> show (term ^. ne_nre . nre_list) <> " in: " <> show elems)
+               (term ^. ne_nre . nre_list == MapTerm)
   where
    searchQuery = NgramsSearchQuery {
                  _nsq_limit       = Limit 10

test/Test/Ngrams/Query/PaginationCorpus.hs

@@ -20,111 +20,113 @@ import Paths_gargantext
 implementationElem :: NgramsElement
 implementationElem = NgramsElement {
     _ne_ngrams = "implementation"
-  , _ne_size = 1
-  , _ne_list = MapTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3, 4, 5 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "code", "functions", "language", "programs" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = MapTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "code", "functions", "language", "programs" ]
+                                }
   }
 
 languagesElem :: NgramsElement
 languagesElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "languages"}
-  , _ne_size = 1
-  , _ne_list = MapTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2 , 3 , 4 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "approach", "use" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = MapTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "approach", "use" ]
+                                }
   }
 
 termsElem :: NgramsElement
 termsElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "terms"}
-  , _ne_size = 1
-  , _ne_list = MapTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2 , 3 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "algorithm", "evaluation", "monad", "programmers" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = MapTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "algorithm", "evaluation", "monad", "programmers" ] }
   }
 
 proofElem :: NgramsElement
 proofElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "proof"}
-  , _ne_size = 1
-  , _ne_list = MapTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "proofs" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = MapTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "proofs" ] }
   }
 
 sideEffectsElem :: NgramsElement
 sideEffectsElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "side effects"}
-  , _ne_size = 1
-  , _ne_list = StopTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3, 4, 5, 6 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = StopTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ ] }
   }
 
 ooElem :: NgramsElement
 ooElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "object oriented"}
-  , _ne_size = 1
-  , _ne_list = StopTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3, 4, 5 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "null pointer exception" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = StopTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "null pointer exception" ] }
   }
 
 javaElem :: NgramsElement
 javaElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "java"}
-  , _ne_size = 1
-  , _ne_list = StopTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "JVM" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = StopTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "JVM" ] }
   }
 
 pascalElem :: NgramsElement
 pascalElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "pascal"}
-  , _ne_size = 1
-  , _ne_list = StopTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "turbo", "borland" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = StopTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "turbo", "borland" ] }
   }
 
 haskellElem :: NgramsElement
 haskellElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "haskell"}
-  , _ne_size = 1
-  , _ne_list = CandidateTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3, 4, 5, 6, 7, 8 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = CandidateTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ ] }
   }
 
 concHaskellElem :: NgramsElement
 concHaskellElem = NgramsElement {
     _ne_ngrams = NgramsTerm {unNgramsTerm = "concurrent haskell"}
-  , _ne_size = 1
-  , _ne_list = CandidateTerm
   , _ne_occurrences = Set.fromList $ map UnsafeMkContextId [ 1, 2, 3, 4, 5 ]
-  , _ne_root = Nothing
-  , _ne_parent = Nothing
-  , _ne_children = mSetFromList [ "Simon Marlow" ]
+  , _ne_nre = NgramsRepoElement { _nre_size = 1
+                                , _nre_list = CandidateTerm
+                                , _nre_root = Nothing
+                                , _nre_parent = Nothing
+                                , _nre_children = mSetFromList [ "Simon Marlow" ] }
   }
 
 -- | A big (for the sake of the tests anyway) corpus which has