use ngrams endpoint multiple times instead of list endpoint to fetch ngrams table

src/Gargantext/Components/NgramsTable.purs

@@ -22,15 +22,14 @@ import Data.Set as Set
 import Data.Symbol (SProxy(..))
 import Data.Tuple (Tuple(..), snd)
 import Data.Tuple.Nested ((/\))
-import Debug.Trace (spy)
 import Effect (Effect)
 import Gargantext.Components.AutoUpdate (autoUpdateElt)
 import Gargantext.Components.Loader (loader)
 import Gargantext.Components.LoadingSpinner (loadingSpinner)
-import Gargantext.Components.NgramsTable.Core (CoreState, NgramsElement(..), NgramsPatch(..), NgramsTable, NgramsTablePatch, NgramsTerm, PageParams, PatchMap(..), Replace, Versioned(..), VersionedNgramsTable, _NgramsElement, _NgramsTable, _PatchMap, _children, _list, _ngrams, _occurrences, _root, addNewNgram, applyNgramsPatches, applyPatchSet, commitPatch, convOrderBy, fromNgramsPatches, initialPageParams, loadNgramsTable, loadNgramsTableAll, ngramsTermText, normNgram, patchSetFromMap, replace, rootsOf, singletonNgramsTablePatch, syncPatches)
+import Gargantext.Components.NgramsTable.Core (CoreState, NgramsElement(..), NgramsPatch(..), NgramsTable, NgramsTablePatch, NgramsTerm, PageParams, PatchMap(..), Replace, Versioned(..), VersionedNgramsTable, _NgramsElement, _NgramsTable, _PatchMap, _children, _list, _ngrams, _occurrences, _root, addNewNgram, applyNgramsPatches, applyPatchSet, commitPatch, convOrderBy, fromNgramsPatches, initialPageParams, loadNgramsTableAll, ngramsTermText, normNgram, patchSetFromMap, replace, rootsOf, singletonNgramsTablePatch, syncPatches)
 import Gargantext.Components.Table as T
 import Gargantext.Sessions (Session)
-import Gargantext.Types (CTabNgramType, Mode(..), OrderBy(..), TabType, TermList(..), readTermList, readTermSize, termLists, termSizes)
+import Gargantext.Types (CTabNgramType, OrderBy(..), TabType, TermList(..), readTermList, readTermSize, termLists, termSizes)
 import Gargantext.Utils (queryMatchesLabel)
 import Gargantext.Utils.Reactix as R2
 import Prelude (class Show, Unit, bind, const, discard, identity, map, mempty, not, pure, show, unit, (#), ($), (&&), (+), (/=), (<$>), (<<<), (<>), (=<<), (==), (||), otherwise, when)
@@ -416,11 +415,9 @@ mainNgramsTableCpt = R.hooksComponent "MainNgramsTable" cpt
       path /\ setPath <- R.useState' $ initialPageParams session nodeId [defaultListId] tabType
       let paint versioned = loadedNgramsTable' {tabNgramType, path: path /\ setPath, versioned}
 
-      -- TODO: get rid of ngramsTble loading
-      -- pure $ loader path loadNgramsTable paint
       pure $ loader path loadNgramsTableAll \loaded -> do
-        case Map.lookup Sources loaded of
-          Just versioned -> paint versioned
+        case Map.lookup tabType loaded of
+          Just (versioned :: VersionedNgramsTable) -> paint versioned
           Nothing -> loadingSpinner {}
 
 type NgramsDepth = {ngrams :: NgramsTerm, depth :: Int}

src/Gargantext/Components/NgramsTable/Core.purs

@@ -54,10 +54,9 @@ import Prelude
 
 import Control.Monad.Cont.Trans (lift)
 import Control.Monad.State (class MonadState, execState)
-import Data.Argonaut (class DecodeJson, class EncodeJson, Json, decodeJson, encodeJson, jsonEmptyObject, (.:), (.:!), (:=), (~>))
+import Data.Argonaut (class DecodeJson, class EncodeJson, decodeJson, encodeJson, jsonEmptyObject, (.:), (.:!), (:=), (~>))
 import Data.Array (head)
 import Data.Array as A
-import Data.Array as Array
 import Data.Bifunctor (lmap)
 import Data.Either (Either(..))
 import Data.Foldable (class Foldable, foldMap, foldl, foldr)
@@ -73,7 +72,7 @@ import Data.Lens.Record (prop)
 import Data.List ((:), List(Nil))
 import Data.Map (Map)
 import Data.Map as Map
-import Data.Maybe (Maybe(..), fromMaybe, isNothing, maybe)
+import Data.Maybe (Maybe(..), isNothing, maybe)
 import Data.Newtype (class Newtype)
 import Data.Set (Set)
 import Data.Set as Set
@@ -81,15 +80,15 @@ import Data.String as S
 import Data.String.Regex (Regex, regex, replace) as R
 import Data.String.Regex.Flags (global, multiline) as R
 import Data.Symbol (SProxy(..))
-import Data.Traversable (class Traversable, traverse, traverse_, sequence)
+import Data.Traversable (class Traversable, for, sequence, traverse, traverse_)
 import Data.TraversableWithIndex (class TraversableWithIndex, traverseWithIndex)
 import Data.Tuple (Tuple(..))
-import Effect.Aff (Aff, error, throwError)
+import Effect.Aff (Aff)
 import Foreign.Object as FO
 import Gargantext.Components.Table as T
 import Gargantext.Routes (SessionRoute(..))
 import Gargantext.Sessions (Session, get, put, post)
-import Gargantext.Types (CTabNgramType(..), Mode, OrderBy(..), ScoreType(..), TabType, TermList(..), TermSize, decodeMode)
+import Gargantext.Types (CTabNgramType(..), OrderBy(..), ScoreType(..), TabSubType(..), TabType(..), TermList(..), TermSize)
 import Gargantext.Utils.KarpRabin (indicesOfAny)
 import Partial (crashWith)
 import Partial.Unsafe (unsafePartial)
@@ -667,51 +666,23 @@ loadNgramsTable
                           , termListFilter, termSizeFilter
                           , searchQuery, scoreType } (Just nodeId)
 
-decodeLists :: FO.Object (Versioned Json) -> Either String NgramsList
-decodeLists obj = do
-  tuples :: Array (Tuple Mode VersionedNgramsTable) <- traverse decodeModeMember $ FO.toUnfoldable obj
-  pure $ Map.fromFoldable tuples
-  where
-    decodeModeMember :: Tuple String (Versioned Json) -> Either String (Tuple Mode VersionedNgramsTable)
-    decodeModeMember (Tuple key versioned) = do
-      mode <- decodeMode key
-      versionedNgramsTable <- decodeVersionedNgramsTable versioned
-      pure $ Tuple mode versionedNgramsTable
-
-    decodeVersionedNgramsTable :: Versioned Json -> Either String VersionedNgramsTable
-    decodeVersionedNgramsTable (Versioned r) = do
-      dataObj :: FO.Object { size :: Int, list :: TermList, children :: Set NgramsTerm } <- decodeJson r."data"
-      let
-        (tuples :: Array _) =
-          FO.toUnfoldable dataObj <#> \(Tuple key values) ->
-            let ngrams = NormNgramsTerm key
-            in Tuple ngrams $ mkNgramsElement ngrams values
-      pure $ Versioned r { "data" = NgramsTable $ Map.fromFoldable tuples }
-
-    mkNgramsElement :: NgramsTerm -> { size :: Int, list :: TermList, children :: Set NgramsTerm } -> NgramsElement
-    mkNgramsElement ngrams r = NgramsElement
-      { ngrams
-      , list: r.list
-      , children: r.children
-      , parent: Nothing
-      , root: Nothing
-      , occurrences: 1 -- TODO: to be fixed by using different route?
-      }
-
-type NgramsList = (Map Mode VersionedNgramsTable)
-
-loadNgramsTableAll :: PageParams -> Aff NgramsList
-loadNgramsTableAll
-  { nodeId, listIds, termListFilter, termSizeFilter, session, scoreType
-  , searchQuery, tabType, params: {offset, limit, orderBy}}
-  = do
-      obj <- get session query
-      case decodeLists obj of
-        Right x -> pure x
-        Left e -> throwError $ error $ "Failed to load ngrams list: " <> e
-
-     where query = ListsRoute listId
-           listId = fromMaybe 0 $ Array.head listIds -- listIds should be singleton list in main lists view
+type NgramsListByTabType = Map TabType VersionedNgramsTable
+
+loadNgramsTableAll :: PageParams -> Aff NgramsListByTabType
+loadNgramsTableAll { nodeId, listIds, session, scoreType } = do
+  let
+    cTagNgramTypes =
+      [ CTabTerms
+      , CTabSources
+      , CTabAuthors
+      , CTabInstitutes
+      ]
+    query tabType = GetNgramsTableAll { tabType, listIds, scoreType } (Just nodeId)
+
+  Map.fromFoldable <$> for cTagNgramTypes \cTagNgramType -> do
+    let tabType = TabCorpus $ TabNgramType cTagNgramType
+    result :: VersionedNgramsTable <- get session $ query tabType
+    pure $ Tuple tabType result
 
 convOrderBy :: T.OrderByDirection T.ColumnName -> OrderBy
 convOrderBy (T.ASC  (T.ColumnName "Score")) = ScoreAsc

src/Gargantext/Ends.purs

@@ -136,6 +136,12 @@ sessionPath (R.GetNgrams opts i)    =
     termSizeFilter MultiTerm = "&minTermSize=2"
     search "" = ""
     search s = "&search=" <> s
+sessionPath (R.GetNgramsTableAll opts i) =
+  sessionPath $ R.NodeAPI Node i
+     $ "ngrams?ngramsType="
+    <> showTabType' opts.tabType
+    <> foldMap (\x -> "&list=" <> show x) opts.listIds
+    <> limitUrl 100000
 sessionPath (R.ListDocument lId dId) =
   sessionPath $ R.NodeAPI NodeList lId ("document/" <> (show $ maybe 0 identity dId))
 sessionPath (R.ListsRoute lId) = "lists/" <> show lId

src/Gargantext/Routes.purs

 module Gargantext.Routes where
 
 import Prelude
+
 import Data.Maybe (Maybe)
-import Gargantext.Types (ChartOpts, CorpusMetricOpts, Id, Limit, ListId, NgramsGetOpts, NodeType, Offset, OrderBy, SearchOpts, SessionId, TabType, TermList)
+import Gargantext.Types (ChartOpts, CorpusMetricOpts, Id, Limit, ListId, NgramsGetOpts, NodeType, Offset, OrderBy, SearchOpts, SessionId, TabType, TermList, NgramsGetTableAllOpts)
 
 data AppRoute
   = Home
@@ -32,6 +33,7 @@ data SessionRoute
   = Tab TabType (Maybe Id)
   | Children NodeType Offset Limit (Maybe OrderBy) (Maybe Id)
   | GetNgrams NgramsGetOpts (Maybe Id)
+  | GetNgramsTableAll NgramsGetTableAllOpts (Maybe Id)
   | PutNgrams TabType (Maybe ListId) (Maybe TermList) (Maybe Id)
   -- ^ This name is not good. In particular this URL is used both in PUT and POST.
   | NodeAPI NodeType (Maybe Id) String

src/Gargantext/Types.purs

@@ -299,6 +299,12 @@ type NgramsGetOpts =
   , searchQuery    :: String
   }
 
+type NgramsGetTableAllOpts =
+  { tabType        :: TabType
+  , listIds        :: Array ListId
+  , scoreType      :: ScoreType
+  }
+
 type SearchOpts =
   { {-id :: Int
     , query    :: Array String
@@ -380,6 +386,7 @@ instance decodeJsonApiVersion :: DecodeJson ApiVersion where
 data CTabNgramType = CTabTerms | CTabSources | CTabAuthors | CTabInstitutes
 
 derive instance eqCTabNgramType :: Eq CTabNgramType
+derive instance ordCTabNgramType :: Ord CTabNgramType
 
 instance showCTabNgramType :: Show CTabNgramType where
   show CTabTerms      = "Terms"
@@ -390,6 +397,7 @@ instance showCTabNgramType :: Show CTabNgramType where
 data PTabNgramType = PTabPatents | PTabBooks | PTabCommunication
 
 derive instance eqPTabNgramType :: Eq PTabNgramType
+derive instance ordPTabNgramType :: Ord PTabNgramType
 
 instance showPTabNgramType :: Show PTabNgramType where
   show PTabPatents       = "Patents"
@@ -399,6 +407,7 @@ instance showPTabNgramType :: Show PTabNgramType where
 data TabSubType a = TabDocs | TabNgramType a | TabTrash | TabMoreLikeFav | TabMoreLikeTrash
 
 derive instance eqTabSubType :: Eq a => Eq (TabSubType a)
+derive instance ordTabSubType :: Ord a => Ord (TabSubType a)
 
 instance showTabSubType :: Show a => Show (TabSubType a) where
   show TabDocs          = "Docs"
@@ -413,6 +422,7 @@ data TabType
   | TabDocument (TabSubType CTabNgramType)    
 
 derive instance eqTabType :: Eq TabType
+derive instance ordTabType :: Ord TabType
 
 derive instance genericTabType :: Generic TabType _