Merge branch 'dev' into 162-dev-haskell-9.2

CHANGELOG.md

+## Version 0.0.6.9.9.4.2
+
+* [BACK][FIX][[Node terms] Random slowness on loading a page list of terms (#199)](https://gitlab.iscpif.fr/gargantext/haskell-gargantext/issues/199)
+* [FRONT][CLEAN] Removing API that do not fully work (yet)
+* [FRONT][FIX] Chat Link 
+
 ## Version 0.0.6.9.9.4.1
 
 

gargantext.cabal

@@ -5,7 +5,7 @@ cabal-version: 1.12
 -- see: https://github.com/sol/hpack
 
 name:           gargantext
-version: 0.0.6.9.9.4.1
+version: 0.0.6.9.9.4.2
 synopsis:       Search, map, share
 description:    Please see README.md
 category:       Data
@@ -25,6 +25,7 @@ data-files:
     ekg-assets/bootstrap-1.4.0.min.css
     ekg-assets/chart_line_add.png
     ekg-assets/cross.png
+    test-data/ngrams/GarganText_NgramsTerms-QuantumComputing.json
 
 library
   exposed-modules:
@@ -848,6 +849,8 @@ test-suite garg-test
       Ngrams.Lang.Occurrences
       Ngrams.Metrics
       Ngrams.NLP
+      Ngrams.Query
+      Ngrams.Query.PaginationCorpus
       Parsers.Date
       Parsers.Types
       Parsers.WOS
@@ -881,7 +884,10 @@ test-suite garg-test
   ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N
   build-depends:
       QuickCheck
+    , aeson
     , base
+    , bytestring
+    , containers
     , duckling
     , extra
     , gargantext
@@ -889,6 +895,8 @@ test-suite garg-test
     , hspec
     , parsec
     , quickcheck-instances
+    , tasty
+    , tasty-hunit
     , text
     , time
     , unordered-containers

package.yaml

@@ -6,7 +6,7 @@ name: gargantext
 #         | | | +-----  Layers * : New versions with API additions
 #         | | | | +---  Layers * : New versions without API breaking changes
 #         | | | | |
-version: '0.0.6.9.9.4.1'
+version: '0.0.6.9.9.4.2'
 synopsis: Search, map, share
 description: Please see README.md
 category: Data
@@ -43,6 +43,7 @@ data-files:
   - ekg-assets/bootstrap-1.4.0.min.css
   - ekg-assets/chart_line_add.png
   - ekg-assets/cross.png
+  - test-data/ngrams/GarganText_NgramsTerms-QuantumComputing.json
 library:
   source-dirs: src
   ghc-options:
@@ -503,7 +504,10 @@ tests:
      - -rtsopts
      - -with-rtsopts=-N
      dependencies:
+     - aeson
      - base
+     - bytestring
+     - containers
      - gargantext
      - gargantext-prelude
      - hspec
@@ -512,6 +516,8 @@ tests:
      - time
      - parsec
      - duckling
+     - tasty
+     - tasty-hunit
      - text
      - unordered-containers
    jobqueue-test:

server

@@ -9,5 +9,5 @@ LOGFILE=$FOLDER"/"$FILE
 
 mkdir -p $FOLDER
 
-env LANG=en_US.UTF-8 ~/.local/bin/gargantext-server --ini gargantext.ini --run Dev +RTS > $LOGFILE 2>&1 & tail -F $LOGFILE # -p
+env LANG=en_US.UTF-8 ~/.local/bin/gargantext-server --ini gargantext.ini --run Prod +RTS > $LOGFILE 2>&1 & tail -F $LOGFILE # -p
 #env LANG=en_US.UTF-8 stack --docker exec gargantext-server -- --ini gargantext.ini --run Dev +RTS > $LOGFILE 2>&1 & tail -F $LOGFILE # -p

src-test/Main.hs

@@ -10,14 +10,12 @@ Portability : POSIX
 
 
 import Gargantext.Prelude
-import Gargantext.Core (Lang(..))
 
 import qualified Core.Utils as Utils
 --import qualified Ngrams.Lang.Fr as Fr
 --import qualified Ngrams.Lang as Lang
-import qualified Ngrams.Lang.Occurrences as Occ
 import qualified Ngrams.NLP              as NLP
-import qualified Ngrams.Metrics          as Metrics
+import qualified Ngrams.Query            as NgramsQuery
 import qualified Parsers.Date            as PD
 -- import qualified Graph.Distance          as GD
 import qualified Graph.Clustering        as Graph
@@ -35,3 +33,4 @@ main = do
 --    GD.test
   Crypto.test
   NLP.main
+  NgramsQuery.main

src-test/Ngrams/Query/PaginationCorpus.hs

+{-# LANGUAGE ScopedTypeVariables #-}
+module Ngrams.Query.PaginationCorpus where
+
+import           Prelude
+
+import           Data.Aeson
+import           Data.Map.Strict (Map)
+import           Gargantext.API.Ngrams
+import           Gargantext.API.Ngrams.Types
+import           Gargantext.Core.Types.Main
+import           Gargantext.Database.Admin.Types.Node
+import           System.IO.Unsafe
+import qualified Data.ByteString as B
+import qualified Data.Map.Strict as Map
+import qualified Data.Set as Set
+
+import Paths_gargantext
+
+
+implementationElem :: NgramsElement
+implementationElem = NgramsElement {
+    _ne_ngrams = "implementation"
+  , _ne_size = 1
+  , _ne_list = MapTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3, NodeId 4, NodeId 5 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "code", "functions", "language", "programs" ]
+  }
+
+languagesElem :: NgramsElement
+languagesElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "languages"}
+  , _ne_size = 1
+  , _ne_list = MapTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2 , NodeId 3 , NodeId 4 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "approach", "use" ]
+  }
+
+termsElem :: NgramsElement
+termsElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "terms"}
+  , _ne_size = 1
+  , _ne_list = MapTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2 , NodeId 3 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "algorithm", "evaluation", "monad", "programmers" ]
+  }
+
+proofElem :: NgramsElement
+proofElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "proof"}
+  , _ne_size = 1
+  , _ne_list = MapTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "proofs" ]
+  }
+
+sideEffectsElem :: NgramsElement
+sideEffectsElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "side effects"}
+  , _ne_size = 1
+  , _ne_list = StopTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3, NodeId 4, NodeId 5, NodeId 6 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ ]
+  }
+
+ooElem :: NgramsElement
+ooElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "object oriented"}
+  , _ne_size = 1
+  , _ne_list = StopTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3, NodeId 4, NodeId 5 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "null pointer exception" ]
+  }
+
+javaElem :: NgramsElement
+javaElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "java"}
+  , _ne_size = 1
+  , _ne_list = StopTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "JVM" ]
+  }
+
+pascalElem :: NgramsElement
+pascalElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "pascal"}
+  , _ne_size = 1
+  , _ne_list = StopTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "turbo", "borland" ]
+  }
+
+haskellElem :: NgramsElement
+haskellElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "haskell"}
+  , _ne_size = 1
+  , _ne_list = CandidateTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3, NodeId 4, NodeId 5, NodeId 6, NodeId 7, NodeId 8 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ ]
+  }
+
+concHaskellElem :: NgramsElement
+concHaskellElem = NgramsElement {
+    _ne_ngrams = NgramsTerm {unNgramsTerm = "concurrent haskell"}
+  , _ne_size = 1
+  , _ne_list = CandidateTerm
+  , _ne_occurrences = Set.fromList [ NodeId 1, NodeId 2, NodeId 3, NodeId 4, NodeId 5 ]
+  , _ne_root = Nothing
+  , _ne_parent = Nothing
+  , _ne_children = mSetFromList [ "Simon Marlow" ]
+  }
+
+-- | A big (for the sake of the tests anyway) corpus which has
+-- * 4 @MapTerm@s
+-- * 4 @StopTerm@s
+-- * 2 @CandidateTerm@s
+paginationCorpus :: Versioned (Map NgramsTerm NgramsElement)
+paginationCorpus = Versioned 0 $ Map.fromList [
+    -- Map terms
+    ( "implementation", implementationElem)
+  , ( "languages", languagesElem)
+  , ( "terms", termsElem)
+  , ("proof", proofElem)
+
+    -- Stop terms
+  , ("side effects", sideEffectsElem)
+  , ("object oriented", ooElem)
+  , ("java", javaElem)
+  , ("pascal", pascalElem)
+
+    -- Candidate terms
+  , ("haskell", haskellElem)
+  , ("concurrent haskell", concHaskellElem)
+  ]
+
+quantumComputingCorpus :: Versioned (Map NgramsTerm NgramsElement)
+quantumComputingCorpus = unsafePerformIO $ do
+  pth      <- getDataFileName "test-data/ngrams/GarganText_NgramsTerms-QuantumComputing.json"
+  jsonBlob <- B.readFile pth
+  case eitherDecodeStrict' jsonBlob of
+    Left err    -> error err
+    Right (Versioned ver (mp :: Map NgramsTerm NgramsRepoElement)) ->
+      pure $ Versioned ver (Map.mapWithKey (\k -> ngramsElementFromRepo k) mp)
+{-# NOINLINE quantumComputingCorpus #-}

src/Gargantext/API/Ngrams/Types.hs

@@ -41,6 +41,7 @@ import Database.PostgreSQL.Simple.ToField (ToField, toJSONField, toField)
 import GHC.Generics (Generic)
 import Gargantext.Core.Text (size)
 import Gargantext.Core.Types (ListType(..), ListId, NodeId, TODO)
+import Gargantext.Core.Types.Query (Limit, Offset, MaxSize, MinSize)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixUntagged, unPrefixSwagger, wellNamedSchema)
 import Gargantext.Database.Admin.Types.Node (ContextId)
 import Gargantext.Database.Prelude (fromField', HasConnectionPool, HasConfig, CmdM')
@@ -268,6 +269,43 @@ instance Arbitrary NgramsTable where
 
 instance ToSchema NgramsTable
 
+------------------------------------------------------------------------
+-- Searching in a Ngram Table
+
+data OrderBy = TermAsc | TermDesc | ScoreAsc | ScoreDesc
+             deriving (Generic, Enum, Bounded, Read, Show)
+
+instance FromHttpApiData OrderBy
+  where
+    parseUrlPiece "TermAsc"   = pure TermAsc
+    parseUrlPiece "TermDesc"  = pure TermDesc
+    parseUrlPiece "ScoreAsc"  = pure ScoreAsc
+    parseUrlPiece "ScoreDesc" = pure ScoreDesc
+    parseUrlPiece _           = Left "Unexpected value of OrderBy"
+
+instance ToHttpApiData OrderBy where
+  toUrlPiece = pack . show
+
+instance ToParamSchema OrderBy
+instance FromJSON  OrderBy
+instance ToJSON    OrderBy
+instance ToSchema  OrderBy
+instance Arbitrary OrderBy
+  where
+    arbitrary = elements [minBound..maxBound]
+
+
+-- | A query on a 'NgramsTable'.
+data NgramsSearchQuery = NgramsSearchQuery
+  { _nsq_limit       :: !Limit
+  , _nsq_offset      :: !(Maybe Offset)
+  , _nsq_listType    :: !(Maybe ListType)
+  , _nsq_minSize     :: !(Maybe MinSize)
+  , _nsq_maxSize     :: !(Maybe MaxSize)
+  , _nsq_orderBy     :: !(Maybe OrderBy)
+  , _nsq_searchQuery :: !(NgramsTerm -> Bool)
+  }
+
 ------------------------------------------------------------------------
 type NgramsTableMap = Map NgramsTerm NgramsRepoElement
 ------------------------------------------------------------------------

src/Gargantext/Core/Types/Query.hs

@@ -28,3 +28,12 @@ newtype Offset = Offset { getOffset :: Int }
                    , Servant.FromHttpApiData, Servant.ToHttpApiData
                    , Swagger.ToParamSchema, Swagger.ToSchema)
 type IsTrash  = Bool
+
+newtype MinSize = MinSize { getMinSize :: Int }
+  deriving newtype ( Show, Eq, Num
+                   , Servant.FromHttpApiData, Servant.ToHttpApiData
+                   , Swagger.ToParamSchema, Swagger.ToSchema)
+newtype MaxSize = MaxSize { getMaxSize :: Int }
+  deriving newtype ( Show, Eq, Num
+                   , Servant.FromHttpApiData, Servant.ToHttpApiData
+                   , Swagger.ToParamSchema, Swagger.ToSchema)