Merge branch '238-dev-async-job-worker' into 'dev'

Resolve "Job workers & persistence"

See merge request !371

README.md

@@ -115,18 +115,22 @@ From inside the `haskell-gargantext/` directory, run
 ### Multi-User with Graphical User Interface (Server Mode)
 
 ``` shell
-$ ~/.local/bin/stack --docker exec gargantext-server -- --run Prod
+$ cabal v2-run gargantext -- server start -m Prod
 ```
 
 Then you can log in with `user1` / `1resu`
 
+To start server and all workers:
+```shell
+$ cabal v2-run gargantext -- server start-all -m Prod
+```
 
 ### Command Line Mode tools
 
 #### Simple cooccurrences computation and indexation from a list of Ngrams
 
 ``` shell
-$ stack --docker exec gargantext-cli -- CorpusFromGarg.csv ListFromGarg.csv Ouput.json
+$ cabal v2-run gargantext -- filter-terms CorpusFromGarg.csv ListFromGarg.csv Ouput.json
 ```
 
 ### Analyzing the ngrams table repo
@@ -380,7 +384,7 @@ The flags have the following meaning:
 In order for some tests to run (like the phylo ones) is **required** to install the `gargantext-cli` via:
 
 ```hs
-cabal v2-install gargantext:exe:gargantext-cli
+cabal v2-install gargantext:exe:gargantext
 ```
 
 ### Modifying a golden test to accept a new (expected) output
@@ -407,7 +411,12 @@ psql -c "ALTER DATABASE \"gargantext_pgmq\" OWNER TO \"gargantua\""
 
 3. Finally launch the worker
 ```bash
-nix-shell --run "cabal v2-run gargantext-cli -- worker run --name default"
+nix-shell --run "cabal v2-run gargantext -- worker run --name default"
+```
+
+Or launch all worker definitions at once:
+```bash
+nix-shell --run "cabal v2-run gargantext -- worker run-all"
 ```
 
 ## Configuration

bin/cli

@@ -4,12 +4,12 @@
 # A couple hygienic options
 set -e -u
 
-# The following command will run `cabal run gargantext-cli --` followed by the
+# The following command will run `cabal run gargantext --` followed by the
 # options provided by the user, from inside a Nix shell. For instance,
 # if the user types
 # $ ./bin/cli someCommand "some string argument"
 # the following will be run from inside a Nix shell:
-# $ cabal run gargantext-cli -- someCommand "some string argument"
+# $ cabal run gargantext -- someCommand "some string argument"
 # It's a little convoluted because we want to keep spaces that were enclosed in
 # quotes or escaped by the user.
-nix-shell --run "$(printf "%q " cabal run gargantext-cli -- "$@")"
+nix-shell --run "$(printf "%q " cabal run gargantext -- "$@")"

bin/gargantext-cli/CLI/Server.hs

+{-|
+Module      : CLI.Server
+Description : Gargantext Server
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
+module CLI.Server where
+
+import Data.Version (showVersion)
+import CLI.Parsers (settings_p)
+import CLI.Types
+import CLI.Worker (runAllWorkers)
+import GHC.IO.Encoding (setLocaleEncoding, utf8)
+import Gargantext.API (startGargantext)
+import Gargantext.API.Admin.EnvTypes (Mode(..))
+import Gargantext.Core.Config.Types (_SettingsFile)
+import Gargantext.Prelude
+import Gargantext.System.Logging (withLogger, logMsg, LogLevel(..), Logger)
+import Options.Applicative
+import Paths_gargantext qualified as PG -- cabal magic build module
+
+
+
+serverCLI :: CLIServer -> IO ()
+serverCLI (CLIS_start serverArgs) = withLogger () $ \ioLogger ->
+  startServerCLI ioLogger serverArgs
+
+serverCLI (CLIS_startAll serverArgs@(ServerArgs { .. })) = withLogger () $ \ioLogger -> do
+  withAsync (startServerCLI ioLogger serverArgs) $ \aServer -> do
+    runAllWorkers ioLogger server_toml
+    wait aServer
+
+serverCLI (CLIS_version) = withLogger () $ \ioLogger -> do
+    -- Sets the locale to avoid encoding issues like in #284.
+  setLocaleEncoding utf8
+  logMsg ioLogger INFO $ "Version: " <> showVersion PG.version
+
+
+serverCmd :: HasCallStack => Mod CommandFields CLI
+serverCmd = command "server" (info (helper <*> (fmap CLISub $ fmap CCMD_server serverParser))
+                               (progDesc "Gargantext server."))
+
+serverParser :: Parser CLIServer
+serverParser = hsubparser (
+  command "start" (info (helper <*> start_p)
+                      (progDesc "Start the server")) <>
+  command "start-all" (info (helper <*> start_all_p)
+                        (progDesc "Start the server and all workers (forked)")) <>
+  command "version" (info (helper <*> version_p)
+                      (progDesc "Show version and exit"))
+  )
+
+start_p :: Parser CLIServer
+start_p = fmap CLIS_start $ ServerArgs
+  <$> mode_p 
+  <*> port_p
+  <*> settings_p
+
+start_all_p :: Parser CLIServer
+start_all_p = fmap CLIS_startAll $ ServerArgs
+  <$> mode_p 
+  <*> port_p
+  <*> settings_p
+
+mode_p :: Parser Mode
+mode_p = option auto ( long "mode"
+                    <> short 'm'
+                    <> metavar "M"
+                    <> help "Possible modes: Dev | Mock | Prod" )
+
+port_p :: Parser Int
+port_p = option auto ( long "port"
+                    <> short 'p'
+                    <> metavar "P"
+                    <> showDefault
+                    <> value 8080
+                    <> help "Port" )
+  
+version_p :: Parser CLIServer
+version_p = pure CLIS_version
+
+
+startServerCLI :: Logger IO -> ServerArgs -> IO ()
+startServerCLI ioLogger (ServerArgs { .. }) = do
+  logMsg ioLogger INFO $ "starting server, mode: " <> show server_mode <> ", port: " <> show server_port <> ", config: " <> _SettingsFile server_toml
+  -- Sets the locale to avoid encoding issues like in #284.
+  setLocaleEncoding utf8
+
+  when (server_mode == Mock) $ do
+    logMsg ioLogger ERROR "Mock mode not supported!"
+    exitFailure
+
+  startGargantext server_mode server_port server_toml

bin/gargantext-cli/CLI/Types.hs

+{-|
+Module      : CLI.Types
+Description : CLI types
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
+{-# LANGUAGE DuplicateRecordFields #-}
 
 module CLI.Types where
 
 import Data.String
 import Data.Text (Text)
+import Gargantext.API.Admin.EnvTypes (Mode)
 import Gargantext.Core.Config.Types (SettingsFile(..))
 import Gargantext.Core.Types (NodeId)
 import Gargantext.Core.Types.Query
@@ -80,8 +94,21 @@ data CLIRoutes
   | CLIR_export FilePath
   deriving (Show, Eq)
 
+data CLIServer
+  = CLIS_start ServerArgs
+  | CLIS_startAll ServerArgs
+  | CLIS_version
+  deriving (Show, Eq)
+
+data ServerArgs = ServerArgs
+  { server_mode :: !Mode
+  , server_port :: !Int
+  , server_toml :: !SettingsFile }
+  deriving (Show, Eq)
+
 data CLIWorker
   = CLIW_run WorkerArgs
+  | CLIW_runAll WorkerAllArgs
   | CLIW_stats WorkerStatsArgs
   deriving (Show, Eq)
 
@@ -91,24 +118,29 @@ data WorkerArgs = WorkerArgs
   , worker_run_single :: !Bool
   } deriving (Show, Eq)
 
+data WorkerAllArgs = WorkerAllArgs
+  { worker_toml       :: !SettingsFile
+  } deriving (Show, Eq)
+
 data WorkerStatsArgs = WorkerStatsArgs
   { ws_toml       :: !SettingsFile
   } deriving (Show, Eq)
 
 data CLICmd
-  = CCMD_clean_csv_corpus
+  = CCMD_admin !AdminArgs
+  | CCMD_clean_csv_corpus
   | CCMD_filter_terms_and_cooc !CorpusFile !TermListFile !OutputFile
-  | CCMD_obfuscate_db !ObfuscateDBArgs
-  | CCMD_admin !AdminArgs
+  | CCMD_golden_file_diff !GoldenFileDiffArgs
   | CCMD_import !ImportArgs
   | CCMD_ini !IniArgs
   | CCMD_init !InitArgs
   | CCMD_invitations !InvitationsArgs
+  | CCMD_obfuscate_db !ObfuscateDBArgs
   | CCMD_phylo !PhyloArgs
   | CCMD_phylo_profile
-  | CCMD_upgrade !UpgradeArgs
-  | CCMD_golden_file_diff !GoldenFileDiffArgs
   | CCMD_routes !CLIRoutes
+  | CCMD_server !CLIServer
+  | CCMD_upgrade !UpgradeArgs
   | CCMD_worker !CLIWorker
   deriving (Show, Eq)
 

bin/gargantext-cli/CLI/Worker.hs

 {-|
-Module      : Worker.hs
+Module      : CLI.Worker
 Description : Gargantext Job Worker
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -16,23 +16,25 @@ import Async.Worker.Broker.Types qualified as BT
 import Async.Worker.Types qualified as W
 import CLI.Types
 import CLI.Parsers
+import Control.Concurrent.Async (forConcurrently_)
 import Data.List qualified as List (cycle, concat, take)
 import Data.Text qualified as T
 import Gargantext.Core.Config (hasConfig, gc_worker)
 import Gargantext.Core.Config.Types (SettingsFile(..))
+import Gargantext.Core.Config.Utils (readConfig)
 import Gargantext.Core.Config.Worker (WorkerDefinition(..), WorkerSettings(..), findDefinitionByName)
 import Gargantext.Core.Worker (withPGMQWorkerCtrlC, withPGMQWorkerSingleCtrlC, initWorkerState)
 import Gargantext.Core.Worker.Env (withWorkerEnv)
 -- import Gargantext.Core.Worker.Jobs (sendJob)
 -- import Gargantext.Core.Worker.Jobs.Types (Job(Ping))
 import Gargantext.Prelude
+import Gargantext.System.Logging (withLogger, logMsg, LogLevel(..), Logger)
 import Options.Applicative
 import Prelude qualified
 
 
--- TODO Command to spawn all workers at once
 -- TODO Command to monitor queues
--- TODO Support for KillWorkerSafely on Ctrl-C (so that the job in progress is moved back into the queue)
+
 
 workerCLI :: CLIWorker -> IO ()
 workerCLI (CLIW_run (WorkerArgs { .. })) = do
@@ -65,6 +67,8 @@ workerCLI (CLIW_run (WorkerArgs { .. })) = do
           withPGMQWorkerCtrlC env wd $ \a _state -> do
             -- _ <- runReaderT (sendJob Ping) env
             wait a
+workerCLI (CLIW_runAll (WorkerAllArgs { .. })) = withLogger () $ \ioLogger -> do
+  runAllWorkers ioLogger worker_toml
 workerCLI (CLIW_stats (WorkerStatsArgs { .. })) = do
   putStrLn ("worker toml: " <> _SettingsFile ws_toml)
 
@@ -95,6 +99,8 @@ workerParser :: Parser CLIWorker
 workerParser = hsubparser (
   command "run" (info (helper <*> worker_p)
                   (progDesc "Run a single worker")) <>
+  command "run-all" (info (helper <*> worker_all_p)
+                     (progDesc "Run all worker definitions")) <>
   command "stats" (info (helper <*> stats_p)
                     (progDesc "Print queue stats"))
   )
@@ -108,6 +114,21 @@ worker_p = fmap CLIW_run $ WorkerArgs
   <*> flag False True ( long "run-single"
                      <> help "Whether to loop or run a single job from queue" )
 
+worker_all_p :: Parser CLIWorker
+worker_all_p = fmap CLIW_runAll $ WorkerAllArgs
+  <$> settings_p
+
 stats_p :: Parser CLIWorker
 stats_p = fmap CLIW_stats $ WorkerStatsArgs
   <$> settings_p
+
+
+runAllWorkers :: Logger IO -> SettingsFile -> IO ()
+runAllWorkers ioLogger worker_toml = do
+  cfg <- readConfig worker_toml
+  let ws = cfg ^. gc_worker
+  forConcurrently_ (_wsDefinitions ws) $ \wd -> do
+    withWorkerEnv worker_toml $ \env -> do
+      logMsg ioLogger INFO $ "Starting worker '" <> T.unpack (_wdName wd) <> "' (queue " <> show (_wdQueue wd) <> ")"
+      withPGMQWorkerCtrlC env wd $ \a _state -> do
+        wait a

bin/gargantext-cli/Main.hs

@@ -11,29 +11,31 @@ Main specifications to index a corpus with a term list
 
  -}
 
-{-# LANGUAGE Strict             #-}
-{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE LambdaCase       #-}
+{-# LANGUAGE Strict           #-}
 {-# LANGUAGE TypeApplications #-}
 
 module Main where
 
-import Prelude
 
-import CLI.FilterTermsAndCooc
-import CLI.ObfuscateDB (obfuscateDB, obfuscateDBCmd)
-import CLI.Types
-import Options.Applicative
 import CLI.Admin (adminCLI, adminCmd)
 import CLI.FileDiff (fileDiffCLI, fileDiffCmd)
+import CLI.FilterTermsAndCooc
 import CLI.Import (importCLI, importCmd)
 import CLI.Ini (iniCLI, iniCmd)
 import CLI.Init (initCLI, initCmd)
 import CLI.Invitations (invitationsCLI, invitationsCmd)
+import CLI.ObfuscateDB (obfuscateDB, obfuscateDBCmd)
 import CLI.Phylo (phyloCLI, phyloCmd)
 import CLI.Phylo.Profile (phyloProfileCLI, phyloProfileCmd)
+import CLI.Server (serverCLI, serverCmd)
 import CLI.Server.Routes (routesCLI, routesCmd)
+import CLI.Types
 import CLI.Upgrade (upgradeCLI, upgradeCmd)
 import CLI.Worker (workerCLI, workerCmd)
+import Options.Applicative
+import Prelude
+
 
 runCLI :: CLI -> IO ()
 runCLI = \case
@@ -63,6 +65,8 @@ runCLI = \case
     -> fileDiffCLI args
   CLISub (CCMD_routes args)
     -> routesCLI args
+  CLISub (CCMD_server args)
+    -> serverCLI args
   CLISub (CCMD_worker args)
     -> workerCLI args
 
@@ -73,7 +77,7 @@ main = runCLI =<< execParser opts
     opts = info (helper <*> allOptions)
       ( fullDesc
      <> progDesc "CLI for the gargantext-server. Type --help for all the commands."
-     <> header "gargantext-cli tools" )
+     <> header "gargantext tools" )
 
 allOptions :: Parser CLI
 allOptions = subparser (
@@ -89,5 +93,6 @@ allOptions = subparser (
                  upgradeCmd <>
                  fileDiffCmd <>
                  routesCmd <>
+                 serverCmd <>
                  workerCmd
                  )

bin/gargantext-server/Main.hs

-{-|
-Module      : Main.hs
-Description : Gargantext starter
-Copyright   : (c) CNRS, 2017-Present
-License     : AGPL + CECILL v3
-Maintainer  : team@gargantext.org
-Stability   : experimental
-Portability : POSIX
-
-Script to start gargantext with different modes (Dev, Prod, Mock).
-
--}
-
-{-# LANGUAGE QuasiQuotes          #-}
-{-# LANGUAGE StandaloneDeriving   #-}
-{-# LANGUAGE Strict               #-}
-{-# LANGUAGE TypeOperators        #-}
--- {-# OPTIONS_GHC -fno-warn-orphans #-}
-{-# LANGUAGE TypeFamilies #-}
-
-
-module Main where
-
-
-import Data.Text (unpack)
-import Data.Version (showVersion)
-import GHC.IO.Encoding
-import Gargantext.API (startGargantext) -- , startGargantextMock)
-import Gargantext.API.Admin.EnvTypes
-import Gargantext.API.Admin.Settings
-import Gargantext.Core.Config.Types (SettingsFile(..))
-import Gargantext.Prelude
-import Gargantext.System.Logging
-import Options.Generic
-import System.Exit (exitSuccess)
-import qualified Paths_gargantext as PG -- cabal magic build module
-
-
-instance ParseRecord Mode
-instance ParseField  Mode
-instance ParseFields Mode
-
-data MyOptions w =
-  MyOptions { run  :: w ::: Mode
-                        <?> "Possible modes: Dev | Mock | Prod"
-            , port :: w ::: Maybe Int
-                        <?> "By default: 8008"
-            , toml :: w ::: Maybe FilePath
-                        <?> "By default: gargantext-settings.toml"
-            , version :: w ::: Bool
-                        <?> "Show version number and exit"
-            }
-   deriving (Generic)
-
-instance ParseRecord (MyOptions Wrapped)
-deriving instance Show (MyOptions Unwrapped)
-
-main :: IO ()
-main = withLogger () $ \ioLogger -> do
-
-  -- Sets the locale to avoid encoding issues like in #284.
-  setLocaleEncoding utf8
-  currentLocale <- getLocaleEncoding
-
-  MyOptions myMode myPort mb_tomlFile myVersion  <- unwrapRecord
-          "Gargantext server"
-  ---------------------------------------------------------------
-  if myVersion then do
-    logMsg ioLogger INFO $ "Version: " <> showVersion PG.version
-    System.Exit.exitSuccess
-  else
-    return ()
-  ---------------------------------------------------------------
-  let myPort' = case myPort of
-        Just p  -> p
-        Nothing -> 8008
-
-      tomlFile = SettingsFile $ case mb_tomlFile of
-          Nothing -> "gargantext-settings.toml"
-          Just i  -> i
-
-  ---------------------------------------------------------------
-  let start = case myMode of
-        Mock -> panicTrace "[ERROR] Mock mode unsupported"
-        _ -> startGargantext myMode myPort' tomlFile
-  logMsg ioLogger INFO $ "Starting with "   <> show myMode <> " mode."
-  logMsg ioLogger INFO $ "Machine locale: " <> show currentLocale
-  start
-  ---------------------------------------------------------------

bin/run

@@ -5,4 +5,4 @@
 set -e -u
 
 echo "Launching Gargantext..."
-nix-shell --run "cabal run gargantext-server -- --run Prod --toml gargantext-settings.toml"
+nix-shell --run "cabal v2-run gargantext -- server start-all --mode Prod --settings-path gargantext-settings.toml"

bin/scripts/start-worker

@@ -8,4 +8,4 @@
 
 # Then launch the service
 
-nix-shell --run "cabal v2-run gargantext-cli -- worker run --name default"
+nix-shell --run "cabal v2-run gargantext -- worker run --name default"

bin/update-project-dependencies

@@ -18,7 +18,7 @@ fi
 # with the `sha256sum` result calculated on the `cabal.project` and
 # `cabal.project.freeze`. This ensures the files stay deterministic so that CI
 # cache can kick in.
-expected_cabal_project_hash="3b00795e0b1c97372e72a3ef464aa809ca90d8c3f1ab580d6a956526c94c160c"
+expected_cabal_project_hash="d5a9510a825fd2352402a7b43d0ebb3ce9342f8449c1dbef8365859e0aff918a"
 expected_cabal_project_freeze_hash="30dd1cf2cb2015351dd0576391d22b187443b1935c2be23599b821ad1ab95f23"
 
 

cabal.project

@@ -50,11 +50,6 @@ source-repository-package
     tag: b9fca8beee0f23c17a6b2001ec834d071709e6e7
     subdir: packages/base
 
-source-repository-package
-    type: git
-    location: https://github.com/adinapoli/servant-job.git
-    tag: 74a3296dfe1f0c4a3ade91336dcc689330e84156
-
 source-repository-package
     type: git
     location: https://github.com/alpmestan/sparse-linear.git

gargantext.cabal

@@ -147,6 +147,7 @@ library
       Gargantext.API.Node.Update
       Gargantext.API.Node.Update.Types
       Gargantext.API.Prelude
+      Gargantext.API.Public.Types
       Gargantext.API.Routes
       Gargantext.API.Routes.Named
       Gargantext.API.Routes.Named.Annuaire
@@ -170,6 +171,8 @@ library
       Gargantext.API.Routes.Named.Tree
       Gargantext.API.Routes.Named.Viz
       Gargantext.API.Routes.Types
+      Gargantext.API.Search.Types
+      Gargantext.API.Table.Types
       Gargantext.API.Types
       Gargantext.API.Viz.Types
       Gargantext.API.Worker
@@ -237,6 +240,7 @@ library
       Gargantext.Core.Types.Phylo
       Gargantext.Core.Types.Query
       Gargantext.Core.Utils
+      Gargantext.Core.Utils.Aeson
       Gargantext.Core.Utils.Prefix
       Gargantext.Core.Viz.Graph.Index
       Gargantext.Core.Viz.Graph.Tools
@@ -332,9 +336,7 @@ library
       Gargantext.API.Node.DocumentsFromWriteNodes
       Gargantext.API.Node.FrameCalcUpload
       Gargantext.API.Node.New
-      Gargantext.API.Public.Types
       Gargantext.API.Search
-      Gargantext.API.Search.Types
       Gargantext.API.Server.Named
       Gargantext.API.Server.Named.EKG
       Gargantext.API.Server.Named.Ngrams
@@ -343,7 +345,6 @@ library
       Gargantext.API.Server.Named.Viz
       Gargantext.API.Swagger
       Gargantext.API.Table
-      Gargantext.API.Table.Types
       Gargantext.API.ThrowAll
       Gargantext.Core.Ext.IMT
       Gargantext.Core.Ext.IMTUser
@@ -391,6 +392,7 @@ library
       Gargantext.Core.Text.Terms.Multi.PosTagging.Types
       Gargantext.Core.Types.Search
       Gargantext.Core.Utils.DateUtils
+      Gargantext.Core.Utils.Swagger
       Gargantext.Core.Viz
       Gargantext.Core.Viz.Chart
       Gargantext.Core.Viz.Graph.API
@@ -585,7 +587,6 @@ library
     , servant-client >= 0.19 && < 0.20
     , servant-client-core >= 0.20 && < 0.21
     , servant-ekg ^>= 0.3.1
-    , servant-job >= 0.2.0.0
     , servant-routes < 0.2
     , servant-server >= 0.18.3 && < 0.21
     , servant-swagger ^>= 1.2
@@ -633,7 +634,7 @@ library
     , zip ^>= 2.0.0
     , zip-archive ^>= 0.4.3
 
-executable gargantext-cli
+executable gargantext
   import:
       defaults
     , optimized
@@ -651,6 +652,7 @@ executable gargantext-cli
       CLI.Phylo
       CLI.Phylo.Common
       CLI.Phylo.Profile
+      CLI.Server
       CLI.Server.Routes
       CLI.Types
       CLI.Upgrade
@@ -694,28 +696,6 @@ executable gargantext-cli
     , vector ^>= 0.12.3.0
   ghc-options: -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wmissing-signatures -Wunused-binds -Wunused-imports -Werror -freduction-depth=300 -fprint-potential-instances
 
-executable gargantext-server
-  import:
-      defaults
-    , optimized
-  main-is: Main.hs
-  other-modules:
-      Paths_gargantext
-  hs-source-dirs:
-      bin/gargantext-server
-  build-depends:
-      cassava ^>= 0.5.2.0
-    , containers ^>= 0.6.7
-    , extra ^>= 1.7.9
-    , gargantext
-    , gargantext-prelude
-    , ini ^>= 0.4.1
-    , optparse-generic ^>= 1.4.7
-    , postgresql-simple ^>= 0.6.4
-    , text ^>= 2.0.2
-    , unordered-containers ^>= 0.2.16.0
-    , vector ^>= 0.12.3.0
-
 common testDependencies
   build-depends:
       base >=4.7 && <5
@@ -776,7 +756,6 @@ common testDependencies
     , servant-auth-client
     , servant-client >= 0.19 && < 0.20
     , servant-client-core >= 0.20 && < 0.21
-    , servant-job
     , servant-server >= 0.18.3 && < 0.21
     , servant-websockets >= 2.0.0 && < 2.1
     , shelly >= 1.9 && < 2
@@ -822,7 +801,6 @@ test-suite garg-test-tasty
       Test.API.Setup
       Test.API.Prelude
       Test.API.UpdateList
-      Test.Core.AsyncUpdates
       Test.Core.Notifications
       Test.Core.Similarity
       Test.Core.Text

src/Gargantext/API.hs

@@ -101,7 +101,7 @@ startGargantext mode port sf@(SettingsFile settingsFile) = withLoggerHoisted mod
           case r of
             Right True  -> pure ()
             Right False -> panicTrace $
-              "You must run 'gargantext-cli init " <> pack settingsFile <>
+              "You must run 'gargantext init " <> pack settingsFile <>
               "' before running gargantext-server (only the first time)."
             Left err -> panicTrace $ "Unexpected exception:" <> show err
         oneHour = Clock.fromNanoSecs 3600_000_000_000

src/Gargantext/API/Admin/Auth.hs

@@ -51,8 +51,9 @@ import Data.UUID.V4 (nextRandom)
 import Gargantext.API.Admin.Auth.Types
 import Gargantext.API.Admin.EnvTypes (Env)
 import Gargantext.API.Auth.PolicyCheck
-import Gargantext.API.Errors
+import Gargantext.API.Errors (BackendInternalError(..), HasAuthenticationError, _AuthenticationError)
 import Gargantext.API.Prelude (authenticationError, HasServerError, GargServerC, _ServerError, GargM, IsGargServer)
+import Gargantext.API.Routes.Named qualified as Named
 import Gargantext.API.Worker (serveWorkerAPI)
 import Gargantext.Core.Config (HasJWTSettings(..))
 import Gargantext.Core.Mail (MailModel(..), mail)
@@ -69,11 +70,9 @@ import Gargantext.Database.Schema.Node (NodePoly(_node_id))
 import Gargantext.Prelude hiding (Handler, reverse, to)
 import Gargantext.Prelude.Crypto.Auth qualified as Auth
 import Gargantext.Prelude.Crypto.Pass.User (gargPass)
-import Servant
-import Servant.API.Generic ()
-import Servant.Auth.Server
-import Servant.Server.Generic
-import qualified Gargantext.API.Routes.Named as Named
+import Servant (HasServer, ServerT, NamedRoutes, errBody, hoistServer, err404)
+import Servant.Auth.Server (makeJWT)
+import Servant.Server.Generic (AsServerT)
 
 ---------------------------------------------------
 

src/Gargantext/API/Admin/Auth/Types.hs

@@ -47,7 +47,7 @@ import Gargantext.Core.Types.Individu (Username, GargPassword(..), arbitraryUser
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.Admin.Types.Node (NodeId(..), ListId, DocId, UserId (..))
 import Gargantext.Prelude hiding (reverse)
-import Servant.Auth.Server
+import Servant.Auth.Server (CookieSettings, JWTSettings, ToJWT, FromJWT)
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
 

src/Gargantext/API/Admin/EnvTypes.hs

--- |
+{-|
+Module      : Gargantext.API.Admin.EnvTypes
+Description : Env definitions in which the Gargantext app is run
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 
 {-# LANGUAGE TemplateHaskell     #-}
 {-# LANGUAGE TypeFamilies        #-}
@@ -25,15 +35,13 @@ module Gargantext.API.Admin.EnvTypes (
   , DevJobHandle(..)
   ) where
 
-import Control.Lens hiding (Level, (:<), (.=))
-import Control.Monad.Except
-import Control.Monad.Reader
+import Control.Lens (to, view)
 import Data.List ((\\))
 import Data.Pool (Pool)
 import Data.Text qualified as T
 import Database.PostgreSQL.Simple (Connection)
-import Gargantext.API.Admin.Orchestrator.Types
-import Gargantext.API.Errors.Types
+import Gargantext.API.Admin.Orchestrator.Types (JobLog, noJobLog)
+import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Prelude (GargM, IsGargServer)
 import Gargantext.Core.Notifications.CentralExchange qualified as CE
 import Gargantext.Core.Notifications.CentralExchange.Types qualified as CET
@@ -51,8 +59,9 @@ import Network.HTTP.Client (Manager)
 import Servant.Auth.Server (JWTSettings)
 import System.Log.FastLogger qualified as FL
 
+
 data Mode = Dev | Mock | Prod
-  deriving (Show, Read, Generic)
+  deriving (Show, Read, Generic, Eq)
 
 -- | Given the 'Mode' the server is running in, it returns the list of
 -- allowed levels. For example for production we ignore everything which

src/Gargantext/API/Admin/Orchestrator/Types.hs

+{-|
+Module      : Gargantext.API.Admin.Orchestrator.Types
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 {-# LANGUAGE TemplateHaskell    #-}
 {-# LANGUAGE TypeOperators      #-}
@@ -5,19 +17,13 @@
 module Gargantext.API.Admin.Orchestrator.Types
   where
 
-import Control.Lens hiding (elements)
-import Data.Aeson
+import Data.Aeson (genericParseJSON, genericToJSON)
 import Data.Morpheus.Types ( GQLType, typeOptions )
-import Data.Proxy
-import Data.Swagger hiding (URL, url, port)
-import GHC.Generics hiding (to)
+import Data.Swagger (ToSchema, URL, declareNamedSchema, defaultSchemaOptions, genericDeclareNamedSchemaUnrestricted)
 import Gargantext.API.GraphQL.UnPrefix qualified as GQLU
 import Gargantext.Core.Types (TODO(..))
+import Gargantext.Core.Utils.Aeson (jsonOptions)
 import Gargantext.Prelude
-import Servant
-import Servant.Job.Async
-import Servant.Job.Types
-import Servant.Job.Utils (jsonOptions)
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary
 
@@ -51,24 +57,6 @@ instance ToSchema ExternalAPIs where
 instance ToSchema URL where
   declareNamedSchema _ = declareNamedSchema (Proxy :: Proxy TODO)
 
-data ScraperInput = ScraperInput
-  { _scin_spider       :: !Text
-  , _scin_query        :: !(Maybe Text)
-  , _scin_user         :: !Text
-  , _scin_corpus       :: !Int
-  , _scin_report_every :: !(Maybe Int)
-  , _scin_limit        :: !(Maybe Int)
-  , _scin_local_file   :: !(Maybe Text)
-  , _scin_count_only   :: !(Maybe Bool)
-  }
-  deriving Generic
-
-makeLenses ''ScraperInput
-
-instance FromJSON ScraperInput where
-  parseJSON = genericParseJSON $ jsonOptions "_scin_"
-
--- Proposal to replace the Corpus.API.Query type which seems to generically named.
 
 data ScraperEvent = ScraperEvent
   { _scev_message :: !(Maybe Text)
@@ -117,19 +105,3 @@ instance ToSchema JobLog -- TODO _scst_ prefix
 instance GQLType JobLog where
   typeOptions _ = GQLU.unPrefix "_scst_"
 
-instance ToSchema ScraperInput  -- TODO _scin_ prefix
-
-instance ToParamSchema Offset -- where
-  -- toParamSchema = panic "TODO"
-
-instance ToParamSchema Limit -- where
-  -- toParamSchema = panic "TODO"
-
-type ScrapersEnv = JobEnv JobLog JobLog
-
-type ScraperAPI  = AsyncJobsAPI JobLog ScraperInput  JobLog
-
-------------------------------------------------------------------------
-data AsyncJobs event ctI input output mode = AsyncJobs
-  { asyncJobsAPI' :: mode :- AsyncJobsAPI' 'Unsafe 'Safe ctI '[JSON] Maybe event input output }
-  deriving Generic

src/Gargantext/API/Admin/Settings.hs

@@ -16,7 +16,7 @@ TODO-SECURITY: Critical
 {-# LANGUAGE ScopedTypeVariables #-}
 
 module Gargantext.API.Admin.Settings
-    where
+where
 
 
 import Codec.Serialise (Serialise(), serialise)
@@ -24,15 +24,15 @@ import Data.ByteString.Lazy qualified as L
 import Data.Pool (Pool)
 import Data.Pool qualified as Pool
 import Database.PostgreSQL.Simple (Connection, connect, close, ConnectInfo)
-import Gargantext.API.Admin.EnvTypes
-import Gargantext.API.Errors.Types
-import Gargantext.API.Prelude
+import Gargantext.API.Admin.EnvTypes (Env(..))
+import Gargantext.API.Errors.Types (BackendInternalError)
+import Gargantext.API.Prelude (GargM)
 import Gargantext.Core.Notifications.Dispatcher qualified as D
 import Gargantext.Core.Config (GargConfig(..))
 import Gargantext.Core.Config.Types (jwtSettings)
-import Gargantext.Core.NodeStory
+import Gargantext.Core.NodeStory (fromDBNodeStoryEnv)
 import Gargantext.Prelude
-import Gargantext.System.Logging
+import Gargantext.System.Logging (Logger)
 import Network.HTTP.Client.TLS (newTlsManager)
 import System.Directory (renameFile)
 import System.IO (hClose)

src/Gargantext/API/Admin/Settings/TOML.hs

-{-# LANGUAGE TemplateHaskell #-}
-module Gargantext.API.Admin.Settings.TOML where
-
-import Control.Lens hiding ((.=))
-import Data.Text qualified as T
-import Gargantext.API.Admin.Settings.CORS
-import Gargantext.API.Admin.Settings.MicroServices
-import Gargantext.Core.Worker.TOML
-import Gargantext.Prelude (panicTrace)
-import Gargantext.System.Logging
-import Prelude
-import Toml
-import Servant.Client.Core.BaseUrl
-
--- | Compatibility bridge until we fix #304 (move to Toml)
-data GargTomlSettings = GargTomlSettings
-  { _gargCorsSettings          :: !CORSSettings
-  , _gargMicroServicesSettings :: !MicroServicesSettings
-  , _gargWorkerSettings        :: !WorkerSettings
-  }
-
-makeLenses ''GargTomlSettings
-
-settingsCodec :: TomlCodec GargTomlSettings
-settingsCodec = GargTomlSettings
-  <$> (Toml.table corsSettingsCodec "cors" .= _gargCorsSettings)
-  <*> (Toml.table  microServicesSettingsCodec "microservices.proxy" .= _gargMicroServicesSettings)
-  <*> (Toml.table workerSettingsCodec "worker" .= _gargWorkerSettings)
-
--- | Extends the 'allowed-origins' in the CORSettings with the URLs embellished
--- with the proxy port.
-addProxyToAllowedOrigins :: GargTomlSettings -> GargTomlSettings
-addProxyToAllowedOrigins stgs =
-  stgs & over gargCorsSettings (addProxies $ stgs ^. gargMicroServicesSettings . msProxyPort)
-  where
-    addProxies :: Int -> CORSSettings -> CORSSettings
-    addProxies port cors =
-      let origins = _corsAllowedOrigins cors
-          mkUrl (CORSOrigin bh) = CORSOrigin $ bh { baseUrlPort = port }
-      in cors { _corsAllowedOrigins = origins <> Prelude.map mkUrl origins }
-
--- | Loads the 'CORSSettings' from the 'toml' file.
-loadGargTomlSettings :: FilePath -> IO GargTomlSettings
-loadGargTomlSettings tomlFile = do
-  tomlRes <- Toml.decodeFileEither settingsCodec tomlFile
-  case tomlRes of
-      Left errs      -> do
-        withLogger () $ \ioLogger -> do
-          logMsg ioLogger ERROR $ T.unpack $ "Error, gargantext-settings.toml parsing failed: " <> Toml.prettyTomlDecodeErrors errs
-          panicTrace "Please fix the errors in your gargantext-settings.toml file."
-      Right settings0 -> case settings0 ^. gargCorsSettings . corsUseOriginsForHosts of
-        True  -> pure $ addProxyToAllowedOrigins $
-                   settings0 & over (gargCorsSettings . corsAllowedHosts)
-                     (\_ -> (settings0 ^. gargCorsSettings . corsAllowedOrigins))
-        False -> pure $ addProxyToAllowedOrigins settings0

src/Gargantext/API/Auth/PolicyCheck.hs

+{-|
+Module      : Gargantext.API.Auth.PolicyCheck
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
 {-# LANGUAGE LambdaCase          #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeFamilies        #-}

src/Gargantext/API/Context.hs

@@ -16,19 +16,15 @@ Portability : POSIX
 module Gargantext.API.Context
   where
 
-import Prelude
-
-import Data.Aeson (FromJSON, ToJSON)
-import Servant
-
 import Gargantext.API.Admin.Auth (withNamedAccess)
 import Gargantext.API.Admin.Auth.Types (PathId(..), AuthenticatedUser)
-import Gargantext.API.Prelude
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.API.Prelude (IsGargServer)
+import Gargantext.API.Routes.Named.Context qualified as Named
+import Gargantext.Database.Admin.Types.Node (ContextId, contextId2NodeId)
 import Gargantext.Database.Prelude (JSONB)
-import Gargantext.Database.Query.Table.Context
+import Gargantext.Database.Query.Table.Context (getContextWith)
+import Gargantext.Prelude 
 import Servant.Server.Generic (AsServerT)
-import Gargantext.API.Routes.Named.Context qualified as Named
 
 -------------------------------------------------------------------
 -- TODO use Context instead of Node

src/Gargantext/API/Count/Types.hs

+{-|
+Module      : Gargantext.API.Count.Types
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 {-# LANGUAGE TemplateHaskell    #-}
+
 module Gargantext.API.Count.Types (
     Scraper(..)
   , QueryBool(..)

src/Gargantext/API/EKG.hs

+{-|
+Module      : Gargantext.API.EKG
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
 {-# OPTIONS_GHC -fno-warn-orphans #-}
-{-# LANGUAGE TypeOperators #-}
 {-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeOperators #-}
+
 module Gargantext.API.EKG where
 
 import Data.HashMap.Strict as HM
 import Data.Text as T
 import Data.Text.IO as T
 import Data.Time.Clock.POSIX (getPOSIXTime)
-import Network.Wai
+import Network.Wai (Middleware)
 import Protolude
 import Servant
-import Servant.Auth
-import Servant.Ekg
+import Servant.Auth (Auth)
+import Servant.Ekg (HasEndpoint, getEndpoint, enumerateEndpoints, monitorEndpoints)
 import System.Metrics
-import qualified System.Metrics.Json as J
+import System.Metrics.Json qualified as J
 
 -- Mimics https://github.com/tibbe/ekg/blob/master/System/Remote/Snap.hs#L98
 type EkgAPI =

src/Gargantext/API/Errors.hs

@@ -21,13 +21,13 @@ module Gargantext.API.Errors (
 
 import Prelude
 
-import Control.Exception.Safe
+import Control.Exception.Safe (displayException)
 import Data.Aeson qualified as JSON
 import Data.Text qualified as T
 import Data.Text.Lazy qualified as TL
 import Data.Text.Lazy.Encoding qualified as TE
 import Data.Validity ( prettyValidation )
-import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Auth.Types (AuthenticationError(..))
 import Gargantext.API.Errors.Class as Class
 import Gargantext.API.Errors.TH (deriveHttpStatusCode)
 import Gargantext.API.Errors.Types as Types
@@ -35,7 +35,7 @@ import Gargantext.Database.Query.Table.Node.Error hiding (nodeError)
 import Gargantext.Database.Query.Tree hiding (treeError)
 import Gargantext.Utils.Jobs.Monad (JobError(..))
 import Network.HTTP.Types.Status qualified as HTTP
-import Servant.Server
+import Servant.Server (ServerError(..), err404, err500)
 
 $(deriveHttpStatusCode ''BackendErrorCode)
 

src/Gargantext/API/Errors/Class.hs

+{-|
+Module      : Gargantext.API.Errors.Class
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
 
 module Gargantext.API.Errors.Class where
 
-import Control.Lens
+import Control.Lens (Prism')
 import Gargantext.API.Admin.Auth.Types (AuthenticationError)
 
+
 class HasAuthenticationError e where
   _AuthenticationError :: Prism' e AuthenticationError

src/Gargantext/API/Errors/TH.hs

+{-|
+Module      : Gargantext.API.Errors.TH
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
 {-# LANGUAGE LambdaCase        #-}
 {-# LANGUAGE TemplateHaskell   #-}
 

src/Gargantext/API/Errors/Types.hs

@@ -23,8 +23,11 @@ Portability : POSIX
 {-# OPTIONS_GHC -Wno-orphans          #-} -- instance IsFrontendErrorData and stage restriction
 
 module Gargantext.API.Errors.Types (
+    HasServerError(..)
+  , serverError
+  
   -- * The main frontend error type
-    FrontendError(..)
+  , FrontendError(..)
 
   -- * The internal backend type and an enumeration of all possible backend error types
   , BackendErrorCode(..)
@@ -43,7 +46,7 @@ module Gargantext.API.Errors.Types (
   , IsFrontendErrorData(..)
   ) where
 
-import Control.Lens (makePrisms)
+import Control.Lens ((#), makePrisms, Prism')
 import Control.Monad.Fail (fail)
 import Data.Aeson (Value(..), (.:), (.=), object, withObject)
 import Data.Aeson.Types (typeMismatch, emptyArray)
@@ -64,8 +67,6 @@ import Gargantext.Prelude hiding (Location, WithStacktrace)
 import Gargantext.Utils.Dict (Dict(..))
 import Gargantext.Utils.Jobs.Monad qualified as Jobs
 import Servant (ServerError)
-import Servant.Job.Core ( HasServerError(..) )
-import Servant.Job.Types qualified as SJ
 
 -- | A 'WithStacktrace' carries an error alongside its
 -- 'CallStack', to be able to print the correct source location
@@ -80,6 +81,19 @@ instance Exception e => Exception (WithStacktrace e) where
   displayException WithStacktrace{..}
     = displayException ct_error <> "\n" <> prettyCallStack ct_callStack
 
+
+
+-------------------------------------------------------------------
+class HasServerError err where
+  _ServerError :: Prism' err ServerError
+
+serverError :: (MonadError err m, HasServerError err) => ServerError -> m a
+serverError e = throwError $ _ServerError # e
+
+instance HasServerError ServerError where
+  _ServerError = identity
+
+
 -------------------------------------------------------------------
 -- | An internal error which can be emitted from the backend and later
 -- converted into a 'FrontendError', for later consumption.
@@ -109,7 +123,7 @@ makePrisms ''BackendInternalError
 
 instance ToJSON BackendInternalError where
   toJSON (InternalJobError s) =
-    object [ ("status", toJSON SJ.IsFailure)
+    object [ ("status", toJSON ("IsFailure" :: Text))
            , ("log", emptyArray)
            , ("id", String mk_id)
            , ("error", String $ T.pack $ show s) ]

src/Gargantext/API/GraphQL.hs

@@ -39,7 +39,7 @@ import Gargantext.API.GraphQL.TreeFirstLevel qualified as GQLTree
 import Gargantext.API.GraphQL.User qualified as GQLUser
 import Gargantext.API.GraphQL.UserInfo qualified as GQLUserInfo
 import Gargantext.API.Prelude (GargM)
-import Gargantext.API.Types
+import Gargantext.API.Types (HTML)
 import Gargantext.Core.Config (HasJWTSettings)
 import Gargantext.Core.NLP (HasNLPServer)
 import Gargantext.Database.Prelude (CmdCommon)
@@ -47,7 +47,7 @@ import Gargantext.Prelude hiding (ByteString)
 import Servant
 import Servant.Auth qualified as SA
 import Servant.Auth.Server qualified as SAS
-import Servant.Server.Generic
+import Servant.Server.Generic (AsServerT)
 
 
 -- | Represents possible GraphQL queries.

src/Gargantext/API/GraphQL/Annuaire.hs

+{-|
+Module      : Gargantext.API.GraphQL.Annuaire
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
+
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE DuplicateRecordFields #-}
 
 module Gargantext.API.GraphQL.Annuaire where
 
-import Control.Lens
+import Control.Lens (Traversal', _Just, ix)
 import Data.Morpheus.Types ( GQLType )
-import Data.Proxy
 import Gargantext.Database.Admin.Types.Hyperdata.Contact
   ( HyperdataContact
   , ContactWho
@@ -17,7 +27,7 @@ import Gargantext.Database.Prelude (CmdCommon)
 import Gargantext.Database.Query.Table.Context (getContextWith)
 import Gargantext.Database.Schema.Node (node_hyperdata)
 import Gargantext.Prelude
-import Gargantext.API.GraphQL.Types
+import Gargantext.API.GraphQL.Types (GqlM)
 
 data AnnuaireContact = AnnuaireContact
   { ac_title        :: !(Maybe Text)

src/Gargantext/API/GraphQL/IMT.hs

+{-|
+Module      : Gargantext.API.GraphQL.IMT
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
+
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE DerivingStrategies #-}
 

src/Gargantext/API/GraphQL/NLP.hs

+{-|
+Module      : Gargantext.API.GraphQL.NLP
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
+
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE DerivingStrategies #-}
 
@@ -11,13 +22,13 @@ module Gargantext.API.GraphQL.NLP
   where
 
 import Control.Lens (view)
+import Data.Map.Strict qualified as Map
 import Data.Morpheus.Types (GQLType)
 import Gargantext.API.GraphQL.Types
 import Gargantext.Core (Lang(..), NLPServerConfig(..), PosTagAlgo)  -- , allLangs)
 import Gargantext.Core.NLP (HasNLPServer(..))
 import Gargantext.Prelude
-import Protolude
-import qualified Data.Map.Strict as Map
+import Protolude qualified
 
 newtype LanguagesArgs
   = LanguagesArgs ()

src/Gargantext/API/GraphQL/PolicyCheck.hs

+{-|
+Module      : Gargantext.API.GraphQL.PolicyCheck
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 
 module Gargantext.API.GraphQL.PolicyCheck where
 

src/Gargantext/API/GraphQL/User.hs

@@ -15,17 +15,17 @@ Portability : POSIX
 module Gargantext.API.GraphQL.User where
 
 import Data.Morpheus.Types ( GQLType )
-import Gargantext.API.Admin.Auth.Types
-import Gargantext.API.Auth.PolicyCheck
-import Gargantext.API.GraphQL.PolicyCheck
-import Gargantext.API.GraphQL.Types
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
+import Gargantext.API.Auth.PolicyCheck (AccessPolicyManager, nodeReadChecks)
+import Gargantext.API.GraphQL.PolicyCheck (withPolicy)
+import Gargantext.API.GraphQL.Types (GqlM, GqlM')
+import Gargantext.Core.Types (NodeId(..), UserId)
 import Gargantext.Core.Types.Individu qualified as Individu
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataUser(..))
 import Gargantext.Database.Prelude (CmdCommon)
 import Gargantext.Database.Query.Table.User qualified as DBUser
 import Gargantext.Database.Schema.User (UserLight(..))
 import Gargantext.Prelude
-import Gargantext.Core.Types
 
 data User m = User
   { u_email     :: Text

src/Gargantext/API/GraphQL/UserInfo.hs

@@ -40,12 +40,14 @@ import Gargantext.Database.Admin.Types.Hyperdata.Contact
   , ct_phone
   , hc_who
   , hc_where)
-import Gargantext.API.Admin.Auth.Types hiding (Valid)
-import Gargantext.API.Auth.PolicyCheck
-import Gargantext.API.GraphQL.PolicyCheck
-import Gargantext.API.GraphQL.Types
+  
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
+import Gargantext.API.Auth.PolicyCheck (AccessPolicyManager, userMe)
+import Gargantext.API.GraphQL.PolicyCheck (withPolicy)
+import Gargantext.API.GraphQL.Types (GqlM, GqlM')
 import Gargantext.API.GraphQL.Utils (AuthStatus(Invalid, Valid), authUser)
 import Gargantext.Core.Config (HasJWTSettings)
+import Gargantext.Core.Types (UserId(..))
 import Gargantext.Core.Types.Individu qualified as Individu
 import Gargantext.Database.Prelude (CmdCommon)
 import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
@@ -53,7 +55,6 @@ import Gargantext.Database.Query.Table.User (getUsersWithHyperdata, getUsersWith
 import Gargantext.Database.Schema.Node (node_id, node_hyperdata, NodePoly (Node, _node_id))
 import Gargantext.Database.Schema.User (UserLight(..))
 import Gargantext.Prelude
-import Gargantext.Core.Types
 
 data UserInfo = UserInfo
   { ui_id             :: Int

src/Gargantext/API/HashedResponse.hs

@@ -12,7 +12,7 @@ Portability : POSIX
 module Gargantext.API.HashedResponse where
 
 import Data.Aeson
-import Data.Swagger
+import Data.Swagger (ToSchema)
 import Gargantext.Prelude
 import Gargantext.Prelude.Crypto.Hash qualified as Crypto (hash)
 

src/Gargantext/API/Job.hs

@@ -26,7 +26,7 @@ import Control.Lens (over, _Just)
 import Data.Text qualified as T
 import Gargantext.API.Admin.Orchestrator.Types
 import Gargantext.Prelude
-import Gargantext.Utils.Jobs.Error
+import Gargantext.Utils.Jobs.Error (ToHumanFriendlyError, mkHumanFriendly)
 
 newtype RemainingSteps = RemainingSteps { _RemainingSteps :: Int }
   deriving (Show, Eq, Num)

src/Gargantext/API/Members.hs

@@ -10,7 +10,7 @@ Portability : POSIX
 
 module Gargantext.API.Members where
 
-import Gargantext.API.Prelude
+import Gargantext.API.Prelude (IsGargServer)
 import Gargantext.API.Routes.Named.Private qualified as Named
 import Gargantext.Database.Action.Share (membersOf)
 import Gargantext.Database.Admin.Types.Node (NodeType(NodeTeam))

src/Gargantext/API/Metrics.hs

@@ -16,21 +16,20 @@ Metrics API
 module Gargantext.API.Metrics
     where
 
-import Control.Lens
 import Data.HashMap.Strict qualified as HashMap
 import Data.Time (UTCTime)
 import Data.Vector (Vector)
-import Gargantext.API.HashedResponse
-import Gargantext.API.Ngrams.NgramsTree
-import Gargantext.API.Ngrams.Types
+import Gargantext.API.HashedResponse (HashedResponse, constructHashedResponse, hash)
+import Gargantext.API.Ngrams.NgramsTree (NgramsTree)
+import Gargantext.API.Ngrams.Types (QueryParamR, TabType, ngramsTypeFromTabType, unNgramsTerm)
 import Gargantext.API.Prelude (IsGargServer)
 import Gargantext.API.Routes.Named.Metrics qualified as Named
 import Gargantext.Core.NodeStory (HasNodeStory)
 import Gargantext.Core.Text.Metrics (Scored(..), {-normalizeGlobal,-} normalizeLocal)
 import Gargantext.Core.Types (CorpusId, ListId, ListType(..))
 import Gargantext.Core.Types.Query (Limit)
-import Gargantext.Core.Viz.Chart
-import Gargantext.Core.Viz.Types
+import Gargantext.Core.Viz.Chart (chartData, histoData, treeData)
+import Gargantext.Core.Viz.Types (Histo)
 import Gargantext.Database.Action.Metrics qualified as Metrics
 import Gargantext.Database.Admin.Types.Hyperdata (HyperdataList(..), hl_chart, hl_pie, hl_scatter, hl_tree)
 import Gargantext.Database.Admin.Types.Metrics (ChartMetrics(..), Metric(..), Metrics(..))

src/Gargantext/API/Middleware.hs

+{-|
+Module      : Gargantext.API.Middleware
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+
 {-# LANGUAGE ViewPatterns      #-}
 
 {-| Edit 'sensitiveKeywords' to extend the list of redacted fields. -}
@@ -5,8 +17,8 @@ module Gargantext.API.Middleware (
   logStdoutDevSanitised
   ) where
 
-import Control.Lens
-import Control.Monad.Logger
+import Control.Lens (Traversal', at, over)
+import Control.Monad.Logger (LogStr, toLogStr)
 import Data.Aeson qualified as A
 import Data.Aeson.Lens qualified as L
 import Data.ByteString (ByteString)
@@ -16,15 +28,15 @@ import Data.ByteString.Char8 qualified as C8
 import Data.ByteString.Lazy qualified as B
 import Data.CaseInsensitive qualified as CI
 import Data.List qualified as L
-import Data.String
+import Data.String (fromString)
 import Data.Text qualified as T
 import Data.Text.Encoding qualified as TE
-import Network.HTTP.Types
-import Network.HTTP.Types.Header
-import Network.Wai
+import Network.HTTP.Types (QueryItem, Status(..))
+import Network.HTTP.Types.Header (Header, hAuthorization, hCookie, hSetCookie)
+import Network.Wai (Middleware, queryString, requestMethod, rawPathInfo)
 import Network.Wai.Middleware.RequestLogger
 import Prelude
-import System.Console.ANSI
+import System.Console.ANSI (Color(..), setSGRCode, SGR(..), ConsoleLayer(..), ColorIntensity(..))
 
 -- | Like 'logStdoutDev' from \"wai-extra\", but redacts (or omits altogether) payloads which might have
 -- sensitive information

src/Gargantext/API/Ngrams/List.hs

@@ -29,7 +29,7 @@ import Data.Vector qualified as Vec
 import Gargantext.API.Admin.EnvTypes (Env)
 import Gargantext.API.Errors.Types (BackendInternalError(InternalServerError))
 import Gargantext.API.Ngrams (setListNgrams)
-import Gargantext.API.Ngrams.List.Types
+import Gargantext.API.Ngrams.List.Types (_wjf_data, _wtf_data)
 import Gargantext.API.Ngrams.Prelude (getNgramsList)
 import Gargantext.API.Ngrams.Types
 import Gargantext.API.Prelude (GargM, serverError, HasServerError)

src/Gargantext/API/Ngrams/List/Types.hs

@@ -13,20 +13,17 @@ Portability : POSIX
 
 module Gargantext.API.Ngrams.List.Types where
 
---import Control.Lens hiding (elements, Indexed)
 import Data.Aeson
-import qualified Data.ByteString.Lazy as BSL
+import Data.ByteString.Lazy qualified as BSL
 import Data.Swagger (ToSchema, declareNamedSchema, genericDeclareNamedSchema)
 import Data.Text
-import qualified Data.Text.Encoding as E
-import Servant.Job.Utils (jsonOptions)
-import Web.FormUrlEncoded (FromForm(..), ToForm, parseUnique)
-
-import Protolude
-
+import Data.Text.Encoding qualified as E
 import Gargantext.API.Ngrams.Types (NgramsList)
 import Gargantext.API.Node.Corpus.New.Types (FileType(..))
+import Gargantext.Core.Utils.Aeson (jsonOptions)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
+import Protolude
+import Web.FormUrlEncoded (FromForm(..), ToForm, parseUnique)
 
 
 

src/Gargantext/API/Ngrams/Types.hs

@@ -43,7 +43,9 @@ import Gargantext.Core.Text (size)
 import Gargantext.Core.Text.Ngrams qualified as Ngrams
 import Gargantext.Core.Types (ListType(..), ListId, NodeId, TODO)
 import Gargantext.Core.Types.Query (Limit, Offset, MaxSize, MinSize)
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixUntagged, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Aeson (jsonOptions)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixUntagged, unPrefixSwagger)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Database.Admin.Types.Node (ContextId)
 import Gargantext.Database.Prelude (fromField', HasConnectionPool, CmdM')
 import Gargantext.Prelude hiding (IsString, hash, from, replace, to)
@@ -51,7 +53,6 @@ import Gargantext.Prelude.Crypto.Hash (IsHashable(..))
 import Gargantext.Utils.Servant (TSV, ZIP)
 import Gargantext.Utils.Zip (zipContentsPure)
 import Servant ( FromHttpApiData(parseUrlPiece), ToHttpApiData(toUrlPiece), Required, Strict, QueryParam', MimeRender(.. ), MimeUnrender(..))
-import Servant.Job.Utils (jsonOptions)
 
 ------------------------------------------------------------------------
 

src/Gargantext/API/Node/Corpus/Annuaire.hs

@@ -18,13 +18,13 @@ import Data.Aeson (genericParseJSON, genericToJSON)
 import Data.Swagger (ToSchema(..), genericDeclareNamedSchema)
 import Gargantext.API.Node.Corpus.New.Types qualified as NewTypes
 import Gargantext.Core (Lang(..))
+import Gargantext.Core.Utils.Aeson (jsonOptions)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)  -- flowAnnuaire
 import Gargantext.Database.Admin.Types.Node (AnnuaireId)
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs.Monad (MonadJobStatus(..))
 import Servant
-import Servant.Job.Utils (jsonOptions)
 import Web.FormUrlEncoded (FromForm)
 
 

src/Gargantext/API/Node/Get.hs

@@ -15,8 +15,8 @@ module Gargantext.API.Node.Get
       where
 
 import Data.Aeson
-import Data.Swagger
-import Gargantext.Database.Admin.Types.Node
+import Data.Swagger (ToSchema)
+import Gargantext.Database.Admin.Types.Node (NodeId, NodeType)
 import Gargantext.Prelude
 
 ------------------------------------------------------------------------

src/Gargantext/API/Node/Phylo/Export.hs

@@ -15,9 +15,9 @@ import Data.Aeson (Value)
 import Data.Text qualified as T
 import Gargantext.API.Prelude (GargNoServer, IsGargServer)
 import Gargantext.API.Routes.Named.Viz qualified as Named
-import Gargantext.Core.Viz.Phylo.API.Tools
+import Gargantext.Core.Viz.Phylo.API.Tools (getPhyloData, phylo2dot, phylo2dot2json)
 import Gargantext.Core.Viz.Phylo.Example (phyloCleopatre)
-import Gargantext.Database.Admin.Types.Node (PhyloId, NodeId,)
+import Gargantext.Database.Admin.Types.Node (PhyloId, NodeId)
 import Gargantext.Prelude
 import Servant
 import Servant.Server.Generic (AsServerT)

src/Gargantext/API/Node/Types.hs

@@ -24,10 +24,10 @@ import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.Corpus.Query qualified as API
 import Gargantext.Core.Text.List.Social (FlowSocialListWith)
 import Gargantext.Core.Types (NodeId)
+import Gargantext.Core.Utils.Aeson (jsonOptions)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.GargDB qualified as GargDB
 import Gargantext.Prelude
-import Servant.Job.Utils (jsonOptions)
 import Web.FormUrlEncoded (FromForm, ToForm)
 
 -------------------------------------------------------

src/Gargantext/API/Prelude.hs

@@ -15,28 +15,25 @@ Portability : POSIX
 module Gargantext.API.Prelude
   ( module Gargantext.API.Prelude
   , HasServerError(..)
-  , serverError
-  )
-  where
+  , serverError ) where
 
 import Control.Lens ((#))
-import Data.Aeson.Types
 import Gargantext.API.Admin.Auth.Types (AuthenticationError)
-import Gargantext.API.Errors.Class
+import Gargantext.API.Errors.Class (HasAuthenticationError, _AuthenticationError)
+import Gargantext.API.Errors.Types (HasServerError(..), serverError)
 import Gargantext.Core.Notifications.CentralExchange.Types (HasCentralExchangeNotification)
 import Gargantext.Core.Config (HasConfig)
 import Gargantext.Core.Mail.Types (HasMail)
 import Gargantext.Core.NLP (HasNLPServer)
-import Gargantext.Core.NodeStory
-import Gargantext.Core.Types
+import Gargantext.Core.NodeStory (HasNodeStory, HasNodeStoryEnv)
+import Gargantext.Core.Types (HasValidationError)
 import Gargantext.Database.Prelude (CmdM, CmdRandom, HasConnectionPool)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError(..))
-import Gargantext.Database.Query.Tree
+import Gargantext.Database.Query.Tree (HasTreeError)
 import Gargantext.Prelude
-import Gargantext.System.Logging
+import Gargantext.System.Logging (MonadLogger)
 import Gargantext.Utils.Jobs.Monad (MonadJobStatus(..), JobHandle)
 import Servant
-import Servant.Job.Core (HasServerError(..), serverError)
 
 authenticationError :: (MonadError e m, HasAuthenticationError e) => AuthenticationError -> m a
 authenticationError = throwError . (_AuthenticationError #)

src/Gargantext/API/Public/Types.hs

+{-|
+Module      : Gargantext.API.Public.Types
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
 
 module Gargantext.API.Public.Types (
     PublicData(..)
-  , defaultPublicData
   ) where
 
 import Data.Aeson
-import Data.Swagger
+import Data.Swagger (ToSchema)
 import Data.Text (Text)
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.Utils.Aeson qualified as GUA
-import Prelude
-import Test.QuickCheck
 
 data PublicData = PublicData
   { title    :: Text
@@ -31,16 +38,3 @@ instance ToJSON    PublicData where
   toJSON = genericToJSON (defaultOptions { sumEncoding = GUA.defaultTaggedObject })
 
 instance ToSchema  PublicData
-instance Arbitrary PublicData where
-  arbitrary = elements
-            $ replicate 6 defaultPublicData
-
-defaultPublicData :: PublicData
-defaultPublicData =
-  PublicData { title = "Title"
-             , abstract = foldl (<>) "" $ replicate 100 "abstract "
-             , img = "images/Gargantextuel-212x300.jpg"
-             , url = "https://.."
-             , date = "YY/MM/DD"
-             , database = "database"
-             , author = "Author" }

src/Gargantext/API/Routes.hs

@@ -17,18 +17,16 @@ Portability : POSIX
 module Gargantext.API.Routes
       where
 
-import Data.Validity
 import Gargantext.API.Admin.EnvTypes (Env)
-import Gargantext.API.Errors.Types
-import Gargantext.API.Prelude
+import Gargantext.API.Errors.Types (BackendInternalError)
+import Gargantext.API.Prelude (GargServer, GargM)
 import Gargantext.API.Routes.Named.Annuaire qualified as Named
 import Gargantext.API.Routes.Named.Corpus qualified as Named
 import Gargantext.API.Worker (serveWorkerAPI)
 import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Core.Worker.Jobs.Types qualified as Jobs
 import Gargantext.Prelude
-import Servant
-import Servant.Auth.Swagger ()
+import Servant (Get, JSON)
 import Servant.Server.Generic (AsServerT)
 
 ----------------------------------------------------------------------

src/Gargantext/API/Routes/Named.hs

@@ -18,20 +18,20 @@ module Gargantext.API.Routes.Named (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Admin.Auth.Types
 import Gargantext.API.Admin.FrontEnd (FrontEndAPI)
-import Gargantext.API.GraphQL
-import Gargantext.API.Routes.Named.Private
-import Gargantext.API.Routes.Named.Public
-import Gargantext.API.Routes.Types
+import Gargantext.API.GraphQL (GraphQLAPI)
+import Gargantext.API.Routes.Named.Private (GargPrivateAPI)
+import Gargantext.API.Routes.Named.Public (GargPublicAPI)
+import Gargantext.API.Routes.Types (WithCustomErrorScheme)
 import Gargantext.API.Worker (WorkerAPI)
 import Gargantext.Core.Notifications.Dispatcher.WebSocket qualified as Dispatcher
 import Servant.API ((:>), (:-), JSON, ReqBody, Post, Get, QueryParam)
 import Servant.API.Description (Summary)
-import Servant.API.NamedRoutes
-import Servant.Auth.Swagger ()
-import Servant.Swagger.UI
+import Servant.API.NamedRoutes (NamedRoutes)
+import Servant.Auth.Swagger ()  -- toSwagger instance
+import Servant.Swagger.UI (SwaggerSchemaUI)
 
 
 newtype API mode = API

src/Gargantext/API/Routes/Named/Annuaire.hs

@@ -16,10 +16,10 @@ module Gargantext.API.Routes.Named.Annuaire (
     AddAnnuaireWithForm(..)
   ) where
 
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Node.Corpus.Annuaire (AnnuaireWithForm)
 import Gargantext.API.Worker (WorkerAPI)
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (AnnuaireId)
 import Servant
 
 newtype AddAnnuaireWithForm mode = AddAnnuaireWithForm

src/Gargantext/API/Routes/Named/Contact.hs

@@ -10,12 +10,12 @@ module Gargantext.API.Routes.Named.Contact (
   ) where
 
 
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Node.Contact.Types (AddContactParams(..))
 import Gargantext.API.Routes.Named.Node (NodeNodeAPI(..))
 import Gargantext.API.Worker (WorkerAPI)
-import Gargantext.Database.Admin.Types.Hyperdata.Contact
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Hyperdata.Contact (HyperdataContact)
+import Gargantext.Database.Admin.Types.Node (NodeId)
 import Servant
 
 

src/Gargantext/API/Routes/Named/Context.hs

@@ -5,8 +5,8 @@ module Gargantext.API.Routes.Named.Context (
     ContextAPI(..)
   ) where
 
-import GHC.Generics
-import Gargantext.Database.Admin.Types.Node
+import GHC.Generics (Generic)
+import Gargantext.Database.Admin.Types.Node (Node)
 import Servant
 
 data ContextAPI a mode = ContextAPI

src/Gargantext/API/Routes/Named/Corpus.hs

@@ -25,13 +25,12 @@ import Data.Aeson.TH (deriveJSON)
 import Data.Swagger (ToSchema(..), genericDeclareNamedSchema)
 import Data.Text (Text)
 import GHC.Generics
--- import Gargantext.API.Admin.Orchestrator.Types
-import Gargantext.API.Node.Corpus.Export.Types
-import Gargantext.API.Node.Types
+import Gargantext.API.Node.Corpus.Export.Types (Corpus)
+import Gargantext.API.Node.Types (NewWithForm, WithQuery)
 import Gargantext.API.Worker (WorkerAPI)
 import Gargantext.Core.Text.Ngrams (NgramsType(..))
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (CorpusId, ListId)
 import Gargantext.Prelude (Bool)
 import Servant
 

src/Gargantext/API/Routes/Named/Count.hs

@@ -8,9 +8,9 @@ module Gargantext.API.Routes.Named.Count (
   , module X
   ) where
 
-import GHC.Generics
-import Servant
+import GHC.Generics (Generic)
 import Gargantext.API.Count.Types as X
+import Servant
 
 
 newtype CountAPI mode = CountAPI

src/Gargantext/API/Routes/Named/Document.hs

@@ -24,8 +24,8 @@ module Gargantext.API.Routes.Named.Document (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
-import Gargantext.API.Node.Document.Export.Types
+import GHC.Generics (Generic)
+import Gargantext.API.Node.Document.Export.Types (DocumentExport, DocumentExportZIP)
 import Gargantext.API.Node.DocumentsFromWriteNodes.Types ( Params(..) )
 import Gargantext.API.Node.DocumentUpload.Types ( DocumentUpload(..),  )
 import Gargantext.API.Worker (WorkerAPI)

src/Gargantext/API/Routes/Named/EKG.hs

@@ -6,7 +6,7 @@ module Gargantext.API.Routes.Named.EKG (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
+import GHC.Generics (Generic)
 import Servant
 import System.Metrics.Json qualified as J
 

src/Gargantext/API/Routes/Named/File.hs

@@ -7,9 +7,9 @@ module Gargantext.API.Routes.Named.File (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
-import Gargantext.API.Node.File.Types
-import Gargantext.API.Node.Types
+import GHC.Generics (Generic)
+import Gargantext.API.Node.File.Types (BSResponse, RESPONSE)
+import Gargantext.API.Node.Types (NewWithFile)
 import Gargantext.API.Worker (WorkerAPI)
 import Servant
 

src/Gargantext/API/Routes/Named/FrameCalc.hs

@@ -5,10 +5,10 @@ module Gargantext.API.Routes.Named.FrameCalc (
     FrameCalcAPI(..)
   ) where
 
-import Servant
-import GHC.Generics
 import Gargantext.API.Node.FrameCalcUpload.Types (FrameCalcUpload)
 import Gargantext.API.Worker (WorkerAPI)
+import GHC.Generics (Generic)
+import Servant
 
 
 newtype FrameCalcAPI mode = FrameCalcAPI

src/Gargantext/API/Routes/Named/List.hs

@@ -19,12 +19,12 @@ module Gargantext.API.Routes.Named.List (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
-import Gargantext.API.Ngrams.List.Types
-import Gargantext.API.Ngrams.Types
+import GHC.Generics (Generic)
+import Gargantext.API.Ngrams.List.Types (WithJsonFile, WithTextFile)
+import Gargantext.API.Ngrams.Types (NgramsList, NgramsListZIP, NgramsTableMap)
 import Gargantext.API.Types (HTML)
 import Gargantext.API.Worker (WorkerAPI)
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (ListId)
 import Gargantext.Utils.Servant qualified as GUS
 import Servant
 

src/Gargantext/API/Routes/Named/Metrics.hs

@@ -10,17 +10,17 @@ module Gargantext.API.Routes.Named.Metrics (
   ) where
 
 import Data.Text (Text)
-import Data.Time
-import Data.Vector
-import GHC.Generics
-import Gargantext.API.HashedResponse
-import Gargantext.API.Ngrams.NgramsTree
-import Gargantext.API.Ngrams.Types
-import Gargantext.Core.Types.Main
+import Data.Time (UTCTime)
+import Data.Vector (Vector)
+import GHC.Generics (Generic)
+import Gargantext.API.HashedResponse (HashedResponse)
+import Gargantext.API.Ngrams.NgramsTree (NgramsTree)
+import Gargantext.API.Ngrams.Types (QueryParamR, TabType)
+import Gargantext.Core.Types.Main (ListType)
 import Gargantext.Core.Types.Query (Limit)
-import Gargantext.Core.Viz.Types
-import Gargantext.Database.Admin.Types.Metrics
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Core.Viz.Types (Histo)
+import Gargantext.Database.Admin.Types.Metrics (ChartMetrics, Metrics)
+import Gargantext.Database.Admin.Types.Node (ListId)
 import Servant
 
 

src/Gargantext/API/Routes/Named/Node.hs

@@ -28,24 +28,24 @@ module Gargantext.API.Routes.Named.Node (
   , UpdateNodeParams(..)
   ) where
 
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Auth.PolicyCheck ( PolicyChecked )
 import Gargantext.API.Ngrams.Types (TabType(..))
 import Gargantext.API.Node.New.Types ( PostNode(..) )
 import Gargantext.API.Node.Types ( RenameNode(..), NodesToScore(..), NodesToCategory(..) )
 import Gargantext.API.Node.Update.Types ( UpdateNodeParams(..), Charts(..), Granularity(..), Method(..) )
-import Gargantext.API.Routes.Named.Document
-import Gargantext.API.Routes.Named.File
-import Gargantext.API.Routes.Named.FrameCalc
-import Gargantext.API.Routes.Named.Metrics
+import Gargantext.API.Routes.Named.Document (DocumentsFromWriteNodesAPI, DocumentUploadAPI)
+import Gargantext.API.Routes.Named.File (FileAsyncAPI, FileAPI)
+import Gargantext.API.Routes.Named.FrameCalc (FrameCalcAPI)
+import Gargantext.API.Routes.Named.Metrics (ChartAPI, PieAPI, ScatterAPI, TreeAPI)
 import Gargantext.API.Routes.Named.Publish (PublishAPI)
-import Gargantext.API.Routes.Named.Search
-import Gargantext.API.Routes.Named.Share as Share
-import Gargantext.API.Routes.Named.Table
-import Gargantext.API.Routes.Named.Viz
+import Gargantext.API.Routes.Named.Search (SearchAPI, SearchResult)
+import Gargantext.API.Routes.Named.Share (ShareNode, UnshareNode)
+import Gargantext.API.Routes.Named.Table (TableAPI, TableNgramsAPI)
+import Gargantext.API.Routes.Named.Viz (PhyloAPI)
 import Gargantext.API.Worker (WorkerAPI)
 import Gargantext.Core.Types
-import Gargantext.Core.Types.Query
+import Gargantext.Core.Types.Query (Limit, Offset)
 import Gargantext.Database.Admin.Types.Hyperdata.User ( HyperdataUser )
 import Gargantext.Database.Query.Facet.Types ( FacetDoc, OrderBy(..) )
 import Prelude
@@ -82,7 +82,7 @@ data NodeAPI a mode = NodeAPI
   , scoreAPI           :: mode :- "score" :> NamedRoutes ScoreAPI
   , searchAPI          :: mode :- "search" :> NamedRoutes (SearchAPI SearchResult)
   , shareAPI           :: mode :- "share" :> NamedRoutes ShareNode
-  , unshareEp          :: mode :- "unshare" :> NamedRoutes Share.UnshareNode
+  , unshareEp          :: mode :- "unshare" :> NamedRoutes UnshareNode
   , publishAPI         :: mode :- "publish" :> (PolicyChecked (NamedRoutes PublishAPI))
   ---- Pairing utilities
   , pairWithEp         :: mode :- "pairwith" :> NamedRoutes PairWith

src/Gargantext/API/Routes/Named/Private.hs

@@ -26,23 +26,23 @@ module Gargantext.API.Routes.Named.Private (
 
 
 import Data.Text (Text)
-import GHC.Generics
-import Gargantext.API.Admin.Auth.Types
-import Gargantext.API.Auth.PolicyCheck
-import Gargantext.API.Routes.Named.Contact
-import Gargantext.API.Routes.Named.Context
-import Gargantext.API.Routes.Named.Corpus
-import Gargantext.API.Routes.Named.Count
-import Gargantext.API.Routes.Named.Document
-import Gargantext.API.Routes.Named.List qualified as List
-import Gargantext.API.Routes.Named.Node
-import Gargantext.API.Routes.Named.Share
-import Gargantext.API.Routes.Named.Table
-import Gargantext.API.Routes.Named.Tree
-import Gargantext.API.Routes.Named.Viz
-import Gargantext.Database.Admin.Types.Hyperdata.Any
-import Gargantext.Database.Admin.Types.Hyperdata.Corpus
-import Gargantext.Database.Admin.Types.Node
+import GHC.Generics (Generic)
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
+import Gargantext.API.Auth.PolicyCheck (PolicyChecked)
+import Gargantext.API.Routes.Named.Contact (ContactAPI)
+import Gargantext.API.Routes.Named.Context (ContextAPI)
+import Gargantext.API.Routes.Named.Corpus (AddWithForm, AddWithQuery, CorpusExportAPI, MakeSubcorpusAPI)
+import Gargantext.API.Routes.Named.Count (CountAPI, Query)
+import Gargantext.API.Routes.Named.Document (DocumentExportAPI)
+import Gargantext.API.Routes.Named.List (GETAPI, JSONAPI, TSVAPI)
+import Gargantext.API.Routes.Named.Node (NodeAPI, NodesAPI, NodeNodeAPI, Roots)
+import Gargantext.API.Routes.Named.Share (ShareURL)
+import Gargantext.API.Routes.Named.Table (TableNgramsAPI)
+import Gargantext.API.Routes.Named.Tree (NodeTreeAPI, TreeFlatAPI)
+import Gargantext.API.Routes.Named.Viz (GraphAPI, PhyloExportAPI)
+import Gargantext.Database.Admin.Types.Hyperdata.Any (HyperdataAny)
+import Gargantext.Database.Admin.Types.Hyperdata.Corpus (HyperdataAnnuaire, HyperdataCorpus)
+import Gargantext.Database.Admin.Types.Node (ContextId, CorpusId, DocId, NodeId)
 import Servant.API
 import Servant.Auth qualified as SA
 
@@ -97,9 +97,9 @@ data GargPrivateAPI' mode = GargPrivateAPI'
   , addWithFormAPI   :: mode :- NamedRoutes AddWithForm
   , addWithQueryEp   :: mode :- NamedRoutes AddWithQuery
   , makeSubcorpusAPI :: mode :- NamedRoutes MakeSubcorpusAPI
-  , listGetAPI       :: mode :- NamedRoutes List.GETAPI
-  , listJsonAPI      :: mode :- NamedRoutes List.JSONAPI
-  , listTsvAPI       :: mode :- NamedRoutes List.TSVAPI
+  , listGetAPI       :: mode :- NamedRoutes GETAPI
+  , listJsonAPI      :: mode :- NamedRoutes JSONAPI
+  , listTsvAPI       :: mode :- NamedRoutes TSVAPI
   , shareUrlAPI      :: mode :- "shareurl" :> NamedRoutes ShareURL
   } deriving Generic
 

src/Gargantext/API/Routes/Named/Public.hs

@@ -7,9 +7,9 @@ module Gargantext.API.Routes.Named.Public (
   , NodeAPI(..)
   ) where
 
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Public.Types qualified as Public
-import Gargantext.API.Routes.Named.File
+import Gargantext.API.Routes.Named.File (FileAPI)
 import Gargantext.Database.Admin.Types.Node ( NodeId )
 import Servant.API
 

src/Gargantext/API/Routes/Named/Publish.hs

+{-|
+Module      : Gargantext.API.Routes.Named.Publish
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+-}
+
 {-# LANGUAGE TypeOperators #-}
 
 module Gargantext.API.Routes.Named.Publish (
@@ -6,12 +16,11 @@ module Gargantext.API.Routes.Named.Publish (
   ) where
 
 import Data.Aeson as JS
-import Data.Swagger
+import Data.Swagger (ToSchema)
 import Gargantext.Database.Query.Table.NodeNode (NodePublishPolicy)
 import GHC.Generics (Generic)
 import Prelude
 import Servant
-import Test.QuickCheck
 
 newtype PublishRequest = PublishRequest
   { pubrq_policy :: NodePublishPolicy
@@ -28,9 +37,6 @@ instance FromJSON PublishRequest where
     pubrq_policy <- o JS..: "policy"
     pure $ PublishRequest{..}
 
-instance Arbitrary PublishRequest where
-  arbitrary = PublishRequest <$> arbitraryBoundedEnum
-
 newtype PublishAPI mode = PublishAPI
   { publishEp :: mode :- Summary "Publish a Corpus Node"
                       :> ReqBody '[JSON] PublishRequest

src/Gargantext/API/Routes/Named/Search.hs

@@ -12,10 +12,10 @@ module Gargantext.API.Routes.Named.Search (
   ) where
 
 
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Search.Types ( SearchQuery(..), SearchType(..), SearchResult(..), SearchResultTypes(..) )
 import Gargantext.Core.Types.Query (Limit, Offset)
-import Gargantext.Database.Query.Facet
+import Gargantext.Database.Query.Facet (OrderBy)
 import Servant
 
 

src/Gargantext/API/Routes/Named/Share.hs

@@ -13,12 +13,12 @@ module Gargantext.API.Routes.Named.Share (
   , ShareNodeParams(..)
   ) where
 
-import Data.Aeson
-import Data.Swagger
+import Data.Aeson (FromJSON(..), ToJSON(..), withText)
+import Data.Swagger (ToSchema, declareNamedSchema)
 import Data.Text qualified as T
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Node.Share.Types ( ShareNodeParams (..) )
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (NodeId, NodeType)
 import Network.URI (parseURI)
 import Prelude
 import Servant

src/Gargantext/API/Routes/Named/Table.hs

@@ -17,15 +17,15 @@ module Gargantext.API.Routes.Named.Table (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
-import Gargantext.API.HashedResponse
+import GHC.Generics (Generic)
+import Gargantext.API.HashedResponse (HashedResponse)
 import Gargantext.API.Ngrams.Types (TabType(..), UpdateTableNgramsCharts, Version, QueryParamR, Versioned, VersionedWithCount, NgramsTable, NgramsTablePatch)
 import Gargantext.API.Ngrams.Types qualified as Ngrams
 import Gargantext.API.Worker (WorkerAPI)
 import Gargantext.API.Table.Types ( TableQuery(..), FacetTableResult )
 import Gargantext.Core.Text.Corpus.Query (RawQuery)
 import Gargantext.Core.Types.Main (ListType)
-import Gargantext.Core.Types.Query
+import Gargantext.Core.Types.Query (Limit, MinSize, MaxSize, Offset)
 import Gargantext.Database.Admin.Types.Node (ListId)
 import Gargantext.Database.Query.Facet.Types qualified as Facet
 import Prelude

src/Gargantext/API/Routes/Named/Tree.hs

@@ -7,9 +7,9 @@ module Gargantext.API.Routes.Named.Tree (
   ) where
 
 import Data.Text (Text)
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.Core.Types.Main
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (NodeType)
 import Servant
 
 data NodeTreeAPI mode = NodeTreeAPI

src/Gargantext/API/Routes/Named/Viz.hs

@@ -18,11 +18,11 @@ module Gargantext.API.Routes.Named.Viz (
 
 import Data.Aeson ( Value )
 import Data.Text (Text)
-import GHC.Generics
+import GHC.Generics (Generic)
 import Gargantext.API.Viz.Types (PhyloData(..))
 import Gargantext.API.Worker (WorkerAPI)
-import Gargantext.Core.Types
-import Gargantext.Core.Viz.Graph.Types
+import Gargantext.Core.Types (ListId, NodeId)
+import Gargantext.Core.Viz.Graph.Types (Graph, GraphLegendAPI, GraphVersions(..), HyperdataGraphAPI)
 import Gargantext.Core.Viz.LegacyPhylo (Level)
 import Gargantext.Core.Viz.Phylo.Legacy.LegacyMain (MinSizeBranch)
 import Servant

src/Gargantext/API/Routes/Types.hs

 {-# LANGUAGE TypeFamilies         #-}
 {-# LANGUAGE ScopedTypeVariables  #-}
 {-# LANGUAGE TypeApplications     #-}
+
 module Gargantext.API.Routes.Types where
 
-import Control.Lens
+import Control.Lens ((&), (%~), traversed)
 import Data.ByteString (ByteString)
 import Data.CaseInsensitive qualified as CI
 import Data.List qualified as L
-import Data.Proxy
+import Data.Proxy (Proxy(..))
 import Data.Set qualified as Set
-import Gargantext.API.Errors
-import Network.Wai hiding (responseHeaders)
+import Gargantext.API.Errors (GargErrorScheme(..), renderGargErrorScheme)
+import Network.HTTP.Types (HeaderName)
+import Network.Wai (requestHeaders)
 import Prelude
-import Servant.API.Routes
+import Servant.API.Routes (HasRoutes, getRoutes, mkHeaderRep, responseHeaders)
 import Servant.API.Routes.Internal.Response (unResponses)
-import Servant.API.Routes.Route
-import Servant.Client hiding (responseHeaders)
+import Servant.API.Routes.Route (routeResponse)
+import Servant.Client (HasClient, Client, clientWithRoute, hoistClientMonad)
 import Servant.Client.Core.Request (addHeader)
-import Servant.Ekg
-import Servant.Server
-import Servant.Server.Internal.Delayed
-import Servant.Server.Internal.DelayedIO
-import Network.HTTP.Types (HeaderName)
+import Servant.Ekg (HasEndpoint, enumerateEndpoints, getEndpoint)
+import Servant.Server (HasServer, ServerT, hoistServerWithContext, route)
+import Servant.Server.Internal.Delayed (addHeaderCheck)
+import Servant.Server.Internal.DelayedIO (DelayedIO, withRequest)
 
 data WithCustomErrorScheme a
 

src/Gargantext/API/Search.hs

@@ -22,14 +22,14 @@ module Gargantext.API.Search
 import Data.Text qualified as T
 import Gargantext.API.Prelude (IsGargServer)
 import Gargantext.API.Routes.Named.Search qualified as Named
-import Gargantext.API.Search.Types
+import Gargantext.API.Search.Types (SearchQuery(..), SearchResult(..), SearchResultTypes(..), SearchType(..))
 import Gargantext.Core.Text.Corpus.Query (RawQuery (..), parseQuery)
-import Gargantext.Core.Types.Search
+import Gargantext.Core.Types.Search (toRow)
 import Gargantext.Database.Action.Flow.Pairing (isPairedWith)
-import Gargantext.Database.Action.Search
-import Gargantext.Database.Admin.Types.Node hiding (DEBUG)
+import Gargantext.Database.Action.Search (searchInCorpus, searchInCorpusWithContacts)
+import Gargantext.Database.Admin.Types.Node (NodeId, NodeType(..))
 import Gargantext.Prelude
-import Gargantext.System.Logging
+import Gargantext.System.Logging (logLocM, LogLevel(..))
 import Servant.Server.Generic (AsServerT)
 
 -----------------------------------------------------------------------

src/Gargantext/API/Search/Types.hs

 
 module Gargantext.API.Search.Types where
 
-import GHC.Generics
-import Data.Aeson hiding (defaultTaggedObject)
-import Data.Swagger hiding (fieldLabelModifier, Contact)
+import Data.Aeson (defaultOptions, genericParseJSON, genericToJSON, sumEncoding, SumEncoding(..))
+import Data.Swagger (ToSchema, declareNamedSchema, genericDeclareNamedSchema)
 import Gargantext.Core.Text.Corpus.Query (RawQuery (..))
-import Gargantext.Core.Types.Search
+import Gargantext.Core.Types.Search (Row)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Prelude
 import Gargantext.Utils.Aeson (defaultTaggedObject)
-import Test.QuickCheck (elements)
-import Test.QuickCheck.Arbitrary
 
 -----------------------------------------------------------------------
 -----------------------------------------------------------------------
@@ -23,8 +20,6 @@ instance FromJSON SearchType where
 instance ToJSON SearchType where
   toJSON = genericToJSON (defaultOptions { sumEncoding = ObjectWithSingleField })
 instance ToSchema SearchType
-instance Arbitrary SearchType where
-  arbitrary = elements [SearchDoc, SearchContact]
 
 -----------------------------------------------------------------------
 data SearchQuery =
@@ -42,9 +37,6 @@ instance ToSchema SearchQuery
     declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "")
 -}
 
-instance Arbitrary SearchQuery where
-  arbitrary = elements [SearchQuery (RawQuery "electrodes") SearchDoc]
-  -- arbitrary = elements [SearchQuery "electrodes" 1 ] --SearchDoc]
 -----------------------------------------------------------------------
 data SearchResult =
   SearchResult { result :: !SearchResultTypes}
@@ -62,9 +54,6 @@ instance ToSchema SearchResult
     declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "")
 -}
 
-instance Arbitrary SearchResult where
-  arbitrary = SearchResult <$> arbitrary
-
 
 data SearchResultTypes =
     SearchResultDoc { docs     :: ![Row] }
@@ -76,13 +65,6 @@ instance FromJSON SearchResultTypes where
 instance ToJSON SearchResultTypes where
   toJSON = genericToJSON (defaultOptions { sumEncoding = defaultTaggedObject })
 
-instance Arbitrary SearchResultTypes where
-  arbitrary = do
-    srd <- SearchResultDoc     <$> arbitrary
-    src <- SearchResultContact <$> arbitrary
-    srn <- pure $ SearchNoResult "No result because.."
-    elements [srd, src, srn]
-
 instance ToSchema SearchResultTypes where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "")
 

src/Gargantext/API/Server/Named.hs

@@ -29,7 +29,7 @@ import Gargantext.Prelude hiding (Handler, catch)
 import Gargantext.System.Logging (logLocM, LogLevel(..))
 import Paths_gargantext qualified as PG -- cabal magic build module
 import Servant
-import Servant.Server.Generic
+import Servant.Server.Generic (AsServer, AsServerT)
 import Servant.Swagger.UI (swaggerSchemaUIServer)
 
 serverGargAPI :: Env -> BackEndAPI (AsServerT (GargM Env BackendInternalError))

src/Gargantext/API/Server/Named/EKG.hs

@@ -3,20 +3,21 @@
 {-# LANGUAGE TypeOperators        #-}
 {-# LANGUAGE UndecidableInstances #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
+
 module Gargantext.API.Server.Named.EKG where
 
 import Data.HashMap.Strict as HM
 import Data.Text as T
 import Data.Text.IO as T
 import Data.Time.Clock.POSIX (getPOSIXTime)
-import Gargantext.API.Routes.Named.EKG
-import Network.Wai
+import Gargantext.API.Routes.Named.EKG (EkgAPI(..))
+import Network.Wai (Middleware)
 import Protolude
 import Servant
-import Servant.Auth
-import Servant.Ekg
-import Servant.Server.Generic
-import System.Metrics
+import Servant.Auth (Auth)
+import Servant.Ekg (HasEndpoint, enumerateEndpoints, getEndpoint, monitorEndpoints)
+import Servant.Server.Generic (AsServer)
+import System.Metrics (Store, newStore, registerCounter, registerGcMetrics, sampleAll)
 import System.Metrics.Json qualified as J
 
 

src/Gargantext/API/Server/Named/Ngrams.hs

@@ -11,16 +11,16 @@ import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Gargantext.API.Admin.Auth (withNamedAccess)
 import Gargantext.API.Admin.Auth.Types (AuthenticatedUser, PathId (..))
-import Gargantext.API.Admin.EnvTypes
+import Gargantext.API.Admin.EnvTypes (Env)
 import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Metrics qualified as Metrics
 import Gargantext.API.Ngrams
 import Gargantext.API.Ngrams.Types
-import Gargantext.API.Prelude
+import Gargantext.API.Prelude (GargM)
 import Gargantext.API.Routes.Named.Table qualified as Named
 import Gargantext.API.Worker (serveWorkerAPI)
 import Gargantext.Core.NodeStory.Types (HasNodeStory)
-import Gargantext.Core.Types hiding (Terms)
+import Gargantext.Core.Types (DocId, ListId, ListType(..), NodeId, NodeType(..))
 import Gargantext.Core.Types.Query (Limit(..), Offset(..))
 import Gargantext.Core.Worker.Jobs.Types qualified as Jobs
 import Gargantext.Database.Admin.Config (userMaster)
@@ -30,7 +30,7 @@ import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
 import Gargantext.Database.Query.Table.Node.Select ( selectNodesWithUsername )
 import Gargantext.Database.Schema.Node (node_id, node_parent_id, node_user_id)
 import Gargantext.Prelude
-import Gargantext.Utils.Jobs.Monad
+import Gargantext.Utils.Jobs.Monad (JobHandle, MonadJobStatus(..), markFailedNoErr)
 import Servant.Server.Generic (AsServerT)
 
 

src/Gargantext/API/Server/Named/Private.hs

 {-# OPTIONS_GHC -Wno-deprecations #-}
+
 module Gargantext.API.Server.Named.Private where
 
 import Gargantext.API.Admin.Auth.Types (AuthenticatedUser(..))
 import Gargantext.API.Admin.EnvTypes (Env)
-import Gargantext.API.Context
+import Gargantext.API.Context (contextAPI)
 import Gargantext.API.Count qualified as Count
-import Gargantext.API.Errors.Types
+import Gargantext.API.Errors.Types (BackendInternalError)
 import Gargantext.API.Members (members)
 import Gargantext.API.Ngrams.List qualified as List
-import Gargantext.API.Node
+import Gargantext.API.Node (annuaireNodeAPI, corpusNodeAPI, nodeAPI, nodeNodeAPI, nodesAPI, roots)
 import Gargantext.API.Node qualified as Tree
 import Gargantext.API.Node.Contact as Contact
 import Gargantext.API.Node.Corpus.Export qualified as CorpusExport
@@ -16,15 +17,14 @@ import Gargantext.API.Node.Corpus.Subcorpus qualified as Subcorpus
 import Gargantext.API.Node.Document.Export (documentExportAPI)
 import Gargantext.API.Node.Phylo.Export qualified as PhyloExport
 import Gargantext.API.Node.ShareURL ( shareURL )
-import Gargantext.API.Prelude
+import Gargantext.API.Prelude (GargM)
 import Gargantext.API.Routes (addCorpusWithForm, addCorpusWithQuery)
 import Gargantext.API.Routes.Named.Private qualified as Named
-import Gargantext.API.Server.Named.Ngrams
+import Gargantext.API.Server.Named.Ngrams (apiNgramsTableDoc)
 import Gargantext.API.Server.Named.Viz qualified as Viz
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Database.Admin.Types.Hyperdata
+import Gargantext.Database.Admin.Types.Hyperdata (HyperdataAny)
 import Gargantext.Prelude
-import Servant.Auth.Swagger ()
 import Servant.Server.Generic (AsServerT)
 
 ---------------------------------------------------------------------

src/Gargantext/API/Server/Named/Public.hs

@@ -9,7 +9,9 @@ import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Gargantext.API.Node.File (fileApi)
 import Gargantext.API.Prelude (serverError, IsGargServer)
-import Gargantext.API.Public.Types
+import Gargantext.API.Public.Types (PublicData(..))
+import Gargantext.API.Routes.Named.File qualified as Named
+import Gargantext.API.Routes.Named.Public qualified as Named
 import Gargantext.Core.Utils.DateUtils (utc2year)
 import Gargantext.Database.Admin.Types.Hyperdata.Corpus ( hc_fields )
 import Gargantext.Database.Admin.Types.Hyperdata.CorpusField
@@ -22,8 +24,6 @@ import Gargantext.Database.Schema.Node ( NodePoly(..), node_date, node_hyperdata
 import Gargantext.Prelude
 import Servant
 import Servant.Server.Generic (AsServerT)
-import qualified Gargantext.API.Routes.Named.File as Named
-import qualified Gargantext.API.Routes.Named.Public as Named
 
 serverPublicGargAPI :: IsGargServer env err m => Text -> Named.GargPublicAPI (AsServerT m)
 serverPublicGargAPI baseUrl =

src/Gargantext/API/Server/Named/Viz.hs

@@ -4,15 +4,14 @@ module Gargantext.API.Server.Named.Viz (
   ) where
 
 import Gargantext.API.Admin.Auth (withNamedAccess)
-import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser, PathId(..))
 import Gargantext.API.Admin.EnvTypes (Env)
 import Gargantext.API.Errors.Types ( BackendInternalError )
 import Gargantext.API.Prelude (GargM)
 import Gargantext.API.Routes.Named.Viz qualified as Named
 import Gargantext.Core.Viz.Graph.API
-import Gargantext.Core.Viz.Graph.GEXF ()
  -- (cooc2graph)
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node (NodeId, UserId)
 import Gargantext.Prelude
 import Servant.Server.Generic (AsServerT)
 

src/Gargantext/API/Swagger.hs

@@ -14,15 +14,14 @@ Portability : POSIX
 ---------------------------------------------------------------------
 module Gargantext.API.Swagger where
 ---------------------------------------------------------------------
-import Control.Lens
+import Control.Lens ((?~))
 import Data.Swagger
 import Data.Version (showVersion)
-import Servant
-import Servant.Swagger
-import qualified Paths_gargantext as PG -- cabal magic build module
-
 import Gargantext.API.Routes.Named qualified as Named
 import Gargantext.Prelude
+import Paths_gargantext qualified as PG -- cabal magic build module
+import Servant
+import Servant.Swagger (toSwagger, subOperations)
 
 backendApiProxy :: Proxy (ToServantApi Named.BackEndAPI)
 backendApiProxy = Proxy

src/Gargantext/API/Table.hs

@@ -30,23 +30,23 @@ module Gargantext.API.Table
   where
 
 import Data.Text qualified as T
-import Gargantext.API.HashedResponse
+import Gargantext.API.HashedResponse (HashedResponse(..), constructHashedResponse)
 import Gargantext.API.Ngrams.Types (TabType(..))
 import Gargantext.API.Prelude (IsGargServer)
 import Gargantext.API.Routes.Named.Table qualified as Named
-import Gargantext.API.Table.Types
+import Gargantext.API.Table.Types (FacetTableResult, TableQuery(..))
 import Gargantext.Core.Text.Corpus.Query (RawQuery, parseQuery, getRawQuery)
 import Gargantext.Core.Types (TableResult(..))
 import Gargantext.Core.Types.Query (Offset, Limit)
 import Gargantext.Database.Action.Learn (FavOrTrash(..), moreLike)
-import Gargantext.Database.Action.Search
-import Gargantext.Database.Admin.Types.Node hiding (ERROR, DEBUG)
+import Gargantext.Database.Action.Search (searchCountInCorpus, searchInCorpus)
+import Gargantext.Database.Admin.Types.Node (ContactId, CorpusId, NodeId)
 import Gargantext.Database.Prelude (CmdM, DbCmd', DBCmd)
 import Gargantext.Database.Query.Facet (FacetDoc , runViewDocuments, runCountDocuments, OrderBy(..), runViewAuthorsDoc)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
 import Gargantext.Prelude
 import Gargantext.System.Logging
-import Servant.Server.Generic
+import Servant.Server.Generic (AsServerT)
 
 tableApi :: IsGargServer env err m => NodeId -> Named.TableAPI (AsServerT m)
 tableApi id' = Named.TableAPI

src/Gargantext/API/Table/Types.hs

@@ -12,8 +12,6 @@ import Gargantext.Core.Types.Query (Offset, Limit)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.Query.Facet (FacetDoc , OrderBy(..))
 import Gargantext.Prelude
-import Test.QuickCheck (elements)
-import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
 
 data TableQuery = TableQuery
   { tq_offset       :: Offset
@@ -29,12 +27,3 @@ $(deriveJSON (unPrefix "tq_") ''TableQuery)
 
 instance ToSchema TableQuery where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "tq_")
-
-instance Arbitrary TableQuery where
-  arbitrary = elements [TableQuery { tq_offset = 0
-                                   , tq_limit = 10
-                                   , tq_orderBy = DateAsc
-                                   , tq_view = Docs
-                                   , tq_query = "electrodes" }]
-
-

src/Gargantext/API/ThrowAll.hs

@@ -25,10 +25,10 @@ module Gargantext.API.ThrowAll (
 
 import Control.Lens ((#))
 import Data.ByteString.Char8 qualified as C8
-import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Auth.Types (AuthenticatedUser(..))
 import Gargantext.API.Admin.EnvTypes (Env)
-import Gargantext.API.Errors.Types
-import Gargantext.API.Prelude
+import Gargantext.API.Errors.Types (BackendInternalError)
+import Gargantext.API.Prelude (GargM, _ServerError)
 import Gargantext.API.Routes.Named.Private qualified as Named
 import Gargantext.API.Server.Named.Private qualified as Named
 import Gargantext.Database.Admin.Types.Node (UserId (..))
@@ -36,7 +36,6 @@ import Gargantext.Prelude hiding (Handler)
 import Network.HTTP.Types.Status (Status(..))
 import Network.Wai (responseLBS)
 import Servant
-import Servant.API.Generic ()
 import Servant.Auth.Server (AuthResult(..))
 import Servant.Server.Generic (AsServerT)
 

src/Gargantext/API/Types.hs

@@ -14,23 +14,21 @@ Portability : POSIX
 
 module Gargantext.API.Types where
 
-import Data.Aeson
+import Data.Aeson (ToJSON, encode, eitherDecode)
+import Data.ByteString.Lazy.Char8 qualified as BS8
 import Data.Either (Either(..))
 import Data.List.NonEmpty (NonEmpty ((:|)))
 import Data.Text (Text)
-import Data.Typeable
-import Gargantext.API.Ngrams.Types ()
-import Gargantext.API.Node.Document.Export.Types ()
+import Data.Text.Encoding qualified as E
+import Data.Typeable (Typeable)
 import Gargantext.Core.Viz.Graph.Types (Graph(..))
 import Network.HTTP.Media ((//), (/:))
 import Prelude (($))
+import Prelude qualified
 import Servant.API.ContentTypes ( Accept(..) , MimeRender(..) , MimeUnrender(..) )
 import Servant.HTML.Blaze qualified as Blaze
-import Servant.Swagger.UI.Core
+import Servant.Swagger.UI.Core (SwaggerUiHtml(..))
 import Servant.XML.Conduit qualified as S
-import qualified Data.ByteString.Lazy.Char8 as BS8
-import qualified Data.Text.Encoding as E
-import qualified Prelude
 
 data HTML deriving (Typeable)
 instance Accept HTML where

src/Gargantext/Core/Notifications/Dispatcher/Types.hs

@@ -45,8 +45,6 @@ import Servant
 -- import Servant.API.NamedRoutes ((:-))
 import Servant.API.WebSocket qualified as WS
 import Servant.Auth.Server (verifyJWT)
-import Servant.Job.Core (Safety(Safe))
-import Servant.Job.Types (JobID, JobStatus(_job_id))
 import Servant.Server.Generic (AsServer, AsServerT)
 import StmContainers.Set as SSet
 

src/Gargantext/Core/Types.hs

@@ -38,7 +38,8 @@ import Data.Swagger (ToParamSchema, ToSchema(..))
 import Data.Text (unpack)
 import Data.Validity ( validationIsValid, Validation )
 import Gargantext.Core.Types.Main
-import Gargantext.Core.Utils.Prefix (unPrefix, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Database.Admin.Types.Node
 import Gargantext.Prelude hiding (Ordering, empty)
 

src/Gargantext/Core/Types/Main.hs

@@ -24,7 +24,8 @@ import Data.Swagger ( ToSchema(..), ToParamSchema, genericDeclareNamedSchema )
 import Data.Text (unpack, pack)
 import Data.TreeDiff
 import Gargantext.Core
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Database.Admin.Types.Node  -- (NodeType(..), Node, Hyperdata(..))
 import Gargantext.Prelude
 import Servant.API (FromHttpApiData(..), ToHttpApiData(..))

src/Gargantext/Core/Utils.hs

@@ -21,6 +21,8 @@ module Gargantext.Core.Utils (
                            , randomString
                            , groupWithCounts
                            , addTuples
+                           , (?!)
+                           , (?|)
                           ) where
 
 import Data.List qualified as List
@@ -30,6 +32,7 @@ import Data.Text qualified as T
 import Gargantext.Core.Utils.Prefix
 import Gargantext.Prelude
 import Prelude ((!!))
+import Prelude qualified
 import System.Random (initStdGen, uniformR)
 
 
@@ -71,3 +74,16 @@ groupWithCounts = map f
 
 addTuples :: (Num a, Num b) => (a, b) -> (a, b) -> (a, b)
 addTuples (a1, b1) (a2, b2) = (a1 + a2, b1 + b2)
+
+
+infixr 4 ?!
+
+-- Reverse infix form of "fromJust" with a custom error message
+(?!) :: Maybe a -> Prelude.String -> a
+(?!) ma msg = ma ?| errorTrace msg
+
+infixr 4 ?|
+
+-- Reverse infix form of "fromMaybe"
+(?|) :: Maybe a -> a -> a
+(?|) = flip fromMaybe

src/Gargantext/Core/Utils/Aeson.hs

+{-|
+Module      : Gargantext.Core.Utils.Aeson
+Description : 
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+module Gargantext.Core.Utils.Aeson where
+
+import Data.Aeson.Types
+import Gargantext.Core.Utils.Swagger (modifier)
+import Gargantext.Prelude
+
+jsonOptions :: Text -> Options
+jsonOptions pref = defaultOptions
+  { fieldLabelModifier = modifier pref
+  , unwrapUnaryRecords = False
+  , omitNothingFields = True
+  }

src/Gargantext/Core/Utils/Prefix.hs

@@ -14,7 +14,6 @@ commentary with @some markup@.
 
 module Gargantext.Core.Utils.Prefix
   ( module Gargantext.Core.Utils.Prefix
-  , wellNamedSchema
   ) where
 
 import Prelude
@@ -24,7 +23,6 @@ import Data.Aeson.TH (Options, fieldLabelModifier, omitNothingFields, sumEncodin
 import Data.Aeson.Types (Parser)
 import Data.Char (toLower)
 import Data.Swagger.SchemaOptions (SchemaOptions, fromAesonOptions)
-import Servant.Job.Utils (wellNamedSchema)
 import Text.Read (readMaybe)
 
 

src/Gargantext/Core/Utils/Swagger.hs

+{-|
+Module      : Gargantext.Core.Utils.Swagger
+Description : Swagger utilities
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+---------------------------------------------------------------------
+module Gargantext.Core.Utils.Swagger where
+---------------------------------------------------------------------
+import Control.Lens ((?~))
+import Data.Swagger
+import Data.Swagger qualified as S
+import Data.Swagger.Declare qualified as S
+import Data.Swagger.Internal.Schema qualified as S
+import Data.Swagger.Internal.TypeShape qualified as S
+import Data.Text qualified as T
+import Gargantext.Core.Utils ((?!))
+import Gargantext.Prelude
+import Prelude qualified
+
+
+wellNamedSchema ::
+     forall a.
+     ( Typeable a -- for the real full name
+     , Generic a
+     , S.GToSchema (Rep a)
+     , S.GenericHasSimpleShape a "genericDeclareNamedSchemaUnrestricted" (S.GenericShape (Rep a))
+     )
+  => Text
+  -> Proxy a
+  -> S.Declare (S.Definitions S.Schema) S.NamedSchema
+wellNamedSchema pref proxy =
+  (S.name ?~ (T.replace " " "_" . T.pack . show . typeRep) proxy) <$>
+  S.genericDeclareNamedSchema (swaggerOptions pref) proxy
+
+swaggerOptions :: Text -> SchemaOptions
+swaggerOptions pref = defaultSchemaOptions
+  { S.fieldLabelModifier = modifier pref
+  , S.unwrapUnaryRecords = False
+  }
+
+modifier :: Text -> Prelude.String -> Prelude.String
+modifier pref field = T.unpack $ T.stripPrefix pref (T.pack field) ?! "Expecting prefix " <> T.unpack pref

src/Gargantext/Database/Admin/Types/Hyperdata/Prelude.hs

@@ -48,7 +48,7 @@ import Data.Text (Text)
 import Database.PostgreSQL.Simple.FromField (FromField, fromField)
 import Database.PostgreSQL.Simple.ToField (ToField, toField, toJSONField)
 import GHC.Generics (Generic)
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
 import Gargantext.Database.Prelude (fromField', JSONB)
 import Gargantext.Prelude
 import Opaleye (DefaultFromField, defaultFromField, Nullable, SqlJsonb, fromPGSFromField)

src/Gargantext/Database/Admin/Types/Metrics.hs

@@ -16,7 +16,8 @@ import Data.Swagger (ToSchema(..), genericDeclareNamedSchema)
 import Data.Vector (Vector)
 import Data.Vector qualified as V
 import Gargantext.Core.Types (ListType(..))
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Prelude
 import Test.QuickCheck.Arbitrary
 

src/Gargantext/Database/Admin/Types/Node.hs

@@ -41,7 +41,8 @@ import Data.TreeDiff
 import Database.PostgreSQL.Simple.FromRow (FromRow, fromRow, field)
 import Fmt ( Buildable(..) )
 import Gargantext.Core (HasDBid(..))
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Database.Schema.Context
 import Gargantext.Database.Schema.Node ( NodePoly(Node), NodePolySearch(NodeSearch) )
 import Gargantext.Prelude

src/Gargantext/Database/Query/Facet/Types.hs

@@ -10,7 +10,8 @@ import Data.Text qualified as T
 import Data.Time (UTCTime)
 import Data.Time.Segment (jour)
 import Gargantext.Database.Admin.Types.Node ( NodeId )
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger, wellNamedSchema)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Core.Utils.Swagger (wellNamedSchema)
 import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument, arbitraryHyperdataDocuments )
 import Opaleye
 import Protolude hiding (null, map, sum, not)

src/Gargantext/Utils/Jobs/Monad.hs

@@ -24,7 +24,6 @@ module Gargantext.Utils.Jobs.Monad (
   , MonadJobStatus(..)
 
   -- * Functions
-  , genSecret
   , markFailedNoErr
   , markFailureNoErr
   ) where
@@ -36,12 +35,8 @@ import Data.Text qualified as T
 import Data.Void (Void)
 import Gargantext.Utils.Jobs.Error
 import Prelude
-import Servant.Job.Core qualified as SJ
 
 
-genSecret :: IO SJ.SecretKey
-genSecret = SJ.generateSecretKey
-
 data JobError
   =
   -- | We expected to find a job tagged internall as \"job\", but we found the input @T.Text@ instead.

stack.yaml

@@ -102,7 +102,6 @@
 - "servant-auth-swagger-0.2.11.0"
 - "servant-client-core-0.20.2"
 - "servant-ekg-0.3.1"
-- "servant-flatten-0.2"
 - "servant-server-0.20.2"
 - "servant-swagger-1.2.1"
 - "servant-swagger-ui-0.3.5.5.0.1"
@@ -176,10 +175,6 @@
   git: "https://github.com/adinapoli/llvm-hs.git"
   subdirs:
   - "llvm-hs-pure"
-- commit: 74a3296dfe1f0c4a3ade91336dcc689330e84156
-  git: "https://github.com/adinapoli/servant-job.git"
-  subdirs:
-  - .
 - commit: a110807651036ca2228a76507ee35bbf7aedf87a
   git: "https://github.com/alpmestan/accelerate-arithmetic.git"
   subdirs:

test-data/test_config.toml

@@ -81,7 +81,7 @@ All = "corenlp://localhost:9000"
 default_visibility_timeout = 1
 
 # default delay before job is visible to the worker
-default_delay = 0
+default_delay = 1
 
 # NOTE This is overridden by Test.Database.Setup
 [worker.database]

test/Test/API/Authentication.hs

@@ -35,7 +35,7 @@ cannedToken :: T.Text
 cannedToken = "eyJhbGciOiJIUzUxMiJ9.eyJkYXQiOnsiaWQiOjF9fQ.t49zZSqkPAulEkYEh4pW17H2uwrkyPTdZKwHyG3KUJ0hzU2UUoPBNj8vdv087RCVBJ4tXgxNbP4j0RBv3gxdqg"
 
 tests :: Spec
-tests = parallel $ aroundAll withTestDBAndPort $ beforeAllWith (\ctx -> setupEnvironment (_sctx_env ctx) >>= (const $ pure ctx)) $ do
+tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith (\ctx -> setupEnvironment (_sctx_env ctx) >>= (const $ pure ctx)) $ do
   describe "Authentication" $ do
     baseUrl <- runIO $ parseBaseUrl "http://localhost"
     manager <- runIO $ newManager defaultManagerSettings

test/Test/API/Errors.hs

@@ -24,7 +24,7 @@ import Text.RawString.QQ (r)
 
 
 tests :: Spec
-tests = parallel $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do
+tests = sequential $ aroundAll withTestDBAndPort $ beforeAllWith dbEnvSetup $ do
   describe "Errors API" $ do
     describe "Prelude" $ do
       it "setup DB triggers and users" $ \ctx -> do