[workers] preliminary drafts

cabal.project

@@ -165,7 +165,12 @@ source-repository-package
     type: git
     location: https://github.com/robstewart57/rdf4h.git
     tag: 4fd2edf30c141600ffad6d730cc4c1c08a6dbce4
-    
+
+source-repository-package
+    type: git
+    location: https://github.com/fpringle/servant-routes
+    tag: df649377e379393ed1f92bc3e5f685e1720a0f99
+        
 allow-older: *
 allow-newer: *
 

devops/docker/docker-compose.yaml

@@ -18,9 +18,27 @@ services:
   #    - ../dbs:/dbs
   #    - ../postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
 
-  postgres:
+  # postgres:
+  #   #image: 'postgres:latest'
+  #   image: 'postgres:14'
+  #   shm_size: 1g  # https://stackoverflow.com/a/56754077
+  #   network_mode: host
+  #   #command: ["postgres", "-c", "log_statement=all"]
+  #   #ports:
+  #   #  - 5432:5432
+  #   environment:
+  #     POSTGRES_USER: gargantua
+  #     POSTGRES_PASSWORD: C8kdcUrAQy66U
+  #     POSTGRES_DB: gargandbV5
+  #   volumes:
+  #     - garg-pgdata14:/var/lib/postgresql/data
+  #     - ../:/gargantext
+  #     - ../dbs:/dbs
+  #     - ../postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
+
+  postgres-16:
     #image: 'postgres:latest'
-    image: 'postgres:14'
+    image: 'postgres:16'
     shm_size: 1g  # https://stackoverflow.com/a/56754077
     network_mode: host
     #command: ["postgres", "-c", "log_statement=all"]
@@ -31,41 +49,42 @@ services:
       POSTGRES_PASSWORD: C8kdcUrAQy66U
       POSTGRES_DB: gargandbV5
     volumes:
-      - garg-pgdata14:/var/lib/postgresql/data
+      - garg-pgdata16:/var/lib/postgresql/data
       - ../:/gargantext
       - ../dbs:/dbs
       - ../postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
 
-  pgadmin:
-    image: 'dpage/pgadmin4'
-    ports:
-      - 8081:80
-    environment:
-      PGADMIN_DEFAULT_EMAIL: admin@localhost.lan
-      PGADMIN_DEFAULT_PASSWORD: admin
+  # pgadmin:
+  #   image: 'dpage/pgadmin4'
+  #   ports:
+  #     - 8081:80
+  #   environment:
+  #     PGADMIN_DEFAULT_EMAIL: admin@localhost.lan
+  #     PGADMIN_DEFAULT_PASSWORD: admin
 
-    depends_on:
-      - postgres
-    links:
-      - postgres
-    volumes:
-      - pgadmin:/var/lib/pgadmin
+  #   depends_on:
+  #     - postgres
+  #   links:
+  #     - postgres
+  #   volumes:
+  #     - pgadmin:/var/lib/pgadmin
 
-  corenlp:
-    #image: 'cgenie/corenlp-garg:latest'
-    image: 'cgenie/corenlp-garg:4.5.4'
-    ports:
-      - 9000:9000
+  # corenlp:
+  #   #image: 'cgenie/corenlp-garg:latest'
+  #   image: 'cgenie/corenlp-garg:4.5.4'
+  #   ports:
+  #     - 9000:9000
 
-  johnsnownlp:
-    image: 'johnsnowlabs/nlp-server:latest'
-    volumes:
-      - js-cache:/home/johnsnowlabs/cache_pretrained
-    ports:
-      - 5000:5000
+  # johnsnownlp:
+  #   image: 'johnsnowlabs/nlp-server:latest'
+  #   volumes:
+  #     - js-cache:/home/johnsnowlabs/cache_pretrained
+  #   ports:
+  #     - 5000:5000
 
 volumes:
   #garg-pgdata:
   garg-pgdata14:
+  garg-pgdata16:
   js-cache:
   pgadmin:

gargantext.cabal

@@ -461,7 +461,7 @@ library
     , accelerate-arithmetic ^>= 1.0.0.1
     , accelerate-llvm-native ^>= 1.3.0.0
     , accelerate-utility ^>= 1.0.0.1
-    , aeson ^>= 1.5.6.0
+    , aeson >= 2.0.3 && < 2.1
     , aeson-pretty ^>= 0.8.9
     , ansi-terminal
     , array ^>= 0.5.4.0
@@ -599,6 +599,7 @@ library
     , servant-flatten ^>= 0.2
     , servant-job >= 0.2.0.0
     , servant-multipart ^>= 0.12.1
+    -- , servant-routes
     , servant-server >= 0.18.3 && < 0.20
     , servant-swagger ^>= 1.1.10
     , servant-swagger-ui ^>= 0.3.5.3.5.0
@@ -626,12 +627,12 @@ library
     , tomland >= 1.3.3.2
     , tuple ^>= 0.3.0.2
     , unordered-containers ^>= 0.2.16.0
-    , unicode-collation >= 0.1.3.6
+    , unicode-collation >= 0.1.3.5 && < 0.2
     , uri-encode ^>= 1.5.0.7
     , utf8-string ^>= 1.0.2
     , uuid ^>= 1.3.15
-    , validity ^>= 0.11.0.1
-    , vector ^>= 0.12.3.0
+    , validity >= 0.12.0 && < 0.13
+    , vector >= 0.13.0 && < 0.13.2
     , wai >= 3.2.4
     , wai-app-static ^>= 3.1.7.3
     , wai-cors ^>= 0.2.7
@@ -673,7 +674,7 @@ executable gargantext-cli
   hs-source-dirs:
       bin/gargantext-cli
   build-depends:
-      aeson ^>= 1.5.6.0
+      aeson >= 2.0.3 && < 2.1
     , async ^>= 2.2.4
     , bytestring ^>= 0.10.12.0
     , cassava ^>= 0.5.2.0
@@ -688,7 +689,7 @@ executable gargantext-cli
     , split ^>= 0.2.3.4
     , text ^>= 1.2.4.1
     , unordered-containers ^>= 0.2.16.0
-    , vector ^>= 0.12.3.0
+    , vector >= 0.13.0 && < 0.13.2
 
 executable gargantext-db-obfuscation
   import:
@@ -786,7 +787,7 @@ executable gargantext-phylo
       StrictData
   ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -O2 -Wmissing-signatures
   build-depends:
-      aeson ^>= 1.5.6.0
+      aeson >= 2.0.3 && < 2.1
     , async ^>= 2.2.4
     , bytestring ^>= 0.10.12.0
     , cassava ^>= 0.5.2.0
@@ -803,7 +804,7 @@ executable gargantext-phylo
     , text ^>= 1.2.4.1
     , time ^>= 1.9.3
     , unordered-containers ^>= 0.2.16.0
-    , vector ^>= 0.7.3
+    , vector >= 0.13.0 && < 0.13.2
 
 executable gargantext-server
   import:
@@ -826,7 +827,7 @@ executable gargantext-server
     , postgresql-simple ^>= 0.6.4
     , text ^>= 1.2.4.1
     , unordered-containers ^>= 0.2.16.0
-    , vector ^>= 0.7.3
+    , vector >= 0.13.0 && < 0.13.2
 
 executable gargantext-upgrade
   import:
@@ -845,6 +846,57 @@ executable gargantext-upgrade
     , postgresql-simple ^>= 0.6.4
     , text ^>= 1.2.4.1
 
+executable gargantext-workers
+  import:
+      defaults
+    , optimized
+  main-is: Main.hs
+  other-modules:
+      Paths_gargantext
+  hs-source-dirs:
+      bin/gargantext-workers
+  build-depends:
+    , gargantext
+    , gargantext-prelude
+    , odd-jobs >=0.2.2 && < 0.3
+
+--     -- | cabal gen-bounds for odd-jobs  
+--     , aeson >= 2.0.3 && < 2.1
+--     , bytestring >= 0.11.4 && < 0.12
+--     , containers >= 0.6.5 && < 0.7
+--     , directory >= 1.3.6 && < 1.4
+--     , either >= 5.0.2 && < 5.1
+--     , fast-logger >= 3.1.2 && < 3.2
+--     , filepath >= 1.4.2 && < 1.5
+--     , foreign-store >= 0.2 && < 0.3
+--     , friendly-time >= 0.4.1 && < 0.5
+--     , generic-deriving >= 1.14.4 && < 1.15
+--     , hdaemonize >= 0.5.6 && < 0.6
+--     , hostname >= 1.0 && < 1.1
+--     , lucid >= 2.11.20230408 && < 2.12
+--     , monad-control >= 1.0.3 && < 1.1
+--     , monad-logger >= 0.3.40 && < 0.4
+--     , mtl >= 2.2.2 && < 2.3
+--     , optparse-applicative >= 0.17.1 && < 0.18
+--     , postgresql-simple >= 0.6.4 && < 0.7
+--     , resource-pool >= 0.4.0 && < 0.5
+--     , safe >= 0.3.19 && < 0.4
+--     , servant >= 0.19.1 && < 0.20
+--     , servant-lucid >= 0.9.0 && < 0.10
+--     , servant-server >= 0.19.2 && < 0.20
+--     , servant-static-th >= 1.0.0 && < 1.1
+--     , string-conv >= 0.2.0 && < 0.3
+--     , text >= 1.2.5 && < 1.3
+--     , text-conversions >= 0.3.1 && < 0.4
+--     , time >= 1.11.1 && < 1.12
+--     , timing-convenience >= 0.1 && < 0.2
+--     , unix >= 2.7.2 && < 2.8
+--     , unliftio >= 0.2.25 && < 0.3
+--     , unliftio-core >= 0.2.1 && < 0.3
+--     , unordered-containers >= 0.2.19 && < 0.3
+--     , wai >= 3.2.3 && < 3.3
+--     , warp >= 3.3.23 && < 3.4
+
 test-suite garg-test-tasty
   import:
     defaults
@@ -892,7 +944,7 @@ test-suite garg-test-tasty
   ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N
   build-depends:
       QuickCheck ^>= 2.14.2
-    , aeson ^>= 1.5.6.0
+    , aeson >= 2.0.3 && < 2.1
     , aeson-pretty ^>= 0.8.9
     , aeson-qq
     , async ^>= 2.2.4
@@ -956,9 +1008,9 @@ test-suite garg-test-tasty
     , tmp-postgres >= 1.34.1 && < 1.35
     , tree-diff
     , unordered-containers ^>= 0.2.16.0
-    , unicode-collation >= 0.1.3.6
-    , validity ^>= 0.11.0.1
-    , vector ^>= 0.12.3.0
+    , unicode-collation >= 0.1.3.5 && < 0.2
+    , validity >= 0.12.0 && < 0.13
+    , vector >= 0.13.0 && < 0.13.2
     , wai
     , wai-extra
     , warp
@@ -990,7 +1042,7 @@ test-suite garg-test-hspec
   ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N
   build-depends:
       QuickCheck ^>= 2.14.2
-    , aeson ^>= 1.5.6.0
+    , aeson >= 2.0.3 && < 2.1
     , aeson-qq
     , async ^>= 2.2.4
     , boolexpr ^>= 0.2
@@ -1046,7 +1098,7 @@ test-suite garg-test-hspec
     , tmp-postgres >= 1.34.1 && < 1.35
     , tree-diff
     , unordered-containers ^>= 0.2.16.0
-    , validity ^>= 0.11.0.1
+    , validity >= 0.12.0 && < 0.13
     , wai
     , wai-extra
     , warp
@@ -1084,9 +1136,9 @@ executable gargantext-phylo-profile
     , text
     , async
     , cryptohash
-    , aeson
+    , aeson >= 2.0.3 && < 2.1
     , split
-    , vector
+    , vector >= 0.13.0 && < 0.13.2
     , directory
   default-language: GHC2021
 

src/Gargantext/API/Routes.hs

@@ -10,7 +10,6 @@ Portability : POSIX
 -}
 
 {-# LANGUAGE ConstraintKinds      #-}
-{-# LANGUAGE KindSignatures       #-}
 {-# LANGUAGE ScopedTypeVariables  #-}
 {-# LANGUAGE TypeFamilies         #-}
 {-# LANGUAGE TypeOperators        #-}
@@ -21,16 +20,16 @@ module Gargantext.API.Routes
       where
 
 import Control.Lens (view)
-import Data.Validity
+import Data.List qualified as L
 import Gargantext.API.Admin.Auth (ForgotPasswordAPI, ForgotPasswordAsyncAPI, withAccess, withPolicyT)
 import Gargantext.API.Admin.Auth.Types (AuthRequest, AuthResponse, AuthenticatedUser(..), PathId(..))
 import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
 import Gargantext.API.Admin.FrontEnd (FrontEndAPI)
-import Gargantext.API.Auth.PolicyCheck
-import Gargantext.API.Context
+import Gargantext.API.Auth.PolicyCheck ( nodeChecks, PolicyChecked )
+import Gargantext.API.Context ( contextAPI, ContextAPI )
 import Gargantext.API.Count  (CountAPI, count, Query)
 import Gargantext.API.Errors (GargErrorScheme (..))
-import Gargantext.API.Errors.Types
+import Gargantext.API.Errors.Types ( BackendInternalError )
 import Gargantext.API.GraphQL qualified as GraphQL
 import Gargantext.API.Members (MembersAPI, members)
 import Gargantext.API.Ngrams (TableNgramsApi, apiNgramsTableDoc)
@@ -43,26 +42,26 @@ import Gargantext.API.Node.Corpus.Export.Types qualified as CorpusExport
 import Gargantext.API.Node.Corpus.New qualified as New
 import Gargantext.API.Node.Document.Export qualified as DocumentExport
 import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
-import Gargantext.API.Prelude
+import Gargantext.API.Prelude ( GargM, GargServer )
 import Gargantext.API.Public qualified as Public
 import Gargantext.Core.Types.Individu (User(..))
-import Gargantext.Core.Viz.Graph.API
-import Gargantext.Database.Admin.Types.Hyperdata
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Core.Viz.Graph.API ( graphAPI, GraphAPI )
+import Gargantext.Database.Admin.Types.Hyperdata.Any ( HyperdataAny )
+import Gargantext.Database.Admin.Types.Hyperdata.Corpus ( HyperdataAnnuaire, HyperdataCorpus )
+import Gargantext.Database.Admin.Types.Node ( ContextId, NodeId, AnnuaireId, CorpusId, DocId )
 import Gargantext.Database.Prelude (HasConfig(..))
 import Gargantext.Prelude
 import Gargantext.Prelude.Config (gc_max_docs_scrapers)
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
 import Network.Wai (requestHeaders)
 import Servant
-import Servant.Auth as SA
+import Servant.Auth as SA ( Auth, Cookie, JWT )
 import Servant.Auth.Swagger ()
-import Servant.Ekg
-import Servant.Server.Internal.Delayed
-import Servant.Server.Internal.DelayedIO
-import Servant.Swagger
-import Servant.Swagger.UI
-import qualified Data.List as L
+import Servant.Ekg ( HasEndpoint(..) )
+import Servant.Server.Internal.Delayed ( addHeaderCheck )
+import Servant.Server.Internal.DelayedIO ( withRequest, DelayedIO )
+import Servant.Swagger ( HasSwagger(..) )
+import Servant.Swagger.UI ( SwaggerSchemaUI )
 
 data WithCustomErrorScheme a