Setup self-contained CoreNLP for CI

.gitlab-ci.yml

 # Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
-image: adinapoli/gargantext:v2.1
+image: adinapoli/gargantext:v2.2
 
 variables:
   STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
@@ -78,16 +78,16 @@ test:
       chown -R test:test /root/.cache/cabal/logs/
       chown -R test:test /root/.cache/cabal/packages/hackage.haskell.org/
 
-      #Spawn the CoreNLP
-      apt-get install docker
-      docker run -p 127.0.0.1:9000:9000/tcp -dt -v /var/run/docker.sock:/var/run/docker.sock cgenie/corenlp-garg:4.5.4
+      cd /builds/gargantext/haskell-gargantext/devops/coreNLP; ./build.sh
+      cd /builds/gargantext/haskell-gargantext
 
-      nix-shell --run "chown -R test:test /root/.config/ && su -m test -c \"export PATH=$PATH:$TEST_NIX_PATH && cd /builds/gargantext/haskell-gargantext && $CABAL --store-dir=$CABAL_STORE_DIR v2-test --test-show-details=streaming --flags test-crypto --ghc-options='-O0 -fclear-plugins'\""
+      nix-shell --run "chown -R test:test /root/.config/ && su -m test -c \"export PATH=$PATH:$TEST_NIX_PATH && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$CABAL_STORE_DIR v2-test --test-show-details=streaming --flags test-crypto --ghc-options='-O0 -fclear-plugins'\""
       chown -R root:root dist-newstyle/
       chown -R root:root /root/
       chown -R root:root $CABAL_STORE_DIR
       chown -R root:root /root/.cache/cabal/logs/
       chown -R root:root /root/.cache/cabal/packages/hackage.haskell.org/
+      chown -Rh root:root /builds/gargantext/haskell-gargantext/devops/coreNLP/stanford-corenlp-current
 
 #docs:
 #  stage: docs

devops/docker/Dockerfile

@@ -4,6 +4,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ARG GHC=8.10.7
 ARG STACK=2.7.3
 ARG CABAL=3.10.1.0
+ARG CORE
 COPY ./shell.nix                                       /builds/gargantext/shell.nix
 COPY ./nix/pkgs.nix                                    /builds/gargantext/nix/pkgs.nix
 COPY ./nix/pinned-22.05.nix                            /builds/gargantext/nix/pinned-22.05.nix
@@ -14,6 +15,8 @@ COPY ./nix/overlays/Cabal-syntax-3.10.1.0.nix          /builds/gargantext/nix/ov
 COPY ./nix/overlays/directory-1.3.7.0.nix              /builds/gargantext/nix/overlays/directory-1.3.7.0.nix
 COPY ./nix/overlays/hackage-security-0.6.2.3.nix       /builds/gargantext/nix/overlays/hackage-security-0.6.2.3.nix
 COPY ./nix/overlays/process-1.6.15.0.nix               /builds/gargantext/nix/overlays/process-1.6.15.0.nix
+COPY ./devops/coreNLP/build.sh                         /builds/gargantext/devops/coreNLP/build.sh
+COPY ./devops/coreNLP/startServer.sh                   /builds/gargantext/devops/coreNLP/startServer.sh
 
 ENV TZ=Europe/Rome
 RUN apt-get update && \
@@ -43,7 +46,9 @@ RUN apt-get update && \
       wget \
       vim \
       xz-utils \
-      zlib1g-dev && \
+      zlib1g-dev \
+      openjdk-18-jdk \
+      unzip && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     mkdir -m 0755 /nix && groupadd -r nixbld && chown root /nix && \
     for n in $(seq 1 10); do useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(command -v nologin)" "nixbld$n"; done
@@ -52,6 +57,8 @@ RUN gpg --batch --keyserver keys.openpgp.org     --recv-keys 7D1E8AFD1D4A16D71FA
     gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+RUN cd /builds/gargantext/devops/coreNLP; ./build.sh; rm -rf *.zip
+
 RUN set -o pipefail && \
     bash <(curl -L https://releases.nixos.org/nix/nix-2.15.0/install) --no-daemon && \
     locale-gen en_US.UTF-8 && chown root -R /nix

gargantext.cabal

@@ -960,6 +960,7 @@ test-suite garg-test-tasty
     , patches-map ^>= 0.1.0.1
     , postgres-options >= 0.2 && < 0.3
     , postgresql-simple >= 0.6.4 && < 0.7
+    , process ^>= 1.6.13.2
     , quickcheck-instances ^>= 0.3.25.2
     , raw-strings-qq
     , recover-rtti >= 0.4 && < 0.5
@@ -1043,6 +1044,7 @@ test-suite garg-test-hspec
     , patches-map ^>= 0.1.0.1
     , postgres-options >= 0.2 && < 0.3
     , postgresql-simple >= 0.6.4 && < 0.7
+    , process ^>= 1.6.13.2
     , quickcheck-instances ^>= 0.3.25.2
     , raw-strings-qq
     , recover-rtti >= 0.4 && < 0.5

test/Database/Operations/Types.hs

@@ -4,6 +4,7 @@
 
 module Database.Operations.Types where
 
+import Control.Concurrent.Async
 import Control.Exception
 import Control.Lens
 import Control.Monad.Except
@@ -36,9 +37,9 @@ nextCounter :: Counter -> IO Int
 nextCounter (Counter ref) = atomicModifyIORef' ref (\old -> (succ old, old))
 
 data TestEnv = TestEnv {
-    test_db          :: !DBHandle
-  , test_config      :: !GargConfig
-  , test_usernameGen :: !Counter
+    test_db                  :: !DBHandle
+  , test_config              :: !GargConfig
+  , test_usernameGen         :: !Counter
   }
 
 newtype TestMonad a = TestMonad { runTestMonad :: ReaderT TestEnv IO a }

test/hspec/Main.hs

@@ -3,10 +3,29 @@ module Main where
 
 import Gargantext.Prelude
 
+import Control.Exception
+import Shelly hiding (FilePath)
+import System.Process
+import System.IO
 import qualified Database.Operations     as DB
 
 import Test.Hspec
 
+startCoreNLPServer :: IO ProcessHandle
+startCoreNLPServer = do
+  devNull <- openFile "/dev/null" WriteMode
+  let p = proc "./startServer.sh" []
+  (_, _, _, hdl) <- createProcess $ p { cwd = Just "devops/coreNLP/stanford-corenlp-current"
+                    , delegate_ctlc = True
+                    , create_group = True
+                    , std_out = UseHandle devNull
+                    , std_err = UseHandle devNull
+                    }
+  pure hdl
+
+stopCoreNLPServer :: ProcessHandle -> IO ()
+stopCoreNLPServer = interruptProcessGroupOf
+
 -- It's especially important to use Hspec for DB tests, because,
 -- unlike 'tasty', 'Hspec' has explicit control over parallelism,
 -- and it's important that DB tests are run according to a very
@@ -14,5 +33,11 @@ import Test.Hspec
 -- Unfortunately it's not possibly to use the 'tasty-hspec' adapter
 -- because by the time we get a 'TestTree' out of the adapter library,
 -- the information about parallelism is lost.
+--
+-- /IMPORTANT/: For these tests to run correctly, you have to run
+-- ./devops/coreNLP/build.sh first. You have to run it only /once/,
+-- and then you are good to go for the time being.
 main :: IO ()
-main = hspec DB.tests
+main = do
+  hSetBuffering stdout NoBuffering
+  bracket startCoreNLPServer stopCoreNLPServer (const (hspec DB.tests))

test/tasty/Main.hs

@@ -14,7 +14,6 @@ import Gargantext.Prelude
 
 import qualified Core.Text.Corpus.Query  as CorpusQuery
 import qualified Core.Utils              as Utils
-import qualified Database.Operations     as DB
 import qualified Graph.Clustering        as Graph
 import qualified Ngrams.NLP              as NLP
 import qualified Ngrams.Query            as NgramsQuery