Merge branch '483-dev-corenlp-in-nix-flake-squashed' into 'dev'

Resolve "Move corenlp to separate repo/flake"

Closes #483

See merge request !434

.gitlab-ci.yml

 # Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
-image: cgenie/gargantext:9.6.6
+image: cgenie/gargantext:9.6.6-with-test-user
 
 variables:
   STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
@@ -17,32 +17,36 @@ stages:
   - bench
   - test
 
-stack:
-  stage: stack
+cabal:
+  stage: cabal
   cache:
-    key: stack.yaml
+    key: cabal.project
     paths:
-      - .stack-root/
-      - .stack-work/
+      - dist-newstyle/
+      - .cabal/
+    policy: pull-push
+  before_script:
+    - sudo chown -R test /builds/gargantext
   script:
-    - echo "Building the project from '$CI_PROJECT_DIR'"
     - git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
     - git config --global --add safe.directory '*'
-    - nix-shell --run "stack build --no-terminal --fast --dry-run"
+    - nix-shell --run "cabal update && ./bin/update-project-dependencies && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
   allow_failure: false
 
-cabal:
-  stage: cabal
+stack:
+  stage: stack
   cache:
-    key: cabal.project
+    key: stack.yaml
     paths:
-      - dist-newstyle/
-      - .cabal/
-    policy: pull-push
+      - .stack-root/
+      - .stack-work/
+  before_script:
+    - sudo chown -R test /builds/gargantext
   script:
+    - echo "Building the project from '$CI_PROJECT_DIR'"
     - git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
     - git config --global --add safe.directory '*'
-    - nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
+    - nix-shell --run "stack build --no-terminal --fast --dry-run"
   allow_failure: false
 
 bench:
@@ -54,6 +58,8 @@ bench:
       - dist-newstyle/
       - .cabal/
     policy: pull-push
+  before_script:
+    - sudo chown -R test /builds/gargantext
   script:
     - nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-bench --flags +no-phylo-debug-logs --ghc-options='-O2 -fclear-plugins'"
   allow_failure: true
@@ -63,16 +69,7 @@ test:
   # The tests needs to run as the 'test' user, because they leverage the
   # initdb utility from postgres that cannot be run by 'root'.
   before_script:
-    - echo "Creating test user..."
-    - mkdir -p /home/test
-    - mkdir -p /root/.config
-    - useradd -U test
-    - chown -R test:test dist-newstyle/
-    - chown -R test:test /root/
-    - chown -R test:test $STORE_DIR
-    - chown -R test:test ${CABAL_DIR}
-    - mkdir -p "$XDG_CACHE_HOME/nix"
-    - chown -R test:test "$XDG_CACHE_HOME/nix"
+    - sudo chown -R test /builds/gargantext
 
   cache:
     key: cabal.project
@@ -91,13 +88,13 @@ test:
       export TEST_NIX_PATH=$(nix-shell --run "echo -n \$PATH")
       echo "Found test nix path at ${TEST_NIX_PATH}"
       nix-shell --run "./bin/update-project-dependencies $STORE_DIR"
-      mkdir -p /root/.cache/cabal/logs
-      chown -R test:test /root/.cache/cabal/logs/
-      chown -R test:test "$TEST_TMPDIR"
+      #mkdir -p /root/.cache/cabal/logs
+      #chown -R test:test /root/.cache/cabal/logs/
+      #chown -R test:test "$TEST_TMPDIR"
 
-      nix-shell --run "chown -R test:test /root/.config/ && su -m test -c \"export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'\""
+      nix-shell --run "export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --verbose=2 --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
 
-      chown -R root:root dist-newstyle/
-      chown -R root:root /root/
-      chown -R root:root $STORE_DIR
-      chown -R root:root /root/.cache/cabal/logs/
+      sudo chown -R root:root dist-newstyle/
+      #chown -R root:root /root/
+      sudo chown -R root:root $STORE_DIR
+      #chown -R root:root /root/.cache/cabal/logs/

README.md

@@ -117,15 +117,17 @@ And provide a name and a password for the master user as requested.
 CoreNLP can be started with nix:
 
 ```shell
-nix-shell --run startCoreNLPServer.sh
+nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"#
 ```
 
 By default, this starts on port `9000`. If you want a different port, then do:
 
 ```shell
-nix-shell --run 'startCoreNLPServer.sh -p 9001'
+nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"# -- -p 9001
 ```
 
+See https://gitlab.iscpif.fr/gargantext/corenlp-nix for more info.
+
 ### Running Gargantext
 
 From inside the `haskell-gargantext/` directory, run

bin/gargantext-cli/CLI/Ini.hs

@@ -58,7 +58,11 @@ iniParser = fmap CCMD_ini $ IniArgs <$>
   (optional . strOption $ long "toml-path" <> help "Path to the output .toml file") <*>
   (flag False True (long "dry-run" <> help "If set, will only output generated .toml file to stdout"))
 
-convertConfigs :: Ini.GargConfig -> IniMail.MailConfig -> IniNLP.NLPConfig -> PGS.ConnectInfo -> Config.GargConfig
+convertConfigs :: Ini.GargConfig
+               -> IniMail.MailConfig
+               -> IniNLP.NLPConfig
+               -> PGS.ConnectInfo
+               -> Config.GargConfig
 convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo =
   Config.GargConfig { _gc_secrets = CTypes.SecretsConfig { _s_master_user = _gc_masteruser
                                                          , _s_secret_key = _gc_secretkey

bin/update-project-dependencies

@@ -17,7 +17,7 @@ fi
 # `cabal.project.freeze`. This ensures the files stay deterministic so that CI
 # cache can kick in.
 expected_cabal_project_hash="eb8fdb1a14aa2f7a13f565cf7fa9f6ab0e2dab9212538aed0db5691015be286b"
-expected_cabal_project_freeze_hash="553b98aadb35506a305bd740cdd71f5fadc1e6d55d10f91cf39daa6735a63d78"
+expected_cabal_project_freeze_hash="a5eb1d9a331266fef56f490712decbd3eaff1fd0daa8bc63f893238a7f47df93"
 
 
 cabal --store-dir=$STORE_DIR v2-build --dry-run

cabal.project.freeze

@@ -236,7 +236,7 @@ constraints: any.Boolean ==0.2.4,
              hedis -dev,
              any.hlcm ==0.2.2,
              any.hmatrix ==0.20.2,
-             hmatrix -disable-default-paths -no-random_r -openblas,
+             hmatrix -disable-default-paths -no-random_r +openblas,
              any.hmatrix-gsl-stats ==0.4.1.8,
              hmatrix-gsl-stats -onlygsl,
              any.hourglass ==0.2.12,

devops/docker/Dockerfile

 FROM ubuntu:noble
 
-## NOTA BENE: In order for this to be built successfully, you have to run ./devops/coreNLP/build.sh first.
-
 ARG DEBIAN_FRONTEND=noninteractive
 
 ENV TZ=Europe/Rome
 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV USER=root
-ENV SHELL /bin/bash
-ENV PATH=/root/.nix-profile/bin:$PATH
-ENV PATH=/root/.local/bin:$PATH
-
 
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
       apt-transport-https \
       ca-certificates \
       curl \
+      fontconfig \
       git \
       gnupg2 \
       locales \
@@ -26,19 +20,32 @@ RUN apt-get update && \
       #zlib1g-dev \
       unzip && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    mkdir -m 0755 /nix && groupadd -r nixbld && chown root /nix && \
-    for n in $(seq 1 10); do useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(command -v nologin)" "nixbld$n"; done && \
     gpg --batch --keyserver keys.openpgp.org     --recv-keys 7D1E8AFD1D4A16D71FADA2F2CCC85C0E40C06A8C && \
     gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01
 
+RUN locale-gen en_US.UTF-8
+
+RUN useradd -m test
+ENV USER=test
+ENV SHELL=/bin/bash
+ENV PATH=/home/test/.nix-profile/bin:$PATH
+ENV PATH=/home/test/.local/bin:$PATH
+RUN mkdir -m 0777 /nix && chown test /nix && \
+    mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext"
+RUN echo 'test ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER test
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-RUN set -o pipefail && \
-    locale-gen en_US.UTF-8 && \
-    bash <(curl -L https://releases.nixos.org/nix/nix-2.26.2/install) --no-daemon && \
-    chown root -R /nix && \
-    . "$HOME/.nix-profile/etc/profile.d/nix.sh" && \
-    mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext" && \
+RUN bash <(curl --proto '=https' --tlsv1.2 -L https://releases.nixos.org/nix/nix-2.30.2/install) --no-daemon && \
+    #chown root -R /nix && \
+    mkdir -vp "$HOME/.config/nix" && \
+    echo "experimental-features = nix-command flakes" >> $HOME/.config/nix/nix.conf
+
+WORKDIR /nix-ci-build
+
+# prefetch corenlp
+RUN nix build 'git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git' && \
     echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \
     echo `which nix-env` && \
     . $HOME/.bashrc && nix-env --version
@@ -46,8 +53,7 @@ RUN set -o pipefail && \
 # We want to cache nix artifacts in the Dockerfile to improve CI speed
 COPY ./shell.nix /nix-ci-build/
 COPY ./nix       /nix-ci-build/
-RUN set -o pipefail && \
-    pushd /nix-ci-build/ && nix-build shell.nix && popd
+RUN nix-build shell.nix
 
 WORKDIR "/builds/gargantext/"
 

nix/corenlp.nix

-# https://nixos.wiki/wiki/Java
-
-{ fetchzip,
-  makeWrapper,
-  stdenv,
-  writeShellScript,
-  jre,
-
-  version ? "4.5.9",
-  hash ? "sha256-DOGBkGJfvR1PoXz2CNoo58HXwGLxvPKMChRqlrFtQLQ=", 
-}:
-
-stdenv.mkDerivation (finalAttrs:
-  let
-    startServer = writeShellScript "startCoreNLPServer.sh" ''
-      set -x
-      PORT=9000
-  
-      while getopts ':p:h' opt; do
-        case $opt in
-          (p) PORT=$OPTARG;;
-          (h) echo "$(basename $0) [-p 9000]"
-              exit 0
-              ;;
-        esac
-      done
-  
-      shift "$((OPTIND - 1))"
-
-      ${jre}/bin/java -mx4g -cp "$CORENLP_PATH/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port $PORT -timeout 15000 "$@"
-    '';
-    # see https://huggingface.co/stanfordnlp/CoreNLP/commits/main
-    versionCommits = {
-      "4.5.8" = "34264e88b7add9e0045f4727bc7d1872385f06aa";
-      "4.5.9" = "06f79ee8b1ec475d7630b1871bfd75a57c77ffa4";
-    };
-    commit = versionCommits."${finalAttrs.version}";
-  in
-
-  {
-    name = "corenlp";
-    inherit version;
-    src = fetchzip {
-      inherit hash;
-      # url = "http://nlp.stanford.edu/software/stanford-corenlp-${finalAttrs.version}.zip";
-      # huggin face is more stable
-      url = "https://huggingface.co/stanfordnlp/CoreNLP/resolve/${commit}/stanford-corenlp-latest.zip";
-    };
-    buildInputs = [
-      jre
-    ];
-    nativeBuildInputs = [
-      makeWrapper
-    ];
-    phases = [ "unpackPhase" "installPhase" ];
-    installPhase = ''
-      runHook preInstall
-      mkdir -p $out/bin
-      mkdir -p $out/share/corenlp
-      cp -r . $out/share/corenlp
-      makeWrapper ${startServer} $out/bin/startCoreNLPServer.sh \
-        --set CORENLP_PATH "$out/share/corenlp"
-      runHook postInstall
-    '';
-  }
-)

nix/pkgs.nix

@@ -8,13 +8,22 @@
 
 rec {
   inherit pkgs;
+
+  # Create a custom stdenv with GCC 14 on Linux. This is because
+  # pkgs.stdenv is GCC version 13 and when using 'nix' inside the
+  # shell, one gets strange 'CXX_ABI' errors.
+  customStdenv =
+    if pkgs.stdenv.isLinux then
+      pkgs.gcc14Stdenv
+    else
+      pkgs.stdenv; # Keep default on other platforms (clang on macOS)
+
   ghcVersion = "ghc966";
   gargGhc = pkgs.haskell.compiler.${ghcVersion};
   cabal_install = pkgs.haskell.lib.compose.justStaticExecutables pkgs.haskell.packages.${ghcVersion}.cabal-install;
 
   graphviz = pkgs.callPackage ./graphviz.nix { };
   igraph_0_10_4 = pkgs.callPackage ./igraph.nix { };
-  corenlp = pkgs.callPackage ./corenlp.nix { }; # 4.5.9
   cabal2stack = pkgs.callPackage ./cabal2stack.nix { ghc = gargGhc; };
   nng_notls = pkgs.nng.overrideAttrs (old: {
     cmakeFlags = (old.cmakeFlags or [ ]) ++ [ "-DNNG_ENABLE_TLS=OFF" ];
@@ -32,7 +41,6 @@ rec {
     blas
     bzip2
     cabal2stack
-    corenlp
     curl
     czmq
     docker-compose
@@ -52,10 +60,11 @@ rec {
     libsodium
     nng_notls
     nil # nix language server
+    openblas
     pcre
     pkg-config
     postgresql
-    stdenv.cc.cc
+    customStdenv.cc.cc
     xz
     zlib
     zeromq
@@ -67,11 +76,10 @@ rec {
   shellHook = ''
     export LD_LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}:$LD_LIBRARY_PATH"
     export LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}"
-    export PATH="${pkgs.gccStdenv}/bin:$PATH"
-    export NIX_CC="${pkgs.gccStdenv}"
-    export CC="${pkgs.gccStdenv}/bin/gcc"
+    export CXX_PATH="${customStdenv.cc.cc.lib}"
+    export LIB_PATHS="${libPaths}"
   '';
-  shell = pkgs.mkShell.override { stdenv = pkgs.gccStdenv; } {
+  shell = pkgs.mkShell.override { stdenv = customStdenv; } {
     name = "gargantext-shell";
     buildInputs = hsBuildInputs ++ nonhsBuildInputs;
     inherit shellHook;

stack.yaml

@@ -387,7 +387,7 @@ flags:
   hmatrix:
     "disable-default-paths": false
     "no-random_r": false
-    openblas: false
+    openblas: true
   "hmatrix-gsl-stats":
     onlygsl: false
   hsc2hs:

test/Main.hs

+{-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 
 module Main where
 
 import Control.Monad ( MonadFail(fail) )
-import Data.Text (isInfixOf)
 import Data.Text qualified as T
-import Gargantext.Prelude hiding (isInfixOf)
-import System.IO ( BufferMode(NoBuffering), hSetBuffering )
+import Gargantext.Prelude
+import System.IO ( BufferMode(NoBuffering), hGetContents, hSetBuffering )
+import System.IO.Error (userError)
+import System.Posix.Process
+import System.Posix.Signals
 import System.Process
-import System.Posix.Process ( getProcessGroupIDOf )
-import System.Posix.Signals ( keyboardSignal, signalProcessGroup )
+-- import System.Posix.Process ( getProcessGroupIDOf )
+-- import System.Posix.Signals ( keyboardSignal, signalProcessGroup )
 import Test.API qualified as API
 import Test.Database.Operations qualified as DB
 import Test.Database.Transactions qualified as DBT
@@ -39,41 +42,35 @@ import Test.Utils.Crypto               qualified as Crypto
 import Test.Utils.Jobs                 qualified as Jobs
 
 
-startCoreNLPServer :: IO (Maybe ProcessHandle)
+startCoreNLPServer :: IO (Handle, Handle, ProcessHandle)
 startCoreNLPServer = do
   putText "calling start core nlp"
-  devNull <- openFile "/dev/null" WriteMode
-  let p = proc "startCoreNLPServer.sh" []
-  let connect = do
-        (_, _, _, hdl) <- createProcess p { cwd = Nothing
-                                    -- NOTE(adn) Issue #451, this one has to stay disabled, because if we
-                                    -- turn it on, despite the confusing documentation on the `process` library
-                                    -- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
-                                    -- delegate it exclusively to the process here, which means that our CoreNLP
-                                    -- server will shut down correctly, but the test running will stop responding
-                                    -- to Ctrl^C requests.
-                                          , delegate_ctlc = False
-                                          , create_group = True
-                                          , std_out = UseHandle devNull
-                                          , std_err = UseHandle devNull
-                                          }
-        pure $ Just hdl
+  let p = proc "nix" [ "run"
+                     , "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git" ]
+  (_, m_stdout_hdl, m_stderr_hdl, hdl) <- (createProcess $ p { cwd = Nothing
+                    -- NOTE(adn) Issue #451, this one has to stay disabled, because if we
+                    -- turn it on, despite the confusing documentation on the `process` library
+                    -- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
+                    -- delegate it exclusively to the process here, which means that our CoreNLP
+                    -- server will shut down correctly, but the test running will stop responding
+                    -- to Ctrl^C requests.
+                    , delegate_ctlc = False
+                    , create_group = True
+                    , std_out = CreatePipe
+                    , std_err = CreatePipe
+                    }) `catch` \(e :: SomeException) ->
+                                 case e of
+                                   _ | True <- "does not exist" `T.isInfixOf` (T.pack . show @SomeException $ e)
+                                     -> fail $ "Cannot execute the 'corenlp' via nix flakes. Make sure you have 'nix' installed."
+                                     | otherwise -> throwIO e
 
-  connect `catch` \e -> do
-    putText $ T.pack $ show @SomeException e
-    case e of
-      _ | True <- "does not exist" `isInfixOf` (T.pack . show @SomeException $ e)
-          -> fail $ "Cannot execute the 'startCoreNLPServer.sh' script. Make sure you are in a nix environment."
-        | True <- "Address already in use" `isInfixOf` (T.pack . show @SomeException $ e)
-          -> do
-              putText "Address already in use, but we hope for the best!"
-              pure Nothing
-      
-        | otherwise -> throwIO e
+  let stdout_hdl = fromMaybe (errorTrace "Can't get stdout handle!") m_stdout_hdl
+  let stderr_hdl = fromMaybe (errorTrace "Can't get stderr handle!") m_stderr_hdl
+                                     
+  pure (stdout_hdl, stderr_hdl, hdl)
 
-killProcessTree :: Maybe ProcessHandle -> IO ()
-killProcessTree Nothing = pure ()
-killProcessTree (Just ph) = do
+killProcessTree :: (Handle, Handle, ProcessHandle) -> IO ()
+killProcessTree (_, _, ph) = do
   pid <- getPid ph
   case pid of
     Nothing -> putText "Process already terminated"
@@ -81,6 +78,23 @@ killProcessTree (Just ph) = do
       pgid <- getProcessGroupIDOf p
       signalProcessGroup keyboardSignal pgid
 
+
+-- NOTE(seeg) createProcess is nonblocking, and so its exception
+-- handling is incomplete:
+--   λ> createProcess $ proc "l" []
+--   *** Exception: l: createProcess: posix_spawnp: does not exist (No such file or directory)
+--   λ> createProcess $ proc "ls" ["haha"]
+--   ls: cannot access 'haha': No such file or directory
+-- Credit for this construct goes to @adn
+waitOrDie :: (Handle, Handle, ProcessHandle) -> IO ()
+waitOrDie (stdout_h, stderr_h, h) = do
+  ec <- waitForProcess h
+  when (ec /= ExitSuccess) $ do
+    out <- hGetContents stdout_h
+    err <- hGetContents stderr_h
+    throwIO $ userError ("Async process quit!\nStdout:\n" <> out <> "\nStderr:\n" <> err)
+
+
 -- It's especially important to use Hspec for DB tests, because,
 -- unlike 'tasty', 'Hspec' has explicit control over parallelism,
 -- and it's important that DB tests are run according to a very
@@ -88,14 +102,10 @@ killProcessTree (Just ph) = do
 -- Unfortunately it's not possibly to use the 'tasty-hspec' adapter
 -- because by the time we get a 'TestTree' out of the adapter library,
 -- the information about parallelism is lost.
---
--- /IMPORTANT/: For these tests to run correctly, you have to run
--- ./devops/coreNLP/build.sh first. You have to run it only /once/,
--- and then you are good to go for the time being.
 main :: IO ()
 main = do
   hSetBuffering stdout NoBuffering
-  bracket startCoreNLPServer killProcessTree (const run_tests)
+  bracket startCoreNLPServer killProcessTree (\h -> race (waitOrDie h) run_tests *> pure())
   where
     run_tests = hspec $ sequential $ do
       API.tests