Merge branch '483-dev-corenlp-in-nix-flake' into 322-dev-flake-nix-with-flake-in-corenlp

.gitlab-ci.yml

@@ -73,6 +73,12 @@ test:
     - chown -R test:test ${CABAL_DIR}
     - mkdir -p "$XDG_CACHE_HOME/nix"
     - chown -R test:test "$XDG_CACHE_HOME/nix"
+    # Fix
+    # opening lock file '/nix/var/nix/db/big-lock': Permission denied
+    # when starting CoreNLP
+    - chmod -R 755 /nix/var/nix/db/
+    - chmod 666 /nix/var/nix/db/big-lock
+
 
   cache:
     key: cabal.project

README.md

@@ -117,15 +117,17 @@ And provide a name and a password for the master user as requested.
 CoreNLP can be started with nix:
 
 ```shell
-nix-shell --run startCoreNLPServer.sh
+nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"#
 ```
 
 By default, this starts on port `9000`. If you want a different port, then do:
 
 ```shell
-nix-shell --run 'startCoreNLPServer.sh -p 9001'
+nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"# -- -p 9001
 ```
 
+See https://gitlab.iscpif.fr/gargantext/corenlp-nix for more info.
+
 ### Running Gargantext
 
 From inside the `haskell-gargantext/` directory, run

bin/gargantext-cli/CLI/Ini.hs

@@ -61,7 +61,11 @@ iniParser = fmap CCMD_ini $ IniArgs <$>
   (optional . strOption $ long "toml-path" <> help "Path to the output .toml file") <*>
   (flag False True (long "dry-run" <> help "If set, will only output generated .toml file to stdout"))
 
-convertConfigs :: Ini.GargConfig -> IniMail.MailConfig -> IniNLP.NLPConfig -> PGS.ConnectInfo -> Config.GargConfig
+convertConfigs :: Ini.GargConfig
+               -> IniMail.MailConfig
+               -> IniNLP.NLPConfig
+               -> PGS.ConnectInfo
+               -> Config.GargConfig
 convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo =
   Config.GargConfig { _gc_secrets = CTypes.SecretsConfig { _s_master_user = _gc_masteruser
                                                          , _s_secret_key = _gc_secretkey

devops/docker/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update && \
       apt-transport-https \
       ca-certificates \
       curl \
+      fontconfig \
       git \
       gnupg2 \
       locales \
@@ -35,9 +36,12 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN set -o pipefail && \
     locale-gen en_US.UTF-8 && \
-    bash <(curl -L https://releases.nixos.org/nix/nix-2.26.2/install) --no-daemon && \
+    bash <(curl -L https://releases.nixos.org/nix/nix-2.29.1/install) --no-daemon && \
     chown root -R /nix && \
     . "$HOME/.nix-profile/etc/profile.d/nix.sh" && \
+    mkdir -vp "$HOME/.config/nix" && echo "experimental-features = nix-command flakes" >> $HOME/.config/nix/nix.conf && \
+    # prefetch corenlp
+    nix build 'git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git' && \
     mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext" && \
     echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \
     echo `which nix-env` && \

nix/corenlp.nix

-# https://nixos.wiki/wiki/Java
-
-{ fetchzip,
-  makeWrapper,
-  stdenv,
-  writeShellScript,
-  jre,
-
-  version ? "4.5.9",
-  hash ? "sha256-DOGBkGJfvR1PoXz2CNoo58HXwGLxvPKMChRqlrFtQLQ=", 
-}:
-
-stdenv.mkDerivation (finalAttrs:
-  let
-    startServer = writeShellScript "startCoreNLPServer.sh" ''
-      set -x
-      PORT=9000
-  
-      while getopts ':p:h' opt; do
-        case $opt in
-          (p) PORT=$OPTARG;;
-          (h) echo "$(basename $0) [-p 9000]"
-              exit 0
-              ;;
-        esac
-      done
-  
-      shift "$((OPTIND - 1))"
-
-      ${jre}/bin/java -mx4g -cp "$CORENLP_PATH/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port $PORT -timeout 15000 "$@"
-    '';
-    # see https://huggingface.co/stanfordnlp/CoreNLP/commits/main
-    versionCommits = {
-      "4.5.8" = "34264e88b7add9e0045f4727bc7d1872385f06aa";
-      "4.5.9" = "06f79ee8b1ec475d7630b1871bfd75a57c77ffa4";
-    };
-    commit = versionCommits."${finalAttrs.version}";
-  in
-
-  {
-    name = "corenlp";
-    inherit version;
-    src = fetchzip {
-      inherit hash;
-      # url = "http://nlp.stanford.edu/software/stanford-corenlp-${finalAttrs.version}.zip";
-      # huggin face is more stable
-      url = "https://huggingface.co/stanfordnlp/CoreNLP/resolve/${commit}/stanford-corenlp-latest.zip";
-    };
-    buildInputs = [
-      jre
-    ];
-    nativeBuildInputs = [
-      makeWrapper
-    ];
-    phases = [ "unpackPhase" "installPhase" ];
-    installPhase = ''
-      runHook preInstall
-      mkdir -p $out/bin
-      mkdir -p $out/share/corenlp
-      cp -r . $out/share/corenlp
-      makeWrapper ${startServer} $out/bin/startCoreNLPServer.sh \
-        --set CORENLP_PATH "$out/share/corenlp"
-      runHook postInstall
-    '';
-  }
-)

nix/pkgs.nix

-{ pkgs ? import (if builtins.elem builtins.currentSystem ["x86_64-darwin" "aarch64-darwin"]
-                 then ./pinned-25.05.darwin.nix
-                 else ./pinned-25.05.nix) {} }:
+{ pkgs ? import
+    (if builtins.elem builtins.currentSystem [ "x86_64-darwin" "aarch64-darwin" ]
+    then ./pinned-25.05.darwin.nix
+    else ./pinned-25.05.nix)
+    { }
+}:
 
 rec {
   inherit pkgs;
+
+  # Create a custom stdenv with GCC 14 on Linux. This is because
+  # pkgs.stdenv is GCC version 13 and when using 'nix' inside the
+  # shell, one gets strange 'CXX_ABI' errors.
+  customStdenv =
+    if pkgs.stdenv.isLinux then
+      pkgs.gcc14Stdenv
+    else
+      pkgs.stdenv; # Keep default on other platforms (clang on macOS)
+
   ghc966 = pkgs.haskell.compiler.ghc966;
   cabal_install = pkgs.haskell.lib.compose.justStaticExecutables pkgs.haskell.packages.ghc966.cabal-install;
-  
-  graphviz = pkgs.callPackage ./graphviz.nix {};
-  igraph_0_10_4 = pkgs.callPackage ./igraph.nix {};
-  corenlp = pkgs.callPackage ./corenlp.nix { };  # 4.5.8
+
+  graphviz = pkgs.callPackage ./graphviz.nix { };
+  igraph_0_10_4 = pkgs.callPackage ./igraph.nix { };
   cabal2stack = pkgs.callPackage ./cabal2stack.nix { ghc = ghc966; };
   nng_notls = pkgs.nng.overrideAttrs (old: {
-    cmakeFlags = (old.cmakeFlags or []) ++ [ "-DNNG_ENABLE_TLS=OFF" ];
+    cmakeFlags = (old.cmakeFlags or [ ]) ++ [ "-DNNG_ENABLE_TLS=OFF" ];
   });
-  
+
   hsBuildInputs = [
     ghc966
     cabal_install
@@ -28,7 +40,6 @@ rec {
     blas
     bzip2
     cabal2stack
-    corenlp
     curl
     czmq
     docker-compose
@@ -47,27 +58,26 @@ rec {
     libpqxx
     libsodium
     nng_notls
-    nil  # nix language server
+    nil # nix language server
+    openblas
     pcre
     pkg-config
     postgresql
-    stdenv.cc.cc
+    customStdenv.cc.cc
     xz
     zlib
     zeromq
-    curl
-  ] ++ ( lib.optionals stdenv.isDarwin [
-       darwin.apple_sdk.frameworks.Accelerate
-       ]);
+  ] ++ (lib.optionals stdenv.isDarwin [
+    darwin.apple_sdk.frameworks.Accelerate
+  ]);
   libPaths = pkgs.lib.makeLibraryPath nonhsBuildInputs;
   shellHook = ''
     export LD_LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}:$LD_LIBRARY_PATH"
     export LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}"
-    export PATH="${pkgs.gccStdenv}/bin:$PATH"
-    export NIX_CC="${pkgs.gccStdenv}"
-    export CC="${pkgs.gccStdenv}/bin/gcc"
+    export CXX_PATH="${customStdenv.cc.cc.lib}"
+    export LIB_PATHS="${libPaths}"
   '';
-  shell = pkgs.mkShell.override { stdenv = pkgs.gccStdenv; } {
+  shell = pkgs.mkShell.override { stdenv = customStdenv; } {
     name = "gargantext-shell";
     buildInputs = hsBuildInputs ++ nonhsBuildInputs;
     inherit shellHook;

test/Main.hs

+{-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 
 module Main where
 
 import Control.Monad
-import Data.Text (isInfixOf)
 import Data.Text qualified as T
 import Gargantext.Prelude hiding (isInfixOf)
 import System.IO
+import System.IO.Error (userError)
 import System.Posix.Process
 import System.Posix.Signals
 import System.Process
@@ -39,12 +40,13 @@ import Test.Utils.Crypto               qualified as Crypto
 import Test.Utils.Jobs                 qualified as Jobs
 
 
-startCoreNLPServer :: IO ProcessHandle
+startCoreNLPServer :: IO (Handle, Handle, ProcessHandle)
 startCoreNLPServer = do
   putText "calling start core nlp"
-  devNull <- openFile "/dev/null" WriteMode
-  let p = proc "startCoreNLPServer.sh" []
-  (_, _, _, hdl) <- (createProcess $ p { cwd = Nothing
+  -- devNull <- openFile "/dev/null" WriteMode
+  let p = proc "nix" [ "run"
+                     , "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git" ]
+  (_, m_stdout_hdl, m_stderr_hdl, hdl) <- (createProcess $ p { cwd = Nothing
                     -- NOTE(adn) Issue #451, this one has to stay disabled, because if we
                     -- turn it on, despite the confusing documentation on the `process` library
                     -- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
@@ -53,16 +55,23 @@ startCoreNLPServer = do
                     -- to Ctrl^C requests.
                     , delegate_ctlc = False
                     , create_group = True
-                    , std_out = UseHandle devNull
-                    , std_err = UseHandle devNull
-                    }) `catch` \e -> case e of
-                                           _ | True <- "does not exist" `isInfixOf` (T.pack . show @SomeException $ e)
-                                             -> fail $ "Cannot execute the 'startCoreNLPServer.sh' script. Make sure you are in a nix environment."
-                                             | otherwise -> throwIO e
-  pure hdl
+                    , std_out = CreatePipe
+                    , std_err = CreatePipe
+                    -- , std_out = UseHandle devNull
+                    -- , std_err = UseHandle devNull
+                    }) `catch` \(e :: SomeException) ->
+                                 case e of
+                                   _ | True <- "does not exist" `T.isInfixOf` (T.pack . show @SomeException $ e)
+                                     -> fail $ "Cannot execute the 'corenlp' via nix flakes. Make sure you have 'nix' installed."
+                                     | otherwise -> throwIO e
 
-killProcessTree :: ProcessHandle -> IO ()
-killProcessTree ph = do
+  let stdout_hdl = fromMaybe (errorTrace "Can't get stdout handle!") m_stdout_hdl
+  let stderr_hdl = fromMaybe (errorTrace "Can't get stderr handle!") m_stderr_hdl
+                                     
+  pure (stdout_hdl, stderr_hdl, hdl)
+
+killProcessTree :: (Handle, Handle, ProcessHandle) -> IO ()
+killProcessTree (_, _, ph) = do
   pid <- getPid ph
   case pid of
     Nothing -> putText "Process already terminated"
@@ -70,6 +79,23 @@ killProcessTree ph = do
       pgid <- getProcessGroupIDOf p
       signalProcessGroup keyboardSignal pgid
 
+
+-- NOTE(seeg) createProcess is nonblocking, and so its exception
+-- handling is incomplete:
+--   λ> createProcess $ proc "l" []
+--   *** Exception: l: createProcess: posix_spawnp: does not exist (No such file or directory)
+--   λ> createProcess $ proc "ls" ["haha"]
+--   ls: cannot access 'haha': No such file or directory
+-- Credit for this construct goes to @adn
+waitOrDie :: (Handle, Handle, ProcessHandle) -> IO ()
+waitOrDie (stdout_h, stderr_h, h) = do
+  ec <- waitForProcess h
+  when (ec /= ExitSuccess) $ do
+    out <- hGetContents stdout_h
+    err <- hGetContents stderr_h
+    throwIO $ userError ("Async process quit!\nStdout:\n" <> out <> "\nStderr:\n" <> err)
+
+
 -- It's especially important to use Hspec for DB tests, because,
 -- unlike 'tasty', 'Hspec' has explicit control over parallelism,
 -- and it's important that DB tests are run according to a very
@@ -77,14 +103,10 @@ killProcessTree ph = do
 -- Unfortunately it's not possibly to use the 'tasty-hspec' adapter
 -- because by the time we get a 'TestTree' out of the adapter library,
 -- the information about parallelism is lost.
---
--- /IMPORTANT/: For these tests to run correctly, you have to run
--- ./devops/coreNLP/build.sh first. You have to run it only /once/,
--- and then you are good to go for the time being.
 main :: IO ()
 main = do
   hSetBuffering stdout NoBuffering
-  bracket startCoreNLPServer killProcessTree (const run_tests)
+  bracket startCoreNLPServer killProcessTree (\h -> race (waitOrDie h) run_tests *> pure())
   where
     run_tests = hspec $ sequential $ do
       API.tests