Squashed commit of the following:

commit 4f25aa65
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Sat Aug 2 13:04:35 2025 +0200

    [CI] sudo with chown at the end of tests

commit a4d26226
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Sat Aug 2 06:55:27 2025 +0200

    [CI] no /root/.config needed

commit 1f8eaa0b
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Aug 1 21:27:01 2025 +0200

    [CI] more permission fixes

commit c4d9b175
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Aug 1 20:43:09 2025 +0200

    [CI] more permission fixes for gitlab-ci

commit f85213fd
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Aug 1 19:34:35 2025 +0200

    [CI] add sudo to dockerfile, change dir owner in CI

commit f14c6246
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Aug 1 08:42:32 2025 +0200

    [CI] modify dockerfile to build with test user

commit 1670cc4a
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 31 19:20:10 2025 +0200

    [CI] add verbose=2 for testing to check what is happening with CI timeouts

commit 210ba723
Merge: 1a213aeb 27192b88
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 31 09:22:52 2025 +0200

    Merge branch 'dev' into 483-dev-corenlp-in-nix-flake

commit 1a213aeb
Merge: d7aa1301 61aac410
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Tue Jul 29 18:58:07 2025 +0200

    Merge branch 'dev' into 483-dev-corenlp-in-nix-flake

commit d7aa1301
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Jul 25 09:57:38 2025 +0200

    [CI] one more try to fix nix permission

commit b45e16eb
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Fri Jul 11 07:24:53 2025 +0200

    [CI] writable permissions for nix db

commit 50f13d32
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 16:34:01 2025 +0200

    [CI] attempt at fixing nix db permissions

commit 76c1b9a6
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 14:44:55 2025 +0200

    [CI] fix /nix dir permissions

    Apparently, this is needed to run 'nix' inside tests (e.g. start corenlp)

commit 7edb97af
Merge: 19e752b8 64e2a689
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 11:56:04 2025 +0200

    Merge branch 'dev' into 483-dev-corenlp-in-nix-flake

commit 19e752b8
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 11:47:58 2025 +0200

    [tests] better startup of corenlp

    Take into account the async nature of createProcess. Add proper watch
    so that we monitor that coreNLP is running.

commit bde033f9
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 10:47:36 2025 +0200

    [nix] fix stdenv to be gcc 14, not 13

commit 1d4ceb07
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 10 07:03:15 2025 +0200

    [nix] add openblas as a dependency

commit 9dd4ebc3
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Wed Jul 9 14:44:28 2025 +0200

    [update-project-dependencie]

commit 120e205c
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Wed Jul 9 12:50:38 2025 +0200

    [corenlp] some fixes to Dockerfile

commit 3a2b2079
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 3 19:21:37 2025 +0200

    [dockerfile] add fontconfig, to see if this is what causes errors on CI

    E.g.:
    https://gitlab.iscpif.fr/gargantext/haskell-gargantext/-/jobs/18643

    paginating QuantumComputing corpus works (MapTerms):            Fontconfig error: Cannot load default config file: No such file: (null)
    Fontconfig error: No writable cache directories

commit d9d4a09c
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Thu Jul 3 14:02:30 2025 +0200

    [nix] don't use experimental features for corenlp nix call

    Also, update Dockerfile with patched `nix.conf`

commit 5ccb44bb
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Wed Jul 2 21:05:47 2025 +0200

    [test] fix calling of nix flakes

commit 1adc512e
Author: Przemysław Kaminski <pk@intrepidus.pl>
Date:   Wed Jul 2 19:42:29 2025 +0200

    [nix] move corenlp to separate flake in core-nix repo

    #483
parent 27192b88
Pipeline #7801 passed with stages
in 54 minutes and 57 seconds
# Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/ # Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
image: cgenie/gargantext:9.6.6 image: cgenie/gargantext:9.6.6-with-test-user
variables: variables:
STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root" STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
...@@ -17,32 +17,36 @@ stages: ...@@ -17,32 +17,36 @@ stages:
- bench - bench
- test - test
stack: cabal:
stage: stack stage: cabal
cache: cache:
key: stack.yaml key: cabal.project
paths: paths:
- .stack-root/ - dist-newstyle/
- .stack-work/ - .cabal/
policy: pull-push
before_script:
- sudo chown -R test /builds/gargantext
script: script:
- echo "Building the project from '$CI_PROJECT_DIR'"
- git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache - git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
- git config --global --add safe.directory '*' - git config --global --add safe.directory '*'
- nix-shell --run "stack build --no-terminal --fast --dry-run" - nix-shell --run "cabal update && ./bin/update-project-dependencies && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
allow_failure: false allow_failure: false
cabal: stack:
stage: cabal stage: stack
cache: cache:
key: cabal.project key: stack.yaml
paths: paths:
- dist-newstyle/ - .stack-root/
- .cabal/ - .stack-work/
policy: pull-push before_script:
- sudo chown -R test /builds/gargantext
script: script:
- echo "Building the project from '$CI_PROJECT_DIR'"
- git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache - git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
- git config --global --add safe.directory '*' - git config --global --add safe.directory '*'
- nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'" - nix-shell --run "stack build --no-terminal --fast --dry-run"
allow_failure: false allow_failure: false
bench: bench:
...@@ -54,6 +58,8 @@ bench: ...@@ -54,6 +58,8 @@ bench:
- dist-newstyle/ - dist-newstyle/
- .cabal/ - .cabal/
policy: pull-push policy: pull-push
before_script:
- sudo chown -R test /builds/gargantext
script: script:
- nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-bench --flags +no-phylo-debug-logs --ghc-options='-O2 -fclear-plugins'" - nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-bench --flags +no-phylo-debug-logs --ghc-options='-O2 -fclear-plugins'"
allow_failure: true allow_failure: true
...@@ -63,16 +69,7 @@ test: ...@@ -63,16 +69,7 @@ test:
# The tests needs to run as the 'test' user, because they leverage the # The tests needs to run as the 'test' user, because they leverage the
# initdb utility from postgres that cannot be run by 'root'. # initdb utility from postgres that cannot be run by 'root'.
before_script: before_script:
- echo "Creating test user..." - sudo chown -R test /builds/gargantext
- mkdir -p /home/test
- mkdir -p /root/.config
- useradd -U test
- chown -R test:test dist-newstyle/
- chown -R test:test /root/
- chown -R test:test $STORE_DIR
- chown -R test:test ${CABAL_DIR}
- mkdir -p "$XDG_CACHE_HOME/nix"
- chown -R test:test "$XDG_CACHE_HOME/nix"
cache: cache:
key: cabal.project key: cabal.project
...@@ -91,13 +88,13 @@ test: ...@@ -91,13 +88,13 @@ test:
export TEST_NIX_PATH=$(nix-shell --run "echo -n \$PATH") export TEST_NIX_PATH=$(nix-shell --run "echo -n \$PATH")
echo "Found test nix path at ${TEST_NIX_PATH}" echo "Found test nix path at ${TEST_NIX_PATH}"
nix-shell --run "./bin/update-project-dependencies $STORE_DIR" nix-shell --run "./bin/update-project-dependencies $STORE_DIR"
mkdir -p /root/.cache/cabal/logs #mkdir -p /root/.cache/cabal/logs
chown -R test:test /root/.cache/cabal/logs/ #chown -R test:test /root/.cache/cabal/logs/
chown -R test:test "$TEST_TMPDIR" #chown -R test:test "$TEST_TMPDIR"
nix-shell --run "chown -R test:test /root/.config/ && su -m test -c \"export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'\"" nix-shell --run "export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --verbose=2 --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
chown -R root:root dist-newstyle/ sudo chown -R root:root dist-newstyle/
chown -R root:root /root/ #chown -R root:root /root/
chown -R root:root $STORE_DIR sudo chown -R root:root $STORE_DIR
chown -R root:root /root/.cache/cabal/logs/ #chown -R root:root /root/.cache/cabal/logs/
...@@ -117,15 +117,17 @@ And provide a name and a password for the master user as requested. ...@@ -117,15 +117,17 @@ And provide a name and a password for the master user as requested.
CoreNLP can be started with nix: CoreNLP can be started with nix:
```shell ```shell
nix-shell --run startCoreNLPServer.sh nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"#
``` ```
By default, this starts on port `9000`. If you want a different port, then do: By default, this starts on port `9000`. If you want a different port, then do:
```shell ```shell
nix-shell --run 'startCoreNLPServer.sh -p 9001' nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"# -- -p 9001
``` ```
See https://gitlab.iscpif.fr/gargantext/corenlp-nix for more info.
### Running Gargantext ### Running Gargantext
From inside the `haskell-gargantext/` directory, run From inside the `haskell-gargantext/` directory, run
......
...@@ -58,7 +58,11 @@ iniParser = fmap CCMD_ini $ IniArgs <$> ...@@ -58,7 +58,11 @@ iniParser = fmap CCMD_ini $ IniArgs <$>
(optional . strOption $ long "toml-path" <> help "Path to the output .toml file") <*> (optional . strOption $ long "toml-path" <> help "Path to the output .toml file") <*>
(flag False True (long "dry-run" <> help "If set, will only output generated .toml file to stdout")) (flag False True (long "dry-run" <> help "If set, will only output generated .toml file to stdout"))
convertConfigs :: Ini.GargConfig -> IniMail.MailConfig -> IniNLP.NLPConfig -> PGS.ConnectInfo -> Config.GargConfig convertConfigs :: Ini.GargConfig
-> IniMail.MailConfig
-> IniNLP.NLPConfig
-> PGS.ConnectInfo
-> Config.GargConfig
convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo = convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo =
Config.GargConfig { _gc_secrets = CTypes.SecretsConfig { _s_master_user = _gc_masteruser Config.GargConfig { _gc_secrets = CTypes.SecretsConfig { _s_master_user = _gc_masteruser
, _s_secret_key = _gc_secretkey , _s_secret_key = _gc_secretkey
......
...@@ -17,7 +17,7 @@ fi ...@@ -17,7 +17,7 @@ fi
# `cabal.project.freeze`. This ensures the files stay deterministic so that CI # `cabal.project.freeze`. This ensures the files stay deterministic so that CI
# cache can kick in. # cache can kick in.
expected_cabal_project_hash="eb8fdb1a14aa2f7a13f565cf7fa9f6ab0e2dab9212538aed0db5691015be286b" expected_cabal_project_hash="eb8fdb1a14aa2f7a13f565cf7fa9f6ab0e2dab9212538aed0db5691015be286b"
expected_cabal_project_freeze_hash="553b98aadb35506a305bd740cdd71f5fadc1e6d55d10f91cf39daa6735a63d78" expected_cabal_project_freeze_hash="a5eb1d9a331266fef56f490712decbd3eaff1fd0daa8bc63f893238a7f47df93"
cabal --store-dir=$STORE_DIR v2-build --dry-run cabal --store-dir=$STORE_DIR v2-build --dry-run
......
...@@ -236,7 +236,7 @@ constraints: any.Boolean ==0.2.4, ...@@ -236,7 +236,7 @@ constraints: any.Boolean ==0.2.4,
hedis -dev, hedis -dev,
any.hlcm ==0.2.2, any.hlcm ==0.2.2,
any.hmatrix ==0.20.2, any.hmatrix ==0.20.2,
hmatrix -disable-default-paths -no-random_r -openblas, hmatrix -disable-default-paths -no-random_r +openblas,
any.hmatrix-gsl-stats ==0.4.1.8, any.hmatrix-gsl-stats ==0.4.1.8,
hmatrix-gsl-stats -onlygsl, hmatrix-gsl-stats -onlygsl,
any.hourglass ==0.2.12, any.hourglass ==0.2.12,
......
FROM ubuntu:noble FROM ubuntu:noble
## NOTA BENE: In order for this to be built successfully, you have to run ./devops/coreNLP/build.sh first.
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Rome ENV TZ=Europe/Rome
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
ENV USER=root
ENV SHELL /bin/bash
ENV PATH=/root/.nix-profile/bin:$PATH
ENV PATH=/root/.local/bin:$PATH
RUN apt-get update && \ RUN apt-get update && \
apt-get install --no-install-recommends -y \ apt-get install --no-install-recommends -y \
apt-transport-https \ apt-transport-https \
ca-certificates \ ca-certificates \
curl \ curl \
fontconfig \
git \ git \
gnupg2 \ gnupg2 \
locales \ locales \
...@@ -26,19 +20,32 @@ RUN apt-get update && \ ...@@ -26,19 +20,32 @@ RUN apt-get update && \
#zlib1g-dev \ #zlib1g-dev \
unzip && \ unzip && \
apt-get clean && rm -rf /var/lib/apt/lists/* && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \
mkdir -m 0755 /nix && groupadd -r nixbld && chown root /nix && \
for n in $(seq 1 10); do useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(command -v nologin)" "nixbld$n"; done && \
gpg --batch --keyserver keys.openpgp.org --recv-keys 7D1E8AFD1D4A16D71FADA2F2CCC85C0E40C06A8C && \ gpg --batch --keyserver keys.openpgp.org --recv-keys 7D1E8AFD1D4A16D71FADA2F2CCC85C0E40C06A8C && \
gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01 gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01
RUN locale-gen en_US.UTF-8
RUN useradd -m test
ENV USER=test
ENV SHELL=/bin/bash
ENV PATH=/home/test/.nix-profile/bin:$PATH
ENV PATH=/home/test/.local/bin:$PATH
RUN mkdir -m 0777 /nix && chown test /nix && \
mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext"
RUN echo 'test ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER test
SHELL ["/bin/bash", "-o", "pipefail", "-c"] SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN set -o pipefail && \ RUN bash <(curl --proto '=https' --tlsv1.2 -L https://releases.nixos.org/nix/nix-2.30.2/install) --no-daemon && \
locale-gen en_US.UTF-8 && \ #chown root -R /nix && \
bash <(curl -L https://releases.nixos.org/nix/nix-2.26.2/install) --no-daemon && \ mkdir -vp "$HOME/.config/nix" && \
chown root -R /nix && \ echo "experimental-features = nix-command flakes" >> $HOME/.config/nix/nix.conf
. "$HOME/.nix-profile/etc/profile.d/nix.sh" && \
mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext" && \ WORKDIR /nix-ci-build
# prefetch corenlp
RUN nix build 'git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git' && \
echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \ echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \
echo `which nix-env` && \ echo `which nix-env` && \
. $HOME/.bashrc && nix-env --version . $HOME/.bashrc && nix-env --version
...@@ -46,8 +53,7 @@ RUN set -o pipefail && \ ...@@ -46,8 +53,7 @@ RUN set -o pipefail && \
# We want to cache nix artifacts in the Dockerfile to improve CI speed # We want to cache nix artifacts in the Dockerfile to improve CI speed
COPY ./shell.nix /nix-ci-build/ COPY ./shell.nix /nix-ci-build/
COPY ./nix /nix-ci-build/ COPY ./nix /nix-ci-build/
RUN set -o pipefail && \ RUN nix-build shell.nix
pushd /nix-ci-build/ && nix-build shell.nix && popd
WORKDIR "/builds/gargantext/" WORKDIR "/builds/gargantext/"
# https://nixos.wiki/wiki/Java
{ fetchzip,
makeWrapper,
stdenv,
writeShellScript,
jre,
version ? "4.5.9",
hash ? "sha256-DOGBkGJfvR1PoXz2CNoo58HXwGLxvPKMChRqlrFtQLQ=",
}:
stdenv.mkDerivation (finalAttrs:
let
startServer = writeShellScript "startCoreNLPServer.sh" ''
set -x
PORT=9000
while getopts ':p:h' opt; do
case $opt in
(p) PORT=$OPTARG;;
(h) echo "$(basename $0) [-p 9000]"
exit 0
;;
esac
done
shift "$((OPTIND - 1))"
${jre}/bin/java -mx4g -cp "$CORENLP_PATH/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port $PORT -timeout 15000 "$@"
'';
# see https://huggingface.co/stanfordnlp/CoreNLP/commits/main
versionCommits = {
"4.5.8" = "34264e88b7add9e0045f4727bc7d1872385f06aa";
"4.5.9" = "06f79ee8b1ec475d7630b1871bfd75a57c77ffa4";
};
commit = versionCommits."${finalAttrs.version}";
in
{
name = "corenlp";
inherit version;
src = fetchzip {
inherit hash;
# url = "http://nlp.stanford.edu/software/stanford-corenlp-${finalAttrs.version}.zip";
# huggin face is more stable
url = "https://huggingface.co/stanfordnlp/CoreNLP/resolve/${commit}/stanford-corenlp-latest.zip";
};
buildInputs = [
jre
];
nativeBuildInputs = [
makeWrapper
];
phases = [ "unpackPhase" "installPhase" ];
installPhase = ''
runHook preInstall
mkdir -p $out/bin
mkdir -p $out/share/corenlp
cp -r . $out/share/corenlp
makeWrapper ${startServer} $out/bin/startCoreNLPServer.sh \
--set CORENLP_PATH "$out/share/corenlp"
runHook postInstall
'';
}
)
...@@ -8,13 +8,22 @@ ...@@ -8,13 +8,22 @@
rec { rec {
inherit pkgs; inherit pkgs;
# Create a custom stdenv with GCC 14 on Linux. This is because
# pkgs.stdenv is GCC version 13 and when using 'nix' inside the
# shell, one gets strange 'CXX_ABI' errors.
customStdenv =
if pkgs.stdenv.isLinux then
pkgs.gcc14Stdenv
else
pkgs.stdenv; # Keep default on other platforms (clang on macOS)
ghcVersion = "ghc966"; ghcVersion = "ghc966";
gargGhc = pkgs.haskell.compiler.${ghcVersion}; gargGhc = pkgs.haskell.compiler.${ghcVersion};
cabal_install = pkgs.haskell.lib.compose.justStaticExecutables pkgs.haskell.packages.${ghcVersion}.cabal-install; cabal_install = pkgs.haskell.lib.compose.justStaticExecutables pkgs.haskell.packages.${ghcVersion}.cabal-install;
graphviz = pkgs.callPackage ./graphviz.nix { }; graphviz = pkgs.callPackage ./graphviz.nix { };
igraph_0_10_4 = pkgs.callPackage ./igraph.nix { }; igraph_0_10_4 = pkgs.callPackage ./igraph.nix { };
corenlp = pkgs.callPackage ./corenlp.nix { }; # 4.5.9
cabal2stack = pkgs.callPackage ./cabal2stack.nix { ghc = gargGhc; }; cabal2stack = pkgs.callPackage ./cabal2stack.nix { ghc = gargGhc; };
nng_notls = pkgs.nng.overrideAttrs (old: { nng_notls = pkgs.nng.overrideAttrs (old: {
cmakeFlags = (old.cmakeFlags or [ ]) ++ [ "-DNNG_ENABLE_TLS=OFF" ]; cmakeFlags = (old.cmakeFlags or [ ]) ++ [ "-DNNG_ENABLE_TLS=OFF" ];
...@@ -32,7 +41,6 @@ rec { ...@@ -32,7 +41,6 @@ rec {
blas blas
bzip2 bzip2
cabal2stack cabal2stack
corenlp
curl curl
czmq czmq
docker-compose docker-compose
...@@ -52,10 +60,11 @@ rec { ...@@ -52,10 +60,11 @@ rec {
libsodium libsodium
nng_notls nng_notls
nil # nix language server nil # nix language server
openblas
pcre pcre
pkg-config pkg-config
postgresql postgresql
stdenv.cc.cc customStdenv.cc.cc
xz xz
zlib zlib
zeromq zeromq
...@@ -67,11 +76,10 @@ rec { ...@@ -67,11 +76,10 @@ rec {
shellHook = '' shellHook = ''
export LD_LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}:$LD_LIBRARY_PATH"
export LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}" export LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}"
export PATH="${pkgs.gccStdenv}/bin:$PATH" export CXX_PATH="${customStdenv.cc.cc.lib}"
export NIX_CC="${pkgs.gccStdenv}" export LIB_PATHS="${libPaths}"
export CC="${pkgs.gccStdenv}/bin/gcc"
''; '';
shell = pkgs.mkShell.override { stdenv = pkgs.gccStdenv; } { shell = pkgs.mkShell.override { stdenv = customStdenv; } {
name = "gargantext-shell"; name = "gargantext-shell";
buildInputs = hsBuildInputs ++ nonhsBuildInputs; buildInputs = hsBuildInputs ++ nonhsBuildInputs;
inherit shellHook; inherit shellHook;
......
...@@ -387,7 +387,7 @@ flags: ...@@ -387,7 +387,7 @@ flags:
hmatrix: hmatrix:
"disable-default-paths": false "disable-default-paths": false
"no-random_r": false "no-random_r": false
openblas: false openblas: true
"hmatrix-gsl-stats": "hmatrix-gsl-stats":
onlygsl: false onlygsl: false
hsc2hs: hsc2hs:
......
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeApplications #-}
module Main where module Main where
import Control.Monad ( MonadFail(fail) ) import Control.Monad ( MonadFail(fail) )
import Data.Text (isInfixOf)
import Data.Text qualified as T import Data.Text qualified as T
import Gargantext.Prelude hiding (isInfixOf) import Gargantext.Prelude
import System.IO ( BufferMode(NoBuffering), hSetBuffering ) import System.IO ( BufferMode(NoBuffering), hGetContents, hSetBuffering )
import System.IO.Error (userError)
import System.Posix.Process
import System.Posix.Signals
import System.Process import System.Process
import System.Posix.Process ( getProcessGroupIDOf ) -- import System.Posix.Process ( getProcessGroupIDOf )
import System.Posix.Signals ( keyboardSignal, signalProcessGroup ) -- import System.Posix.Signals ( keyboardSignal, signalProcessGroup )
import Test.API qualified as API import Test.API qualified as API
import Test.Database.Operations qualified as DB import Test.Database.Operations qualified as DB
import Test.Database.Transactions qualified as DBT import Test.Database.Transactions qualified as DBT
...@@ -39,13 +42,12 @@ import Test.Utils.Crypto qualified as Crypto ...@@ -39,13 +42,12 @@ import Test.Utils.Crypto qualified as Crypto
import Test.Utils.Jobs qualified as Jobs import Test.Utils.Jobs qualified as Jobs
startCoreNLPServer :: IO (Maybe ProcessHandle) startCoreNLPServer :: IO (Handle, Handle, ProcessHandle)
startCoreNLPServer = do startCoreNLPServer = do
putText "calling start core nlp" putText "calling start core nlp"
devNull <- openFile "/dev/null" WriteMode let p = proc "nix" [ "run"
let p = proc "startCoreNLPServer.sh" [] , "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git" ]
let connect = do (_, m_stdout_hdl, m_stderr_hdl, hdl) <- (createProcess $ p { cwd = Nothing
(_, _, _, hdl) <- createProcess p { cwd = Nothing
-- NOTE(adn) Issue #451, this one has to stay disabled, because if we -- NOTE(adn) Issue #451, this one has to stay disabled, because if we
-- turn it on, despite the confusing documentation on the `process` library -- turn it on, despite the confusing documentation on the `process` library
-- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead -- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
...@@ -54,26 +56,21 @@ startCoreNLPServer = do ...@@ -54,26 +56,21 @@ startCoreNLPServer = do
-- to Ctrl^C requests. -- to Ctrl^C requests.
, delegate_ctlc = False , delegate_ctlc = False
, create_group = True , create_group = True
, std_out = UseHandle devNull , std_out = CreatePipe
, std_err = UseHandle devNull , std_err = CreatePipe
} }) `catch` \(e :: SomeException) ->
pure $ Just hdl
connect `catch` \e -> do
putText $ T.pack $ show @SomeException e
case e of case e of
_ | True <- "does not exist" `isInfixOf` (T.pack . show @SomeException $ e) _ | True <- "does not exist" `T.isInfixOf` (T.pack . show @SomeException $ e)
-> fail $ "Cannot execute the 'startCoreNLPServer.sh' script. Make sure you are in a nix environment." -> fail $ "Cannot execute the 'corenlp' via nix flakes. Make sure you have 'nix' installed."
| True <- "Address already in use" `isInfixOf` (T.pack . show @SomeException $ e)
-> do
putText "Address already in use, but we hope for the best!"
pure Nothing
| otherwise -> throwIO e | otherwise -> throwIO e
killProcessTree :: Maybe ProcessHandle -> IO () let stdout_hdl = fromMaybe (errorTrace "Can't get stdout handle!") m_stdout_hdl
killProcessTree Nothing = pure () let stderr_hdl = fromMaybe (errorTrace "Can't get stderr handle!") m_stderr_hdl
killProcessTree (Just ph) = do
pure (stdout_hdl, stderr_hdl, hdl)
killProcessTree :: (Handle, Handle, ProcessHandle) -> IO ()
killProcessTree (_, _, ph) = do
pid <- getPid ph pid <- getPid ph
case pid of case pid of
Nothing -> putText "Process already terminated" Nothing -> putText "Process already terminated"
...@@ -81,6 +78,23 @@ killProcessTree (Just ph) = do ...@@ -81,6 +78,23 @@ killProcessTree (Just ph) = do
pgid <- getProcessGroupIDOf p pgid <- getProcessGroupIDOf p
signalProcessGroup keyboardSignal pgid signalProcessGroup keyboardSignal pgid
-- NOTE(seeg) createProcess is nonblocking, and so its exception
-- handling is incomplete:
-- λ> createProcess $ proc "l" []
-- *** Exception: l: createProcess: posix_spawnp: does not exist (No such file or directory)
-- λ> createProcess $ proc "ls" ["haha"]
-- ls: cannot access 'haha': No such file or directory
-- Credit for this construct goes to @adn
waitOrDie :: (Handle, Handle, ProcessHandle) -> IO ()
waitOrDie (stdout_h, stderr_h, h) = do
ec <- waitForProcess h
when (ec /= ExitSuccess) $ do
out <- hGetContents stdout_h
err <- hGetContents stderr_h
throwIO $ userError ("Async process quit!\nStdout:\n" <> out <> "\nStderr:\n" <> err)
-- It's especially important to use Hspec for DB tests, because, -- It's especially important to use Hspec for DB tests, because,
-- unlike 'tasty', 'Hspec' has explicit control over parallelism, -- unlike 'tasty', 'Hspec' has explicit control over parallelism,
-- and it's important that DB tests are run according to a very -- and it's important that DB tests are run according to a very
...@@ -88,14 +102,10 @@ killProcessTree (Just ph) = do ...@@ -88,14 +102,10 @@ killProcessTree (Just ph) = do
-- Unfortunately it's not possibly to use the 'tasty-hspec' adapter -- Unfortunately it's not possibly to use the 'tasty-hspec' adapter
-- because by the time we get a 'TestTree' out of the adapter library, -- because by the time we get a 'TestTree' out of the adapter library,
-- the information about parallelism is lost. -- the information about parallelism is lost.
--
-- /IMPORTANT/: For these tests to run correctly, you have to run
-- ./devops/coreNLP/build.sh first. You have to run it only /once/,
-- and then you are good to go for the time being.
main :: IO () main :: IO ()
main = do main = do
hSetBuffering stdout NoBuffering hSetBuffering stdout NoBuffering
bracket startCoreNLPServer killProcessTree (const run_tests) bracket startCoreNLPServer killProcessTree (\h -> race (waitOrDie h) run_tests *> pure())
where where
run_tests = hspec $ sequential $ do run_tests = hspec $ sequential $ do
API.tests API.tests
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment