Skip to content

haskell-gargantext
haskell-gargantext
Verified Commit cd0aa9ca authored by Przemyslaw Kaminski's avatar Przemyslaw Kaminski
Browse files
Options

Merge branch 'dev' into 494-dev-phylo-for-hh-mm-ss

parents 36741105 84bbdd5e
Hide whitespace changes
Inline Side-by-side
Showing with 131 additions and 170 deletions
.gitlab-ci.yml
View file @ cd0aa9ca
# Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
image: cgenie/gargantext:9.6.6
image: cgenie/gargantext:9.6.6-with-test-user
variables:
STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
......@@ -17,32 +17,36 @@ stages:
- bench
- test
stack:
stage: stack
cabal:
stage: cabal
cache:
key: stack.yaml
key: cabal.project
paths:
- .stack-root/
- .stack-work/
- dist-newstyle/
- .cabal/
policy: pull-push
before_script:
- sudo chown -R test /builds/gargantext
script:
- echo "Building the project from '$CI_PROJECT_DIR'"
- git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
- git config --global --add safe.directory '*'
- nix-shell --run "stack build --no-terminal --fast --dry-run"
- nix-shell --run "cabal update && ./bin/update-project-dependencies && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
allow_failure: false
cabal:
stage: cabal
stack:
stage: stack
cache:
key: cabal.project
key: stack.yaml
paths:
- dist-newstyle/
- .cabal/
policy: pull-push
- .stack-root/
- .stack-work/
before_script:
- sudo chown -R test /builds/gargantext
script:
- echo "Building the project from '$CI_PROJECT_DIR'"
- git config --global --add safe.directory $XDG_CACHE_HOME/nix/tarball-cache
- git config --global --add safe.directory '*'
- nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-build all --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
- nix-shell --run "stack build --no-terminal --fast --dry-run"
allow_failure: false
bench:
......@@ -54,6 +58,8 @@ bench:
- dist-newstyle/
- .cabal/
policy: pull-push
before_script:
- sudo chown -R test /builds/gargantext
script:
- nix-shell --run "./bin/update-project-dependencies $STORE_DIR && cabal --store-dir=$STORE_DIR v2-bench --flags +no-phylo-debug-logs --ghc-options='-O2 -fclear-plugins'"
allow_failure: true
......@@ -63,16 +69,7 @@ test:
# The tests needs to run as the 'test' user, because they leverage the
# initdb utility from postgres that cannot be run by 'root'.
before_script:
- echo "Creating test user..."
- mkdir -p /home/test
- mkdir -p /root/.config
- useradd -U test
- chown -R test:test dist-newstyle/
- chown -R test:test /root/
- chown -R test:test $STORE_DIR
- chown -R test:test ${CABAL_DIR}
- mkdir -p "$XDG_CACHE_HOME/nix"
- chown -R test:test "$XDG_CACHE_HOME/nix"
- sudo chown -R test /builds/gargantext
cache:
key: cabal.project
......@@ -91,13 +88,13 @@ test:
export TEST_NIX_PATH=$(nix-shell --run "echo -n \$PATH")
echo "Found test nix path at ${TEST_NIX_PATH}"
nix-shell --run "./bin/update-project-dependencies $STORE_DIR"
mkdir -p /root/.cache/cabal/logs
chown -R test:test /root/.cache/cabal/logs/
chown -R test:test "$TEST_TMPDIR"
#mkdir -p /root/.cache/cabal/logs
#chown -R test:test /root/.cache/cabal/logs/
#chown -R test:test "$TEST_TMPDIR"
nix-shell --run "chown -R test:test /root/.config/ && su -m test -c \"export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'\""
nix-shell --run "export PATH=$PATH:$TEST_NIX_PATH && export TMPDIR=$TEST_TMPDIR && cd /builds/gargantext/haskell-gargantext; $CABAL --store-dir=$STORE_DIR v2-test --test-show-details=streaming --verbose=2 --flags 'test-crypto no-phylo-debug-logs' --ghc-options='-O0 -fclear-plugins'"
chown -R root:root dist-newstyle/
chown -R root:root /root/
chown -R root:root $STORE_DIR
chown -R root:root /root/.cache/cabal/logs/
sudo chown -R root:root dist-newstyle/
#chown -R root:root /root/
sudo chown -R root:root $STORE_DIR
#chown -R root:root /root/.cache/cabal/logs/
README.md
View file @ cd0aa9ca
......@@ -117,15 +117,17 @@ And provide a name and a password for the master user as requested.
CoreNLP can be started with nix:
```shell
nix-shell --run startCoreNLPServer.sh
nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"#
```
By default, this starts on port `9000`. If you want a different port, then do:
```shell
nix-shell --run 'startCoreNLPServer.sh -p 9001'
nix run "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git"# -- -p 9001
```
See https://gitlab.iscpif.fr/gargantext/corenlp-nix for more info.
### Running Gargantext
From inside the `haskell-gargantext/` directory, run
......
bin/gargantext-cli/CLI/Ini.hs
View file @ cd0aa9ca
......@@ -58,7 +58,11 @@ iniParser = fmap CCMD_ini $ IniArgs <$>
(optional . strOption $ long "toml-path" <> help "Path to the output .toml file") <*>
(flag False True (long "dry-run" <> help "If set, will only output generated .toml file to stdout"))
convertConfigs :: Ini.GargConfig -> IniMail.MailConfig -> IniNLP.NLPConfig -> PGS.ConnectInfo -> Config.GargConfig
convertConfigs :: Ini.GargConfig
-> IniMail.MailConfig
-> IniNLP.NLPConfig
-> PGS.ConnectInfo
-> Config.GargConfig
convertConfigs ini@(Ini.GargConfig { .. }) iniMail nlpConfig connInfo =
Config.GargConfig { _gc_secrets = CTypes.SecretsConfig { _s_master_user = _gc_masteruser
, _s_secret_key = _gc_secretkey
......
bin/update-project-dependencies
View file @ cd0aa9ca
......@@ -17,7 +17,7 @@ fi
# `cabal.project.freeze`. This ensures the files stay deterministic so that CI
# cache can kick in.
expected_cabal_project_hash="eb8fdb1a14aa2f7a13f565cf7fa9f6ab0e2dab9212538aed0db5691015be286b"
expected_cabal_project_freeze_hash="553b98aadb35506a305bd740cdd71f5fadc1e6d55d10f91cf39daa6735a63d78"
expected_cabal_project_freeze_hash="a5eb1d9a331266fef56f490712decbd3eaff1fd0daa8bc63f893238a7f47df93"
cabal --store-dir=$STORE_DIR v2-build --dry-run
......
cabal.project.freeze
View file @ cd0aa9ca
......@@ -236,7 +236,7 @@ constraints: any.Boolean ==0.2.4,
hedis -dev,
any.hlcm ==0.2.2,
any.hmatrix ==0.20.2,
hmatrix -disable-default-paths -no-random_r -openblas,
hmatrix -disable-default-paths -no-random_r +openblas,
any.hmatrix-gsl-stats ==0.4.1.8,
hmatrix-gsl-stats -onlygsl,
any.hourglass ==0.2.12,
......
devops/docker/Dockerfile
View file @ cd0aa9ca
FROM ubuntu:noble
## NOTA BENE: In order for this to be built successfully, you have to run ./devops/coreNLP/build.sh first.
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/Rome
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
ENV USER=root
ENV SHELL /bin/bash
ENV PATH=/root/.nix-profile/bin:$PATH
ENV PATH=/root/.local/bin:$PATH
RUN apt-get update && \
apt-get install --no-install-recommends -y \
apt-transport-https \
ca-certificates \
curl \
fontconfig \
git \
gnupg2 \
locales \
......@@ -26,19 +20,32 @@ RUN apt-get update && \
#zlib1g-dev \
unzip && \
apt-get clean && rm -rf /var/lib/apt/lists/* && \
mkdir -m 0755 /nix && groupadd -r nixbld && chown root /nix && \
for n in $(seq 1 10); do useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(command -v nologin)" "nixbld$n"; done && \
gpg --batch --keyserver keys.openpgp.org --recv-keys 7D1E8AFD1D4A16D71FADA2F2CCC85C0E40C06A8C && \
gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01
RUN locale-gen en_US.UTF-8
RUN useradd -m test
ENV USER=test
ENV SHELL=/bin/bash
ENV PATH=/home/test/.nix-profile/bin:$PATH
ENV PATH=/home/test/.local/bin:$PATH
RUN mkdir -m 0777 /nix && chown test /nix && \
mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext"
RUN echo 'test ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER test
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN set -o pipefail && \
locale-gen en_US.UTF-8 && \
bash <(curl -L https://releases.nixos.org/nix/nix-2.26.2/install) --no-daemon && \
chown root -R /nix && \
. "$HOME/.nix-profile/etc/profile.d/nix.sh" && \
mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext" && \
RUN bash <(curl --proto '=https' --tlsv1.2 -L https://releases.nixos.org/nix/nix-2.30.2/install) --no-daemon && \
#chown root -R /nix && \
mkdir -vp "$HOME/.config/nix" && \
echo "experimental-features = nix-command flakes" >> $HOME/.config/nix/nix.conf
WORKDIR /nix-ci-build
# prefetch corenlp
RUN nix build 'git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git' && \
echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \
echo `which nix-env` && \
. $HOME/.bashrc && nix-env --version
......@@ -46,8 +53,7 @@ RUN set -o pipefail && \
# We want to cache nix artifacts in the Dockerfile to improve CI speed
COPY ./shell.nix /nix-ci-build/
COPY ./nix /nix-ci-build/
RUN set -o pipefail && \
pushd /nix-ci-build/ && nix-build shell.nix && popd
RUN nix-build shell.nix
WORKDIR "/builds/gargantext/"
nix/corenlp.nix deleted 100644 → 0
View file @ 36741105
# https://nixos.wiki/wiki/Java
{ fetchzip,
makeWrapper,
stdenv,
writeShellScript,
jre,
version ? "4.5.9",
hash ? "sha256-DOGBkGJfvR1PoXz2CNoo58HXwGLxvPKMChRqlrFtQLQ=",
}:
stdenv.mkDerivation (finalAttrs:
let
startServer = writeShellScript "startCoreNLPServer.sh" ''
set -x
PORT=9000
while getopts ':p:h' opt; do
case $opt in
(p) PORT=$OPTARG;;
(h) echo "$(basename $0) [-p 9000]"
exit 0
;;
esac
done
shift "$((OPTIND - 1))"
${jre}/bin/java -mx4g -cp "$CORENLP_PATH/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port $PORT -timeout 15000 "$@"
'';
# see https://huggingface.co/stanfordnlp/CoreNLP/commits/main
versionCommits = {
"4.5.8" = "34264e88b7add9e0045f4727bc7d1872385f06aa";
"4.5.9" = "06f79ee8b1ec475d7630b1871bfd75a57c77ffa4";
};
commit = versionCommits."${finalAttrs.version}";
in
{
name = "corenlp";
inherit version;
src = fetchzip {
inherit hash;
# url = "http://nlp.stanford.edu/software/stanford-corenlp-${finalAttrs.version}.zip";
# huggin face is more stable
url = "https://huggingface.co/stanfordnlp/CoreNLP/resolve/${commit}/stanford-corenlp-latest.zip";
};
buildInputs = [
jre
];
nativeBuildInputs = [
makeWrapper
];
phases = [ "unpackPhase" "installPhase" ];
installPhase = ''
runHook preInstall
mkdir -p $out/bin
mkdir -p $out/share/corenlp
cp -r . $out/share/corenlp
makeWrapper ${startServer} $out/bin/startCoreNLPServer.sh \
--set CORENLP_PATH "$out/share/corenlp"
runHook postInstall
'';
}
)
nix/pkgs.nix
View file @ cd0aa9ca
......@@ -8,13 +8,22 @@
rec {
inherit pkgs;
# Create a custom stdenv with GCC 14 on Linux. This is because
# pkgs.stdenv is GCC version 13 and when using 'nix' inside the
# shell, one gets strange 'CXX_ABI' errors.
customStdenv =
if pkgs.stdenv.isLinux then
pkgs.gcc14Stdenv
else
pkgs.stdenv; # Keep default on other platforms (clang on macOS)
ghcVersion = "ghc966";
gargGhc = pkgs.haskell.compiler.${ghcVersion};
cabal_install = pkgs.haskell.lib.compose.justStaticExecutables pkgs.haskell.packages.${ghcVersion}.cabal-install;
graphviz = pkgs.callPackage ./graphviz.nix { };
igraph_0_10_4 = pkgs.callPackage ./igraph.nix { };
corenlp = pkgs.callPackage ./corenlp.nix { }; # 4.5.9
cabal2stack = pkgs.callPackage ./cabal2stack.nix { ghc = gargGhc; };
nng_notls = pkgs.nng.overrideAttrs (old: {
cmakeFlags = (old.cmakeFlags or [ ]) ++ [ "-DNNG_ENABLE_TLS=OFF" ];
......@@ -32,7 +41,6 @@ rec {
blas
bzip2
cabal2stack
corenlp
curl
czmq
docker-compose
......@@ -52,10 +60,11 @@ rec {
libsodium
nng_notls
nil # nix language server
openblas
pcre
pkg-config
postgresql
stdenv.cc.cc
customStdenv.cc.cc
xz
zlib
zeromq
......@@ -67,11 +76,10 @@ rec {
shellHook = ''
export LD_LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}:$LD_LIBRARY_PATH"
export LIBRARY_PATH="${pkgs.gfortran.cc.lib}:${libPaths}"
export PATH="${pkgs.gccStdenv}/bin:$PATH"
export NIX_CC="${pkgs.gccStdenv}"
export CC="${pkgs.gccStdenv}/bin/gcc"
export CXX_PATH="${customStdenv.cc.cc.lib}"
export LIB_PATHS="${libPaths}"
'';
shell = pkgs.mkShell.override { stdenv = pkgs.gccStdenv; } {
shell = pkgs.mkShell.override { stdenv = customStdenv; } {
name = "gargantext-shell";
buildInputs = hsBuildInputs ++ nonhsBuildInputs;
inherit shellHook;
......
stack.yaml
View file @ cd0aa9ca
......@@ -387,7 +387,7 @@ flags:
hmatrix:
"disable-default-paths": false
"no-random_r": false
openblas: false
openblas: true
"hmatrix-gsl-stats":
onlygsl: false
hsc2hs:
......
test/Main.hs
View file @ cd0aa9ca
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
module Main where
import Control.Monad ( MonadFail(fail) )
import Data.Text (isInfixOf)
import Data.Text qualified as T
import Gargantext.Prelude hiding (isInfixOf)
import System.IO ( BufferMode(NoBuffering), hSetBuffering )
import Gargantext.Prelude
import System.IO ( BufferMode(NoBuffering), hGetContents, hSetBuffering )
import System.IO.Error (userError)
import System.Posix.Process
import System.Posix.Signals
import System.Process
import System.Posix.Process ( getProcessGroupIDOf )
import System.Posix.Signals ( keyboardSignal, signalProcessGroup )
-- import System.Posix.Process ( getProcessGroupIDOf )
-- import System.Posix.Signals ( keyboardSignal, signalProcessGroup )
import Test.API qualified as API
import Test.Database.Operations qualified as DB
import Test.Database.Transactions qualified as DBT
......@@ -39,41 +42,35 @@ import Test.Utils.Crypto qualified as Crypto
import Test.Utils.Jobs qualified as Jobs
startCoreNLPServer :: IO (Maybe ProcessHandle)
startCoreNLPServer :: IO (Handle, Handle, ProcessHandle)
startCoreNLPServer = do
putText "calling start core nlp"
devNull <- openFile "/dev/null" WriteMode
let p = proc "startCoreNLPServer.sh" []
let connect = do
(_, _, _, hdl) <- createProcess p { cwd = Nothing
-- NOTE(adn) Issue #451, this one has to stay disabled, because if we
-- turn it on, despite the confusing documentation on the `process` library
-- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
-- delegate it exclusively to the process here, which means that our CoreNLP
-- server will shut down correctly, but the test running will stop responding
-- to Ctrl^C requests.
, delegate_ctlc = False
, create_group = True
, std_out = UseHandle devNull
, std_err = UseHandle devNull
}
pure $ Just hdl
let p = proc "nix" [ "run"
, "git+https://gitlab.iscpif.fr/gargantext/corenlp-nix.git" ]
(_, m_stdout_hdl, m_stderr_hdl, hdl) <- (createProcess $ p { cwd = Nothing
-- NOTE(adn) Issue #451, this one has to stay disabled, because if we
-- turn it on, despite the confusing documentation on the `process` library
-- it will cause the Haskell RTS to completely ignore the Ctrl^c and instead
-- delegate it exclusively to the process here, which means that our CoreNLP
-- server will shut down correctly, but the test running will stop responding
-- to Ctrl^C requests.
, delegate_ctlc = False
, create_group = True
, std_out = CreatePipe
, std_err = CreatePipe
}) `catch` \(e :: SomeException) ->
case e of
_ | True <- "does not exist" `T.isInfixOf` (T.pack . show @SomeException $ e)
-> fail $ "Cannot execute the 'corenlp' via nix flakes. Make sure you have 'nix' installed."
| otherwise -> throwIO e
connect `catch` \e -> do
putText $ T.pack $ show @SomeException e
case e of
_ | True <- "does not exist" `isInfixOf` (T.pack . show @SomeException $ e)
-> fail $ "Cannot execute the 'startCoreNLPServer.sh' script. Make sure you are in a nix environment."
| True <- "Address already in use" `isInfixOf` (T.pack . show @SomeException $ e)
-> do
putText "Address already in use, but we hope for the best!"
pure Nothing
| otherwise -> throwIO e
let stdout_hdl = fromMaybe (errorTrace "Can't get stdout handle!") m_stdout_hdl
let stderr_hdl = fromMaybe (errorTrace "Can't get stderr handle!") m_stderr_hdl
pure (stdout_hdl, stderr_hdl, hdl)
killProcessTree :: Maybe ProcessHandle -> IO ()
killProcessTree Nothing = pure ()
killProcessTree (Just ph) = do
killProcessTree :: (Handle, Handle, ProcessHandle) -> IO ()
killProcessTree (_, _, ph) = do
pid <- getPid ph
case pid of
Nothing -> putText "Process already terminated"
......@@ -81,6 +78,23 @@ killProcessTree (Just ph) = do
pgid <- getProcessGroupIDOf p
signalProcessGroup keyboardSignal pgid
-- NOTE(seeg) createProcess is nonblocking, and so its exception
-- handling is incomplete:
-- λ> createProcess $ proc "l" []
-- *** Exception: l: createProcess: posix_spawnp: does not exist (No such file or directory)
-- λ> createProcess $ proc "ls" ["haha"]
-- ls: cannot access 'haha': No such file or directory
-- Credit for this construct goes to @adn
waitOrDie :: (Handle, Handle, ProcessHandle) -> IO ()
waitOrDie (stdout_h, stderr_h, h) = do
ec <- waitForProcess h
when (ec /= ExitSuccess) $ do
out <- hGetContents stdout_h
err <- hGetContents stderr_h
throwIO $ userError ("Async process quit!\nStdout:\n" <> out <> "\nStderr:\n" <> err)
-- It's especially important to use Hspec for DB tests, because,
-- unlike 'tasty', 'Hspec' has explicit control over parallelism,
-- and it's important that DB tests are run according to a very
......@@ -88,14 +102,10 @@ killProcessTree (Just ph) = do
-- Unfortunately it's not possibly to use the 'tasty-hspec' adapter
-- because by the time we get a 'TestTree' out of the adapter library,
-- the information about parallelism is lost.
--
-- /IMPORTANT/: For these tests to run correctly, you have to run
-- ./devops/coreNLP/build.sh first. You have to run it only /once/,
-- and then you are good to go for the time being.
main :: IO ()
main = do
hSetBuffering stdout NoBuffering
bracket startCoreNLPServer killProcessTree (const run_tests)
bracket startCoreNLPServer killProcessTree (\h -> race (waitOrDie h) run_tests *> pure())
where
run_tests = hspec $ sequential $ do
API.tests
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment