Merge branch 'dev' into dev-tests-refactor

.gitlab-ci.yml

 # Optimising CI speed by using tips from https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
-image: adinapoli/gargantext:v3.2
+image: adinapoli/gargantext:v3.3
 
 variables:
   STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
@@ -37,11 +37,12 @@ cabal:
       - .cabal/
     policy: pull-push
   script:
-    - nix-shell --run "./bin/update-project-dependencies $CABAL_STORE_DIR && cabal --store-dir=$CABAL_STORE_DIR v2-build --ghc-options='-O2 -fclear-plugins'"
+    - nix-shell --run "./bin/update-project-dependencies $CABAL_STORE_DIR && cabal --store-dir=$CABAL_STORE_DIR v2-build --flags test-crypto --ghc-options='-O0 -fclear-plugins'"
   allow_failure: false
 
 bench:
   stage: bench
+  when: manual # trigger it manually, as it causes full recompilation with optimisations enabled.
   cache:
     key: cabal.project
     paths:
@@ -50,7 +51,7 @@ bench:
     policy: pull-push
   script:
     - nix-shell --run "./bin/update-project-dependencies $CABAL_STORE_DIR && cabal --store-dir=$CABAL_STORE_DIR v2-bench --ghc-options='-O2 -fclear-plugins'"
-  allow_failure: false
+  allow_failure: true
 
 test:
   stage: test

CHANGELOG.md

+## Version 0.0.6.9.9.9.5
+
+* [BACK][CI][Improving CI](https://gitlab.iscpif.fr/gargantext/haskell-gargantext/merge_requests/247)
+* [FRONT][ERGO][Display Phylomemy parameters (#580)](https://gitlab.iscpif.fr/gargantext/purescript-gargantext/issues/580)
+
 ## Version 0.0.6.9.9.9.4.9.3
 
 * [FRONT][FIX][Try to invite someone without email (#600)](https://gitlab.iscpif.fr/gargantext/purescript-gargantext/issues/600)

README.md

@@ -189,6 +189,18 @@ From the Backend root folder (haskell-gargantext):
 
 For frontend development and compilation, see the [Frontend Readme.md](https://gitlab.iscpif.fr/gargantext/purescript-gargantext#dev)
 
+### Running tests
+
+From nix shell:
+
+```
+cabal v2-test --test-show-details=streaming
+```
+
+Or, from "outside":
+```
+nix-shell --run "cabal v2-test --test-show-details=streaming"
+```
 ### Working on libraries
 
 When a devlopment is needed on libraries (for instance, the HAL crawler in https://gitlab.iscpif.fr/gargantext/crawlers):

bin/install

 #!/bin/bash
 
-#stack install --nix --profile --test --fast --no-install-ghc --skip-ghc-check
-
-# Another solution to build
-# nix-shell
-# stack build
-# If you have a linker issue:
-# stack build --extra-lib-dirs=/nix/store/r3x96j3kmcs8dv4l02rrjmbhm535jycy-icu4c-72.1/lib
-
-if [[ $1 == "dev" ]] ;
-  then
-    echo "DEV install"
-    #env LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 stack --nix install --no-install-ghc --skip-ghc-check --no-haddock-deps
-  else
-    echo "PROD install (with documentation)"
-    nix-shell --run "cabal update && cabal build && cabal --overwrite-policy=always install"
-    #env LANG=C.UTF-8 stack install --haddock --nix --test --no-install-ghc --skip-ghc-check --no-haddock-deps
-fi
+echo "GarganText, build, install, test and documentation"
+nix-shell --run "cabal update                                     \\
+                 && cabal build                                   \\
+                 && cabal --overwrite-policy=always install       \\
+                 && cabal v2-test --test-show-details=streaming   \\
+                 && cabal haddock"

bin/install-cabal2stack

+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+current_dir=$(basename "$PWD")
+
+if [ "$current_dir" == "bin" ]; then
+    source ./setup-ci-environment
+else
+    source ./bin/setup-ci-environment
+fi
+
+cabal --store-dir=$STORE_DIR v2-update "hackage.haskell.org,${INDEX_STATE}"
+
+# Install cabal2stack if it can't be found.
+if ! cabal2stack --help &> /dev/null
+then
+    echo "cabal2stack could not be found"
+    CURDIR=$PWD
+    git clone https://github.com/iconnect/cabal2stack.git cabal2stack-installer
+    cd cabal2stack-installer
+    cabal --store-dir=$STORE_DIR v2-install --index-state="${INDEX_STATE}" --overwrite-policy=always
+    cd $CURDIR
+    rm -rf cabal2stack-installer
+fi

bin/setup-ci-environment

+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+DEFAULT_STORE=$HOME/.cabal
+STORE_DIR="${1:-$DEFAULT_STORE}"
+INDEX_STATE="2023-12-10T10:34:46Z"

bin/update-project-dependencies

@@ -2,9 +2,15 @@
 
 set -euxo pipefail
 
-DEFAULT_STORE=$HOME/.cabal
-STORE_DIR="${1:-$DEFAULT_STORE}"
-INDEX_STATE="2023-12-10T10:34:46Z"
+current_dir=$(basename "$PWD")
+
+if [ "$current_dir" == "bin" ]; then
+    source ./setup-ci-environment
+    ./install-cabal2stack
+else
+    source ./bin/setup-ci-environment
+    ./bin/install-cabal2stack
+fi
 
 # README!
 # Every time you modify the `cabal.project`, you have to make sure to update
@@ -15,20 +21,6 @@ INDEX_STATE="2023-12-10T10:34:46Z"
 expected_cabal_project_hash="3f5d6b7f26cac4aa5a7f87ba0227a7671041dfe46643ddef79512eb49bd876ec"
 expected_cabal_project_freeze_hash="745c65c246998cfda4d2a7a22df44a9f1f7fb0927e2afc2f16712861bf552c76"
 
-cabal --store-dir=$STORE_DIR v2-update "hackage.haskell.org,${INDEX_STATE}"
-
-# Install cabal2stack if it can't be found.
-if ! cabal2stack --help &> /dev/null
-then
-    echo "cabal2stack could not be found"
-    CURDIR=$PWD
-    git clone https://github.com/iconnect/cabal2stack.git cabal2stack-installer
-    cd cabal2stack-installer
-    cabal --store-dir=$STORE_DIR v2-install --index-state="${INDEX_STATE}" --overwrite-policy=always
-    cd $CURDIR
-    rm -rf cabal2stack-installer
-fi
-
 cabal --store-dir=$STORE_DIR v2-build --dry-run
 cabal2stack --system-ghc --allow-newer --resolver lts-21.17 --resolver-file devops/stack/lts-21.17.yaml -o stack.yaml
 cabal --store-dir=$STORE_DIR v2-freeze

devops/docker/Dockerfile

-FROM adinapoli/gargantext:v3.1
+FROM ubuntu:noble
 
 ## NOTA BENE: In order for this to be built successfully, you have to run ./devops/coreNLP/build.sh first.
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG GHC=9.4.7
+ARG CORENLP=4.5.4
+ARG CORE
 COPY ./shell.nix                                       /builds/gargantext/shell.nix
 COPY ./nix/pkgs.nix                                    /builds/gargantext/nix/pkgs.nix
 COPY ./nix/pinned-23.11.nix                            /builds/gargantext/nix/pinned-23.11.nix
 COPY ./devops/coreNLP/build.sh                         /root/devops/coreNLP/build.sh
 COPY ./devops/coreNLP/startServer.sh                   /root/devops/coreNLP/startServer.sh
+COPY ./bin/setup-ci-environment                        /builds/gargantext/bin/setup-ci-environment
+COPY ./bin/install-cabal2stack                         /builds/gargantext/bin/install-cabal2stack
 
 ENV TZ=Europe/Rome
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+      apt-transport-https \
+      autoconf \
+      automake \
+      build-essential \
+      ca-certificates \
+      curl \
+      gcc \
+      git \
+      gnupg2 \
+      libffi-dev \
+      libffi8 \
+      libgmp-dev \
+      libgmp10 \
+      libncurses-dev \
+      libncurses6 \
+      libnuma-dev \
+      libtinfo6 \
+      locales \
+      lsb-release \
+      software-properties-common \
+      strace \
+      sudo \
+      wget \
+      vim \
+      xz-utils \
+      zlib1g-dev \
+      openjdk-21-jdk \
+      unzip && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    mkdir -m 0755 /nix && groupadd -r nixbld && chown root /nix && \
+    for n in $(seq 1 10); do useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(command -v nologin)" "nixbld$n"; done
 
+RUN gpg --batch --keyserver keys.openpgp.org     --recv-keys 7D1E8AFD1D4A16D71FADA2F2CCC85C0E40C06A8C && \
+    gpg --batch --keyserver keyserver.ubuntu.com --recv-keys FE5AB6C91FEA597C3B31180B73EDE9E8CFBAEF01
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+RUN cd /root/devops/coreNLP; ./build.sh
+
+RUN set -o pipefail && \
+    bash <(curl -L https://releases.nixos.org/nix/nix-2.15.0/install) --no-daemon && \
+    locale-gen en_US.UTF-8 && chown root -R /nix
+
 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
 ENV USER=root
 ENV SHELL /bin/bash
+RUN . "$HOME/.nix-profile/etc/profile.d/nix.sh" && \
+    mkdir -p "/builds/gargantext/" && chmod 777 -R "/builds/gargantext" && \
+    echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> "$HOME/.bashrc" && \
+    echo `which nix-env`
+
 ENV PATH=/root/.nix-profile/bin:$PATH
+
+RUN . $HOME/.bashrc && nix-env --version
+
 ENV PATH=/root/.local/bin:$PATH
 
-RUN cd /builds/gargantext && nix-shell
+RUN cd /builds/gargantext && nix-shell --run "./bin/install-cabal2stack"
 WORKDIR "/builds/gargantext/"
+

gargantext.cabal

@@ -5,7 +5,7 @@ cabal-version: 2.0
 -- see: https://github.com/sol/hpack
 
 name:           gargantext
-version: 0.0.6.9.9.9.4.9.3
+version: 0.0.6.9.9.9.5
 synopsis:       Search, map, share
 description:    Please see README.md
 category:       Data

src/Gargantext/Core/Viz/Phylo.hs

@@ -37,6 +37,8 @@ import Data.Vector (Vector)
 import Gargantext.Core.Utils.Prefix (unPrefix)
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 import Gargantext.Prelude
+import Test.QuickCheck
+import Test.QuickCheck.Instances.Text()
 
 ---------------------
 -- | PhyloConfig | --
@@ -678,3 +680,79 @@ instance NFData Sort
 instance NFData Tagger
 instance NFData PhyloLabel
 
+-- Arbitrary instances
+
+instance Arbitrary PhyloConfig where
+  arbitrary = PhyloConfig <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary 
+                          <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary 
+                          <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary 
+                          <*> vectorOf 10 arbitrary <*> arbitrary <*> vectorOf 10 arbitrary
+
+instance Arbitrary CorpusParser where
+  arbitrary = oneof [ Wos <$> arbitrary
+                    , Csv <$> arbitrary
+                    , Csv' <$> arbitrary
+                    ]
+
+instance Arbitrary ListParser where
+  arbitrary = elements [V3, V4]
+
+instance Arbitrary PhyloSimilarity where
+  arbitrary = oneof [ WeightedLogJaccard <$> arbitrary <*> arbitrary
+                    , WeightedLogSim <$> arbitrary <*> arbitrary
+                    , Hamming <$> arbitrary <*> arbitrary
+                    ]
+
+instance Arbitrary SeaElevation where
+  arbitrary = oneof [ Constante <$> arbitrary <*> arbitrary
+                    , Adaptative <$> arbitrary
+                    , Evolving <$> arbitrary
+                    ]
+
+instance Arbitrary Synchrony where
+  arbitrary = oneof [ ByProximityThreshold <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
+                    , ByProximityDistribution <$> arbitrary <*> arbitrary
+                    ]
+
+instance Arbitrary SynchronyScope where
+  arbitrary = elements [SingleBranch, SiblingBranches, AllBranches]
+
+instance Arbitrary SynchronyStrategy where
+  arbitrary = elements [MergeRegularGroups, MergeAllGroups]
+
+instance Arbitrary Quality where
+  arbitrary = Quality <$> arbitrary <*> arbitrary
+
+instance Arbitrary TimeUnit where
+  arbitrary = oneof [ Epoch <$> arbitrary <*> arbitrary <*> arbitrary
+                    , Year <$> arbitrary <*> arbitrary <*> arbitrary
+                    , Month <$> arbitrary <*> arbitrary <*> arbitrary
+                    , Week <$> arbitrary <*> arbitrary <*> arbitrary
+                    , Day <$> arbitrary <*> arbitrary <*> arbitrary
+                    ]
+
+instance Arbitrary Cluster where
+  arbitrary = oneof [ Fis <$> arbitrary <*> arbitrary
+                    , MaxClique <$> arbitrary <*> arbitrary <*> arbitrary
+                    ]
+
+instance Arbitrary MaxCliqueFilter where
+  arbitrary = elements [ByThreshold, ByNeighbours]
+
+instance Arbitrary PhyloLabel where
+  arbitrary = oneof [ BranchLabel <$> arbitrary <*> arbitrary
+                    , GroupLabel <$> arbitrary <*> arbitrary
+                    ]
+
+instance Arbitrary Tagger where
+  arbitrary = elements [MostInclusive, MostEmergentInclusive, MostEmergentTfIdf]
+
+instance Arbitrary Sort where
+  arbitrary = oneof [ ByBirthDate <$> arbitrary
+                    , ByHierarchy <$> arbitrary]
+
+instance Arbitrary Order where
+  arbitrary = elements [Asc, Desc]
+
+instance Arbitrary Filter where
+  arbitrary = ByBranchSize <$> arbitrary

src/Gargantext/Core/Viz/Phylo/API.hs

@@ -26,8 +26,8 @@ import Data.Text qualified as T
 import Gargantext.API.Prelude
 import Gargantext.Core.Types (TODO(..))
 import Gargantext.Core.Types.Phylo (GraphData(..))
-import Gargantext.Core.Viz.LegacyPhylo
-import Gargantext.Core.Viz.Phylo (defaultConfig)
+import Gargantext.Core.Viz.LegacyPhylo hiding (Phylo(..))
+import Gargantext.Core.Viz.Phylo (PhyloConfig(..), defaultConfig, _phylo_param, _phyloParam_config)
 import Gargantext.Core.Viz.Phylo.API.Tools
 import Gargantext.Core.Viz.Phylo.Example (phyloCleopatre)
 import Gargantext.Core.Viz.Phylo.Legacy.LegacyMain
@@ -76,6 +76,7 @@ instance ToSchema Value where declareNamedSchema _ = declareNamedSchema (Proxy :
 data PhyloData = PhyloData { pd_corpusId :: NodeId
                            , pd_listId   :: NodeId
                            , pd_data     :: GraphData
+                           , pd_config   :: PhyloConfig
                            }
   deriving (Generic, Show, Eq)
 
@@ -85,6 +86,7 @@ instance ToJSON PhyloData where
       "pd_corpusId" .= toJSON pd_corpusId
     , "pd_listId"   .= toJSON pd_listId
     , "pd_data"     .= toJSON pd_data
+    , "pd_config"   .= toJSON pd_config
     ]
 
 instance FromJSON PhyloData where
@@ -92,10 +94,11 @@ instance FromJSON PhyloData where
     pd_corpusId <- o .: "pd_corpusId"
     pd_listId   <- o .: "pd_listId"
     pd_data     <- o .: "pd_data"
+    pd_config   <- o .: "pd_config"
     pure $ PhyloData{..}
 
 instance Arbitrary PhyloData where
-  arbitrary = PhyloData <$> arbitrary <*> arbitrary <*> arbitrary
+  arbitrary = PhyloData <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
 
 instance ToSchema PhyloData
 
@@ -131,20 +134,21 @@ getPhylo phyloId lId _level _minSizeBranch = do
   listId   <- case lId of
                 Nothing -> defaultList corpusId
                 Just ld -> pure ld
-  theData <- getPhyloDataJson phyloId
+  (gd, phyloConfig) <- getPhyloDataJson phyloId
   -- printDebug "getPhylo" theData
-  pure $ PhyloData corpusId listId theData
+  pure $ PhyloData corpusId listId gd phyloConfig
 
 
 
-getPhyloDataJson :: PhyloId -> GargNoServer GraphData
+getPhyloDataJson :: PhyloId -> GargNoServer (GraphData, PhyloConfig)
 getPhyloDataJson phyloId = do
   maybePhyloData <- getPhyloData phyloId
   let phyloData = fromMaybe phyloCleopatre maybePhyloData
+  let phyloConfig = _phyloParam_config $ _phylo_param phyloData
   phyloJson <- liftBase $ phylo2dot2json phyloData
   case parseEither parseJSON phyloJson of
     Left err -> panicTrace $ T.pack $ "[Gargantext.Core.Viz.Phylo.API] getPhyloDataJson: " <> err
-    Right gd -> pure gd
+    Right gd -> pure (gd, phyloConfig)
 
 
 -- getPhyloDataSVG phId _lId l msb  = do
@@ -194,6 +198,7 @@ putPhylo = undefined
 -- instance Arbitrary Phylo             where arbitrary     = elements [phylo]
 instance Arbitrary PhyloGroup        where arbitrary     = elements []
 -- instance Arbitrary PhyloView         where arbitrary     = elements [phyloView]
+-- instance Arbitrary PhyloConfig       where arbitrary     = elements []
 instance FromHttpApiData DisplayMode where parseUrlPiece = readTextData
 instance FromHttpApiData ExportMode  where parseUrlPiece = readTextData
 instance FromHttpApiData Filiation   where parseUrlPiece = readTextData

test-data/phylo/open_science.json

@@ -1314,5 +1314,73 @@
         "phyloSeaRiseSteps": "0.1"
     },
     "pd_listId": 185785,
-    "pd_corpusId": 185783
+    "pd_corpusId": 185783,
+    "pd_config": {
+        "clique": {
+            "_mcl_filter": "ByThreshold",
+            "_mcl_size": 5,
+            "_mcl_threshold": 1.0e-4,
+            "tag": "MaxClique"
+        },
+        "corpusParser": {
+            "_csv_limit": 150000,
+            "tag": "Csv"
+        },
+        "corpusPath": "corpus.csv",
+        "defaultMode": false,
+        "exportFilter": [
+            {
+                "_branch_size": 3
+            }
+        ],
+        "exportLabel": [
+            {
+                "_branch_labelSize": 2,
+                "_branch_labelTagger": "MostEmergentTfIdf",
+                "tag": "BranchLabel"
+            },
+            {
+                "_group_labelSize": 2,
+                "_group_labelTagger": "MostEmergentInclusive",
+                "tag": "GroupLabel"
+            }
+        ],
+        "exportSort": {
+            "_sort_order": "Desc",
+            "tag": "ByHierarchy"
+        },
+        "findAncestors": true,
+        "listParser": "V4",
+        "listPath": "list.csv",
+        "outputPath": "data/",
+        "phyloName": "Phylo Name",
+        "phyloQuality": {
+            "_qua_granularity": 0.8,
+            "_qua_minBranch": 3
+        },
+        "phyloScale": 2,
+        "phyloSynchrony": {
+            "_bpt_scope": "AllBranches",
+            "_bpt_sensibility": 0,
+            "_bpt_strategy": "MergeAllGroups",
+            "_bpt_threshold": 0.5,
+            "tag": "ByProximityThreshold"
+        },
+        "seaElevation": {
+            "_cons_gap": 0.1,
+            "_cons_start": 0.1,
+            "tag": "Constante"
+        },
+        "similarity": {
+            "_wlj_minSharedNgrams": 2,
+            "_wlj_sensibility": 0.5,
+            "tag": "WeightedLogJaccard"
+        },
+        "timeUnit": {
+            "_year_matchingFrame": 5,
+            "_year_period": 3,
+            "_year_step": 1,
+            "tag": "Year"
+        }
+    }
 }