build: add support for `haskell.nix`

.gitignore

@@ -45,3 +45,7 @@ devops/docker/js-cache
 
 cabal.project.local
 gargantext_profile_out.dot
+
+# Nix
+!flake.lock
+nix/cache/haskell-nix/gc-roots/

cabal.project.freeze

@@ -255,7 +255,7 @@ constraints: any.Cabal ==3.8.1.0,
              any.hlint ==3.5,
              hlint +ghc-lib +gpl -hsyaml +threaded,
              any.hmatrix ==0.20.2,
-             hmatrix -disable-default-paths -no-random_r -openblas,
+             hmatrix -disable-default-paths -no-random_r +openblas,
              any.hmatrix-gsl-stats ==0.4.1.8,
              hmatrix-gsl-stats -onlygsl,
              any.hourglass ==0.2.12,

nix/overlays/haskell-nix/gargantext.nix

+{ lib, ... }:
+{
+  modules = [
+    ({ pkgs, ... }: {
+      # Start StanfordCoreNLPServer to pass the Hspec tests
+      # Use `nix run` instead of `inputs.coreNLP` directly
+      # to avoid downloading it if not running garg-test-hspec
+      packages.gargantext.components.tests.garg-test-hspec.preCheck = ''
+        export GARGANTEXT_CORENLP_SERVER="${pkgs.nixFlakes}/bin/nix run .#coreNLP"
+      '';
+
+      packages.gargantext.components.tests.garg-test-hspec.testFlags = [
+        # FIXME:
+        # test/Test/Database/Operations.hs:64:7:
+        # 1) Database.Read/Writes, Corpus creation, Can add documents to a Corpus
+        #      uncaught exception: HttpException
+        #      HttpExceptionRequest Request {
+        #        host                 = "localhost"
+        #        port                 = 9000
+        #        secure               = False
+        #        requestHeaders       = [("Accept","application/json")]
+        #        path                 = "/"
+        #        queryString          = "?properties=%7B%22annotators%22:%22tokenize,ssplit,pos,ner%22,%22outputFormat%22:%22json%22%7D"
+        #        method               = "POST"
+        #        proxy                = Nothing
+        #        rawBody              = False
+        #        redirectCount        = 10
+        #        responseTimeout      = ResponseTimeoutDefault
+        #        requestVersion       = HTTP/1.1
+        #        proxySecureMode      = ProxySecureWithConnect
+        #      }
+        #       (ConnectionFailure Network.Socket.connect: <socket: 18>: does not exist (Connection refused))
+        "--match" "'!/Database/Read/Writes/Corpus creation/Can add documents to a Corpus/'"
+      ];
+
+      # Beware those failures are non-deterministic.
+      # For reproducing it may help to use a project's variant like "coverage":
+      #     $ nix -L build .#project.projectVariants.coverage.hsPkgs.gargantext.components.tests.garg-test-tasty
+      #     $ for i in {1..100}; do result/bin/garg-test-tasty --hide-successes || break; done
+      # Note that testFlags are only applied on checks.*, not on tests.*
+      packages.gargantext.components.tests.garg-test-tasty = {
+        build-tools = [
+          pkgs.graphviz
+        ];
+        testFlags = [
+          "--pattern"
+          ("'" + lib.concatStringsSep " && " [
+            # FAIL: (non-deterministic)
+            # Graph Clustering
+            #     Cross
+            #       Partition test:                                                 FAIL (0.14s)
+            #         uncaught exception: ErrorCall
+            #         *** Internal error in package accelerate ***
+            #         *** Please submit a bug report at https://github.com/AccelerateHS/accelerate/issues
+            #         Encountered a duplicate 'Tag'
+            #           81: Tag 1, 77: Tag 1, 79: Tag 1, 84: Tag 1
+            #         CallStack (from HasCallStack):
+            #           internalError: Data.Array.Accelerate.Trafo.Sharing:2274:19
+            #           buildInitialEnvExp: Data.Array.Accelerate.Trafo.Sharing:2702:17
+            #           determineScopesExp: Data.Array.Accelerate.Trafo.Sharing:2608:17
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2636:26
+            #           scopesFun1: Data.Array.Accelerate.Trafo.Sharing:2429:60
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2490:33
+            #           travF2MEA: Data.Array.Accelerate.Trafo.Sharing:2406:38
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2464:33
+            #           travEA: Data.Array.Accelerate.Trafo.Sharing:2397:38
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2509:34
+            #           travF2A2: Data.Array.Accelerate.Trafo.Sharing:2405:38
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2490:33
+            #           travF2MEA: Data.Array.Accelerate.Trafo.Sharing:2406:38
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2490:33
+            #           travF2MEA: Data.Array.Accelerate.Trafo.Sharing:2406:38
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2331:46
+            #           determineScopesSharingAcc: Data.Array.Accelerate.Trafo.Sharing:2718:17
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2818:33
+            #           travAE: Data.Array.Accelerate.Trafo.Sharing:2767:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2790:32
+            #           travE2: Data.Array.Accelerate.Trafo.Sharing:2749:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2780:30
+            #           travE1: Data.Array.Accelerate.Trafo.Sharing:2766:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2789:32
+            #           travE2: Data.Array.Accelerate.Trafo.Sharing:2749:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2780:30
+            #           travE1: Data.Array.Accelerate.Trafo.Sharing:2751:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2800:32
+            #           travE3: Data.Array.Accelerate.Trafo.Sharing:2760:36
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2712:56
+            #           determineScopesSharingExp: Data.Array.Accelerate.Trafo.Sharing:2696:60
+            #           determineScopesExp: Data.Array.Accelerate.Trafo.Sharing:2608:17
+            #           scopesExp: Data.Array.Accelerate.Trafo.Sharing:2636:26
+            #           scopesFun1: Data.Array.Accelerate.Trafo.Sharing:2400:60
+            #           scopesAcc: Data.Array.Accelerate.Trafo.Sharing:2331:46
+            #           determineScopesSharingAcc: Data.Array.Accelerate.Trafo.Sharing:2317:37
+            #           determineScopesAcc: Data.Array.Accelerate.Trafo.Sharing:3101:5
+            #           recoverSharingAcc: Data.Array.Accelerate.Trafo.Sharing:240:34
+            #           convertOpenAcc: Data.Array.Accelerate.Trafo.Sharing:162:35
+            #           convertAccWith: Data.Array.Accelerate.Trafo:71:37
+            "!/Partition test/"
+
+            # FIXME: FAIL (non-deterministic)
+            #     Starting 1 job runners.
+            #     Starting 1 job runners.
+            #     expected: [Nothing,Just 10,Just 5]
+            #      but got: [Nothing,Nothing,Nothing]
+            "!/can fetch the latest job status/"
+
+            # FIXME: FAIL (non-deterministic)
+            # expected: [Just 100]
+            #  but got: [Nothing]
+            # expected: [Just 50]
+            #  but got: [Nothing]
+            "!/can spin two separate jobs and track their status separately/"
+
+            # FIXME: FAIL
+            # -- | Build the coocurency matrix for 62 unit of time
+            # -- | Group 22 docs by 62 unit of time
+            # Exception:
+            # *** Internal error in package accelerate ***
+            # *** Please submit a bug report at https://github.com/AccelerateHS/accelerate/issues
+            # Encountered a duplicate 'Tag'
+            #   208: Tag 1, 212: Tag 1, 210: Tag 1, 205: Tag 1
+            "!/phyloCleopatre returns expected data/"
+
+            # FIXME: FAIL
+            # huge error diff
+            "!/phylo2dot2json.is deterministic/"
+
+            # FIXME: FAIL
+            # Exception: /nix/store/yx2cnkj4hq6zk867nkvpffvyd9qy2slp-gargantext-test-garg-test-tasty-0.0.7.1.1-data/share/ghc-9.4.7/x86_64-linux-ghc-9.4.7/gargantext-0.0.7.1.1/test-data/phylo/GarganText_NgramsList-187482.csv: openBinaryFile: does not exist (No such file or directory)
+            #"!/toPhylo.is deterministic/"
+
+            # FIXME: recursively call cabal
+            "!/test vector works/"
+
+            # FIXME: FAIL (deterministic when coverage is enabled):
+            # nix -L build .#project.projectVariants.haskell-nix-ghc.flake"'".ciJobs.coverage.gargantext
+            #
+            # check if similarities optimizations are well implemented
+            #     uncaught exception: ErrorCall
+            #     *** Internal error in package accelerate ***
+            #     *** Please submit a bug report at https://github.com/AccelerateHS/accelerate/issues
+            #     Encountered a duplicate 'Tag'
+            #       259: Tag 1, 261: Tag 1
+            "!/Conditional/"
+          ] + "'")
+        ];
+      };
+    })
+  ];
+}

nix/overlays/haskell-nix/weeder.nix

+{ config, pkgs, lib, haskellLib, ... }:
+let
+  cfg = config.weeder;
+  toml = pkgs.formats.toml {};
+in
+{
+  options.weeder = {
+    analysis = lib.mkOption {
+      type = lib.types.package;
+      default = null;
+      internal = true;
+    };
+    packages = lib.mkOption {
+      type = lib.types.unspecified;
+      default = haskellLib.selectProjectPackages;
+    };
+    settings = lib.mkOption {
+      type = toml.type;
+      default = {};
+    };
+  };
+
+  config = {
+    modules = [
+      {
+        # Enable writeHieFiles by default.
+        # See https://github.com/input-output-hk/haskell.nix/issues/298#issuecomment-767936405
+        # for what's going on in this trick.
+        options.packages = lib.mkOption {
+          type = lib.types.attrsOf (
+            lib.types.submodule (
+              { config, ... }: {
+                config = lib.mkIf
+                  ( (cfg.packages { ${config.package.identifier.name} = config.package; })
+                  ? ${config.package.identifier.name} ) {
+                  writeHieFiles = lib.mkForce true;
+                };
+              }
+            )
+          );
+        };
+      }
+    ];
+
+    weeder.analysis =
+      pkgs.runCommand "weeder-${config.name}" {
+        buildInputs = [
+          pkgs.glibcLocales
+          pkgs.haskell.packages.${config.compiler-nix-name}.weeder
+        ];
+        allHieFiles = pkgs.linkFarm "allHieFiles" (
+          lib.concatMap (package:
+            let
+              lib-hies = lib.optional (package.components ? library) {
+                name = "${package.identifier.name}-library";
+                path = package.components.library.hie;
+              };
+
+              exe-hies = lib.concatMap (exe:
+                lib.optional (package.components.exes.${exe} ? hie) {
+                  name = "${package.identifier.name}-exe-${exe}";
+                  path = package.components.exes.${exe}.hie;
+                })
+                (lib.attrNames package.components.exes);
+
+              test-hies = lib.concatMap (test-name:
+                let
+                  test = package.components.tests.${test-name};
+                  is-doctest =
+                    # doctest tests build _all_ components of a package.
+                    # The GHC id of these packages will be different,
+                    # which means that when we run weeder, all this code
+                    # will be uncalled.  These are false positives, so
+                    # we don't include hie files from anything that
+                    # depends on `doctest`.
+                    lib.any (x: x.identifier.name or "" == "doctest") test.config.depends;
+                in
+                lib.optional (!is-doctest) {
+                  name = "${package.identifier.name}-test-${test-name}";
+                  path = test.hie;
+                })
+                (lib.attrNames package.components.tests);
+            in
+            lib-hies ++ exe-hies ++ test-hies
+          )
+          (builtins.attrValues (cfg.packages config.hsPkgs))
+        );
+      } ''
+        export LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
+        cd $allHieFiles
+        weeder >$out --no-default-fields --config ${toml.generate "weeder.toml" cfg.settings} ||
+        test $? = 228 # One or more weeds found
+      '';
+  };
+}

nix/overlays/nixpkgs/graphviz.nix

+finalPkgs: previousPkgs: {
+  # FIXME: update to a version >= 10.0.1
+  # to include 2a265497 from https://gitlab.com/graphviz/graphviz/-/merge_requests/3487
+  graphviz = previousPkgs.graphviz.overrideAttrs (finalAttrs: previousAttrs: {
+    patches = (previousAttrs.patches or [ ]) ++ [
+      # Increase the YY_BUF_SIZE
+      # See https://gitlab.iscpif.fr/gargantext/haskell-gargantext/issues/290#note_9015
+      graphviz/graphviz-yy-buf-size.diff
+    ];
+  });
+}

nix/overlays/nixpkgs/graphviz/graphviz-yy-buf-size.diff

+--- a/lib/cgraph/scan.l
++++ b/lib/cgraph/scan.l
+@@ -32,7 +32,7 @@
+ #include <stdbool.h>
+ #include <stddef.h>
+ #include <string.h>
+-// #define YY_BUF_SIZE 128000
++#define YY_BUF_SIZE 128000
+ #define GRAPH_EOF_TOKEN		'@'		/* lex class must be defined below */
+ 	/* this is a workaround for linux flex */
+ static int line_num = 1;
\ No newline at end of file

nix/overlays/nixpkgs/igraph.nix

+finalPkgs: previousPkgs: {
+  # WARNING: the nixpkgs pinned brings igraph-0.10.7 not igraph-0.10.4
+  # as expected by haskell-igraph-0.10.4
+  # Note that igraph uses the lapack and blas alternative selector packages
+  # which default to using openblas.
+  igraph = (previousPkgs.igraph.override {
+    # fop pulls openjdk, scipy, blas… just to build PDF docs,
+    fop = null;
+  }).overrideAttrs (previousAttrs: {
+    cmakeFlags = previousAttrs.cmakeFlags ++ [
+      # Disable Open Multi-Processing, to let GHC handle the kernel threads.
+      "-DIGRAPH_OPENMP_SUPPORT=OFF"
+    ];
+  });
+  # Dependency of igraph
+  plfit = previousPkgs.plfit.overrideAttrs (previousAttrs: {
+    cmakeFlags = previousAttrs.cmakeFlags ++ [
+      # Disable Open Multi-Processing, to let GHC handle the kernel threads.
+      "-DPLFIT_USE_OPENMP=OFF"
+    ];
+  });
+}

nix/overlays/nixpkgs/openblas.nix

+finalPkgs: previousPkgs: {
+  # Dependency of igraph and hmatrix
+  openblas = previousPkgs.openblas.override {
+    # Let GHC handle the kernel threads.
+    # See also https://github.com/OpenMathLib/OpenBLAS/issues/2543
+    singleThreaded = true;
+  };
+}

nix/patches/accelerate-arithmetic/0001-remove-test-using-removed-realBandedGramian.patch

+From 64cd06549f19ad447f18a45d872b59b2b2db9135 Mon Sep 17 00:00:00 2001
+From: Julien Moutinho <julm+gargantext@sourcephile.fr>
+Date: Wed, 28 Jun 2023 23:07:32 +0200
+Subject: [PATCH] remove test using removed `realBandedGramian`
+
+---
+ test/Test.hs                                         | 1 -
+ test/Test/Data/Array/Accelerate/Arithmetic/Sparse.hs | 7 -------
+ 2 files changed, 8 deletions(-)
+
+diff --git a/test/Test.hs b/test/Test.hs
+index ebc140f..ea736ee 100644
+--- a/test/Test.hs
++++ b/test/Test.hs
+@@ -10,7 +10,6 @@ import Test.QuickCheck (quickCheck)
+ test :: IO ()
+ test = mapM_ (\(msg,act) -> putStr (msg++": ") >> act) $
+    ("sparseMatrix", quickCheck (\(Mod.Blind x) -> Sparse.multiplication x)) :
+-   ("bandedGramian", quickCheck (\(Mod.Blind x) -> Sparse.bandedGramian x)) :
+    ("flattenMatrix", quickCheck (\(Mod.Blind x) -> LinAlg.flattenMatrix x)) :
+    ("restoreMatrix", quickCheck (\(Mod.Blind x) -> LinAlg.restoreMatrix x)) :
+    ("flattenRestoreMatrix", quickCheck (\(Mod.Blind x) -> LinAlg.flattenRestoreMatrix x)) :
+diff --git a/test/Test/Data/Array/Accelerate/Arithmetic/Sparse.hs b/test/Test/Data/Array/Accelerate/Arithmetic/Sparse.hs
+index da497dd..15e7e51 100644
+--- a/test/Test/Data/Array/Accelerate/Arithmetic/Sparse.hs
++++ b/test/Test/Data/Array/Accelerate/Arithmetic/Sparse.hs
+@@ -73,10 +73,3 @@ instance (QC.Arbitrary a, A.Elt a) => QC.Arbitrary (BandGramian a) where
+       return $
+          BandGramian (A.lift width)
+             (Sparse.Rows (A.lift cols) (A.use m))
+-
+-
+-bandedGramian :: BandGramian Word32 -> Bool
+-bandedGramian (BandGramian width m) =
+-   Sparse.multiplyColumnsRows (Sparse.transposeRows m) m
+-   =!=
+-   BandMatrix.flattenSymmetric (Sparse.realBandedGramian width m)
+-- 
+2.40.1
+

test/drivers/hspec/Main.hs

@@ -7,9 +7,11 @@ import Gargantext.Prelude hiding (isInfixOf)
 import Control.Monad
 import Data.Text (isInfixOf)
 import Shelly hiding (FilePath)
+import System.Environment (lookupEnv)
 import System.IO
 import System.Process
 import Test.Hspec
+import qualified Data.List as List
 import qualified Data.Text as T
 import qualified Test.API                     as API
 import qualified Test.Database.Operations     as DB
@@ -18,15 +20,20 @@ import qualified Test.Database.Operations     as DB
 startCoreNLPServer :: IO ProcessHandle
 startCoreNLPServer = do
   devNull <- openFile "/dev/null" WriteMode
-  let p = proc "./startServer.sh" []
-  (_, _, _, hdl) <- (createProcess $ p { cwd = Just "devops/coreNLP/stanford-corenlp-current"
+  -- Make the CoreNLP server configurable by an envvar, eg. set by nix develop
+  coreNLPEnv <- lookupEnv "GARGANTEXT_CORENLP_SERVER"
+  let (procName, procArgs) = case coreNLPEnv of
+                               Just str | name:args <- List.words str -> (name,args)
+                               _ -> ("./startServer.sh", [])
+  let p = proc procName procArgs
+  (_, _, _, hdl) <- (createProcess $ p { cwd = maybe (Just "devops/coreNLP/stanford-corenlp-current") (const Nothing) coreNLPEnv
                     , delegate_ctlc = True
                     , create_group = True
                     , std_out = UseHandle devNull
                     , std_err = UseHandle devNull
                     }) `catch` \e -> case e of
                                            _ | True <- "does not exist" `isInfixOf` (T.pack . show @SomeException $ e)
-                                             -> fail $ "Cannot execute the 'startServer.sh' script. If this is the " <>
+                                             -> fail $ "Cannot execute "<>procName<>". If this is the " <>
                                                        "first time you are running the tests, you have to run " <>
                                                        "cd devops/coreNLP && ./build.sh first. You have to run it only once, " <>
                                                        "and then you are good to go for the time being."