Merge branch 'dev-getting-started-readme' of...

Merge branch 'dev-getting-started-readme' of ssh://gitlab.iscpif.fr:20022/gargantext/haskell-gargantext into dev-merge

README.md

@@ -18,38 +18,83 @@ progress. Please report and improve this documentation if you encounter
 issues.
 
 ### Build Core Code
+
 #### Docker
-  curl -sSL https://gitlab.iscpif.fr/gargantext/haskell-gargantext/raw/master/devops/docker-install | sh
+
+``` sh
+curl -sSL https://gitlab.iscpif.fr/gargantext/haskell-gargantext/raw/master/devops/docker/docker-install | sh
+```
 
 #### Debian
-  curl -sSL https://gitlab.iscpif.fr/gargantext/haskell-gargantext/raw/master/devops/debian-install | sh
+
+``` sh
+curl -sSL https://gitlab.iscpif.fr/gargantext/haskell-gargantext/raw/master/devops/debian/install | sh
+```
 
 ### Add dependencies
 
 1. CoreNLP is needed (EN and FR); This dependency will not be needed
    soon.
-  - wget https://dl.gargantext.org/coreNLP.tar.bz2
-  - tar xvjf coreNLP.tar.bz2
-  - ./startServer.sh
+
+``` sh
+./devops/install-corenlp
+```
 
 2. Louvain C++ needed to draw the socio-semantic graphs
-  - git clone https://gitlab.iscpif.fr/gargantext/clustering-louvain-cplusplus.git
-  - cd clustering-louvain-cplusplus
-  - ./install
+
+NOTE: This is already added in the Docker build.
+
+``` sh
+git clone https://gitlab.iscpif.fr/gargantext/clustering-louvain-cplusplus.git
+cd clustering-louvain-cplusplus
+./install
+```
 
 ### Initialization
 
-Users has to be created first (user1 is created as instance):
+#### Docker
+
+Run PostgreSQL first:
+
+``` sh
+cd devops/docker
+docker-compose up
+```
 
-- stack install
-- ~/.local/bin/gargantext-init "gargantext.ini"
+Initialization schema should be loaded automatically (from `devops/postgres/schema.sql`).
+
+#### Gargantext
+
+Users have to be created first (`user1` is created as instance):
+
+``` sh
+stack install
+~/.local/bin/gargantext-init "gargantext.ini"
+```
+
+For Docker env, run:
+
+``` sh
+stack --docker run gargantext-init -- gargantext.ini
+```
+
+### Importing data
+
+You can import some data with:
+``` sh
+docker run --rm -it -p 9000:9000 cgenie/corenlp-garg
+stack exec gargantext-import -- "corpusCsvHal" "user1" "IMT3" gargantext.ini 10000 ./1000.csv
+```
 
 ## Use Cases
 
 ### Multi-User with Graphical User Interface (Server Mode)
 
+``` sh
 ~/.local/bin/stack --docker exec gargantext-server -- --ini "gargantext.ini" --run Prod
-Then you can log in with user1:1resu
+```
+
+Then you can log in with `user1:1resu`.
 
 
 ### Command Line Mode tools

build-shell.nix

+{ghc}:
+with (import <nixpkgs> {});
+
+haskell.lib.buildStackProject {
+  inherit ghc;
+  name = "gargantext";
+  buildInputs = [
+    docker-compose
+    blas
+    bzip2
+    #gfortran
+    gfortran.cc.lib
+    glibc
+    gmp
+    gsl
+    igraph
+    liblapack
+    pcre
+    postgresql
+    #stack
+    xz
+    zlib
+  ];
+}

coreNLP/Dockerfile

+FROM openjdk
+
+ADD home/debian/CoreNLP /CoreNLP
+WORKDIR /CoreNLP
+
+CMD ./startServer.sh

default.nix

+with (import <nixpkgs> {});
+stdenv.mkDerivation rec {
+  name = "env";
+  env = buildEnv {
+    name = name;
+    paths = buildInputs;
+  };
+  buildInputs = [
+    docker-compose
+    #glibc
+    #gmp
+    #gsl
+    #igraph
+    #pcre
+    #postgresql
+    #stack
+    #xz
+  ];
+  builder = builtins.toFile "builder.sh" ''
+    source $stdenv/setup
+    touch $out
+  '';
+}

devops/docker/Dockerfile

-from fpco/stack-build:lts-14.6
+from fpco/stack-build:lts-14.22
 
 RUN apt-get update && \
     apt-get install -y git libigraph0-dev && \

devops/docker/docker-compose.yaml

@@ -10,9 +10,15 @@ services:
       POSTGRES_PASSWORD: C8kdcUrAQy66U
       POSTGRES_DB: gargandbV5
     volumes:
-      - pgdata:/var/lib/postgresql/data
+      - garg-pgdata:/var/lib/postgresql/data
       - ../:/gargantext
       - ../dbs:/dbs
+      - ../postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
+
+  corenlp:
+    image: 'cgenie/corenlp-garg'
+    ports:
+      - 9000:9000
 
 volumes:
-  pgdata:
+  garg-pgdata:

devops/docker/docker-import

+#!/bin/bash
+
 if [ "$#" -lt 3 ]; then
   echo "Usage: $0 <name> <path> <limit>"
   exit 1

devops/docker/docker-install

@@ -37,27 +37,27 @@ exec sudo -E /usr/bin/docker \"\$@\"" >> $DOCKERBIN
 fi
 ########################################################################
 
-curl -sSL https://get.haskellstack.org/ | sh
-stack update
-stack upgrade
-
+if stack --version;
+then
+  echo "Haskell stack installed, ok"
+else
+  curl -sSL https://get.haskellstack.org/ | sh
+  stack update
+  stack upgrade
+fi
 
 git clone https://gitlab.iscpif.fr/gargantext/haskell-gargantext.git
 
 cd haskell-gargantext
 git clone https://gitlab.iscpif.fr/gargantext/purescript-gargantext
 
-mkdir deps
-cd deps
-git clone https://gitlab.iscpif.fr/gargantext/clustering-louvain.git
-git clone https://github.com/np/servant-job.git
-git clone https://github.com/np/patches-map
-git clone https://gitlab.com/npouillard/patches-class.git
-git clone https://github.com/delanoe/haskell-opaleye
-git clone -b next --single-branch https://github.com/delanoe/hsparql
-cd ..
+../install-deps $(pwd)
+
+pushd devops/docker
+docker build --pull -t fpco/stack-build:lts-14.22-garg .
+popd
 
-stack docker pull
+#stack docker pull
 stack --docker setup
 stack --docker build
 stack --docker install

devops/docker/docker-postgres

+#!/bin/bash
+
 set -eu
+
 docker stop dbgarg || :
 docker rm --volumes dbgarg || :
 export PGPASSWORD="$(grep DB_PASS gargantext.ini | \awk '{print $3}')"

devops/docker/docker-prod

+#!/bin/bash
+
 stack --docker exec gargantext-server -- --run Prod --ini gargantext.ini

devops/install-corenlp

+#!/bin/bash
+
+if [ ! -d coreNLP ]; then
+    mkdir -v coreNLP
+fi
+
+pushd coreNLP
+
+wget https://dl.gargantext.org/coreNLP.tar.bz2
+tar xvjf coreNLP.tar.bz2
+pushd home/debian/CoreNLP
+./startServer.sh

devops/install-deps

+#!/bin/bash
+
+DIR=$1
+DEPS_DIR=${DIR}/deps
+
+function clone_or_update() {
+    REPO=$1
+    echo "Checking repo ${REPO}"
+    # strip only dir name from URL
+    DIR=${REPO##*/}
+    # strip the remaining '.git' suffix
+    RAW_DIR=${DIR%.*}
+    if [ -d "${RAW_DIR}" ]; then
+        pushd ${RAW_DIR}
+        git pull
+        popd ..
+    else
+        git clone "$@"
+    fi
+}
+
+if [ ! -d "${DEPS_DIR}" ]; then
+    mkdir ${DIR}/deps
+fi
+
+cd ${DIR}/deps
+
+clone_or_update https://gitlab.iscpif.fr/gargantext/clustering-louvain.git
+clone_or_update https://github.com/np/servant-job.git
+clone_or_update https://github.com/np/patches-map
+clone_or_update https://gitlab.com/npouillard/patches-class.git
+clone_or_update https://github.com/delanoe/haskell-opaleye
+clone_or_update https://github.com/delanoe/hsparql -b next --single-branch

stack.yaml

-resolver: lts-14.16
+resolver: lts-14.22
 flags: {}
 extra-package-dbs: []
 packages:
@@ -7,7 +7,13 @@ packages:
 
 docker:
   enable: false
-  repo: 'fpco/stack-build:lts-14.6-garg'
+  repo: 'fpco/stack-build:lts-14.22-garg'
+
+nix:
+  enable: false
+  #packages: [gmp]
+  add-gc-roots: true
+  shell-file: build-shell.nix
 
 allow-newer: true
 extra-deps: