merge done

.gitignore

@@ -23,6 +23,8 @@ doc
 deps
 _darcs
 *.pdf
+*.sql
+*.ini
 
 # Runtime
 

README.md

@@ -65,6 +65,17 @@ Initialization schema should be loaded automatically (from `devops/postgres/sche
 
 #### Gargantext
 
+##### Fix the passwords
+
+Change the passwords in gargantext.ini_toModify then move it:
+
+mv gargantext.ini_toModify gargantext.ini
+
+(.gitignore avoids adding this file to the repository by mistake)
+
+
+##### Run Gargantext
+
 Users have to be created first (`user1` is created as instance):
 
 ``` sh
@@ -72,7 +83,14 @@ stack install
 ~/.local/bin/gargantext-init "gargantext.ini"
 ```
 
-For Docker env, run:
+For Docker env, first create the appropriate image:
+
+``` sh
+cd devops/docker
+docker build -t fpco/stack-build:lts-14.27-garg .
+```
+
+then run:
 
 ``` sh
 stack --docker run gargantext-init -- gargantext.ini

bin/build

+#!/bin/bash
+
+stack build # --profile # --test # --haddock

bin/gargantext-adaptative-phylo/Main.hs

@@ -10,13 +10,8 @@ Portability : POSIX
 Adaptative Phylo binaries
  -}
 
-{-# LANGUAGE DataKinds          #-}
-{-# LANGUAGE DeriveGeneric      #-}
-{-# LANGUAGE FlexibleInstances  #-}
-{-# LANGUAGE NoImplicitPrelude  #-}
 {-# LANGUAGE StandaloneDeriving #-}
 {-# LANGUAGE TypeOperators      #-}
-{-# LANGUAGE OverloadedStrings  #-}
 {-# LANGUAGE Strict             #-}
 
 module Main where
@@ -29,11 +24,11 @@ import Data.String (String)
 import Data.Text  (Text, unwords, unpack)
 
 import Gargantext.Prelude
-import Gargantext.Database.Types.Node (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
 import Gargantext.Text.Context (TermList)
 import Gargantext.Text.Corpus.Parsers.CSV (csv_title, csv_abstract, csv_publication_year)
 import Gargantext.Text.Corpus.Parsers (FileFormat(..),parseFile)
-import Gargantext.Text.List.CSV (csvGraphTermList)
+import Gargantext.Text.List.CSV (csvMapTermList)
 import Gargantext.Text.Terms.WithList (Patterns, buildPatterns, extractTermsWithList)
 import Gargantext.Viz.AdaptativePhylo
 import Gargantext.Viz.Phylo.PhyloMaker  (toPhylo)
@@ -149,7 +144,7 @@ main = do
         Right config -> do
 
             printIOMsg "Parse the corpus"
-            mapList <- csvGraphTermList (listPath config)
+            mapList <- csvMapTermList (listPath config)
             corpus  <- fileToDocs (corpusParser config) (corpusPath config) mapList
             printIOComment (show (length corpus) <> " parsed docs from the corpus")
 
@@ -185,4 +180,4 @@ main = do
                       <> "-sens_" <> sensibility
                       <> ".dot"
 
-            dotToFile output dot
+            dotToFile output dot
\ No newline at end of file

bin/gargantext-cli/CleanCsvCorpus.hs

@@ -11,7 +11,6 @@ Given a Gargantext CSV File and its Query This script cleans and
 compress the contexts around the main terms of the query.
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module CleanCsvCorpus  where
 

bin/gargantext-cli/Main.hs

@@ -11,11 +11,6 @@ Main specifications to index a corpus with a term list
 
  -}
 
-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StandaloneDeriving #-}
 {-# LANGUAGE TypeOperators     #-}
 {-# LANGUAGE Strict             #-}
@@ -58,7 +53,7 @@ import Gargantext.Text.Terms
 import Gargantext.Text.Context
 import Gargantext.Text.Terms.WithList
 import Gargantext.Text.Corpus.Parsers.CSV (readFile, csv_title, csv_abstract, csv_publication_year)
-import Gargantext.Text.List.CSV (csvGraphTermList)
+import Gargantext.Text.List.CSV (csvMapTermList)
 import Gargantext.Text.Terms (terms)
 import Gargantext.Text.Metrics.Count (coocOnContexts, Coocs)
 
@@ -108,7 +103,7 @@ main = do
                            <$> readFile corpusFile
 
   -- termListMap :: [Text]
-  termList <- csvGraphTermList termListFile
+  termList <- csvMapTermList termListFile
 
   putStrLn $ show $ length termList
 

bin/gargantext-import/Main.hs

@@ -11,33 +11,28 @@ Import a corpus binary.
 
  -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE FlexibleContexts  #-}
-{-# LANGUAGE RankNTypes        #-}
 {-# LANGUAGE Strict            #-}
 
 module Main where
 
-import Data.Either
-import Prelude (read)
 import Control.Exception (finally)
+import Data.Either
+import Data.Text (Text)
+import Gargantext.API.Node () -- instances
+import Gargantext.API.Admin.Settings (withDevEnv, runCmdDev, DevEnv)
+import Gargantext.API.Prelude (GargError)
+import Gargantext.Core (Lang(..))
+import Gargantext.Core.Types.Individu (User(..))
+import Gargantext.Database.Action.Flow (FlowCmdM, flowCorpusFile, flowAnnuaire, TermType(..))
+import Gargantext.Database.Query.Table.User (insertUsersDemo)
+import Gargantext.Database.Admin.Types.Hyperdata (toHyperdataDocument)
+import Gargantext.Database.Admin.Types.Node (CorpusId)
+import Gargantext.Database.Prelude (Cmd)
 import Gargantext.Prelude
-import Gargantext.Database.Flow (FlowCmdM, flowCorpusFile, flowAnnuaire)
 import Gargantext.Text.Corpus.Parsers (FileFormat(..))
-import Gargantext.Database.Utils (Cmd, )
-import Gargantext.Database.Types.Node (CorpusId, toHyperdataDocument)
-import Gargantext.Database.Schema.User (insertUsersDemo)
-import Gargantext.Text.Terms (TermType(..))
-import Gargantext.Core (Lang(..))
-import Gargantext.API.Types (GargError)
-import Gargantext.API.Node () -- instances
-import Gargantext.API.Settings (withDevEnv, runCmdDev, DevEnv)
+import Prelude (read)
 import System.Environment (getArgs)
---import Gargantext.Text.Corpus.Parsers.GrandDebat (readFile, GrandDebatReference(..))
-import Data.Text (Text)
 import qualified Data.Text as Text
-import Control.Monad.IO.Class (liftIO)
 
 main :: IO ()
 main = do
@@ -53,14 +48,13 @@ main = do
     tt = (Multi EN)
     format = CsvGargV3 -- CsvHal --WOS
     corpus :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    corpus = flowCorpusFile (cs user) (Left (cs name :: Text)) (read limit :: Int) tt  format corpusPath
+    corpus = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt  format corpusPath
 
     corpusCsvHal :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    corpusCsvHal = flowCorpusFile (cs user) (Left (cs name :: Text)) (read limit :: Int) tt CsvHal corpusPath
+    corpusCsvHal = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt CsvHal corpusPath
 
     annuaire :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    annuaire = flowAnnuaire (cs user) (Left "Annuaire") (Multi EN) corpusPath
-
+    annuaire = flowAnnuaire (UserName $ cs user) (Left "Annuaire") (Multi EN) corpusPath
 
   {-
   let debatCorpus :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
@@ -86,7 +80,6 @@ main = do
           then runCmdDev env corpusCsvHal
           else pure 0 --(cs "false")
  
-
     _ <- if fun == "annuaire"
             then runCmdDev env annuaire
             else pure 0

bin/gargantext-init/Main.hs

@@ -11,28 +11,28 @@ Import a corpus binary.
 
  -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE FlexibleContexts  #-}
-{-# LANGUAGE RankNTypes        #-}
 {-# LANGUAGE Strict            #-}
 
 module Main where
 
 import Data.Either (Either(..))
 import Data.Maybe (Maybe(..))
-import System.Environment (getArgs)
-import Gargantext.Prelude
-import Gargantext.Database.Flow (getOrMkRoot, getOrMk_RootWithCorpus)
-import Gargantext.Database.Schema.Node (getOrMkList)
-import Gargantext.Database.Utils (Cmd, )
-import Gargantext.Database.Types.Node (CorpusId, RootId, HyperdataCorpus, ListId)
-import Gargantext.Database.Schema.User (insertUsersDemo, UserId)
-import Gargantext.API.Types (GargError)
+import Gargantext.API.Admin.Settings (withDevEnv, runCmdDev)
+import Gargantext.API.Prelude (GargError)
 import Gargantext.API.Node () -- instances
-import Gargantext.API.Settings (withDevEnv, runCmdDev)
-import Gargantext.Database.Config (userMaster, corpusMasterName)
-import Gargantext.Database.Init (initTriggers)
+import Gargantext.Core.Types.Individu (User(..))
+import Gargantext.Database.Action.Flow (getOrMkRoot, getOrMk_RootWithCorpus)
+import Gargantext.Database.Query.Table.Node (getOrMkList)
+import Gargantext.Database.Query.Table.User (insertUsersDemo)
+import Gargantext.Database.Admin.Config (userMaster, corpusMasterName)
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Trigger.Init (initTriggers)
+import Gargantext.Database.Admin.Types.Hyperdata (HyperdataCorpus)
+import Gargantext.Database.Admin.Types.Node (CorpusId, RootId, ListId)
+import Gargantext.Database.Prelude (Cmd, )
+import Gargantext.Prelude
+import System.Environment (getArgs)
+
 main :: IO ()
 main = do
   [iniPath] <- getArgs
@@ -42,13 +42,13 @@ main = do
 
   let
     mkRoots :: Cmd GargError [(UserId, RootId)]
-    mkRoots = mapM getOrMkRoot ["gargantua", "user1", "user2"]
+    mkRoots = mapM getOrMkRoot $ map UserName ["gargantua", "user1", "user2"]
     -- TODO create all users roots
 
   let
     initMaster :: Cmd GargError (UserId, RootId, CorpusId, ListId)
     initMaster = do
-      (masterUserId, masterRootId, masterCorpusId) <- getOrMk_RootWithCorpus userMaster (Left corpusMasterName) (Nothing :: Maybe HyperdataCorpus)
+      (masterUserId, masterRootId, masterCorpusId) <- getOrMk_RootWithCorpus (UserName userMaster) (Left corpusMasterName) (Nothing :: Maybe HyperdataCorpus)
       masterListId <- getOrMkList masterCorpusId masterUserId
       _triggers <- initTriggers masterListId
       pure (masterUserId, masterRootId, masterCorpusId, masterListId)

bin/gargantext-phylo/Main.hs

@@ -11,53 +11,42 @@ Phylo binaries
 
  -}
 
-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StandaloneDeriving #-}
 {-# LANGUAGE TypeOperators      #-}
 {-# LANGUAGE Strict             #-}
 
 module Main where
 
-import System.Directory (doesFileExist) 
-
+import Control.Concurrent.Async as CCA (mapConcurrently)
+import Control.Monad (mapM)
 import Data.Aeson
-import Data.Text (Text, unwords, unlines)
 import Data.List ((++),concat)
+import Data.Maybe
+import Data.Text (Text, unwords, unlines)
 import GHC.Generics
 import GHC.IO (FilePath)
+import Gargantext.Database.Admin.Types.Hyperdata
+import Gargantext.Database.Admin.Types.Node
 import Gargantext.Prelude
-import Gargantext.Text.List.CSV (csvGraphTermList)
-import Gargantext.Text.Corpus.Parsers.CSV (csv_title, csv_abstract, csv_publication_year)
-import qualified Gargantext.Text.Corpus.Parsers.CSV as CSV
+import Gargantext.Text.Context (TermList)
 import Gargantext.Text.Corpus.Parsers (FileFormat(..),parseFile)
+import Gargantext.Text.Corpus.Parsers.CSV (csv_title, csv_abstract, csv_publication_year)
+import Gargantext.Text.List.CSV (csvMapTermList)
 import Gargantext.Text.Terms.WithList
-import Gargantext.Text.Context (TermList)
-
-import Control.Monad (mapM)
-
-import System.Environment
-
 import Gargantext.Viz.Phylo
-import Gargantext.Viz.Phylo.Tools
 import Gargantext.Viz.Phylo.LevelMaker
+import Gargantext.Viz.Phylo.Tools
 import Gargantext.Viz.Phylo.View.Export
 import Gargantext.Viz.Phylo.View.ViewMaker
-
-import Gargantext.Database.Types.Node
-import Data.Maybe
-
-import Control.Concurrent.Async as CCA (mapConcurrently)
-
-import qualified Data.Map    as DM
-import qualified Data.Vector as DV
+import System.Directory (doesFileExist)
+import System.Environment
+import qualified Data.ByteString.Lazy as L
 import qualified Data.List   as DL
+import qualified Data.Map    as DM
 import qualified Data.Text   as DT
+import qualified Data.Vector as DV
+import qualified Gargantext.Text.Corpus.Parsers.CSV as CSV
 import qualified Prelude     as P
-import qualified Data.ByteString.Lazy as L
 
 
 --------------
@@ -123,7 +112,10 @@ filterTerms patterns (y,d) = (y,termsInText patterns d)
   where
     --------------------------------------
     termsInText :: Patterns -> Text -> [Text]
-    termsInText pats txt = DL.nub $ DL.concat $ map (map unwords) $ extractTermsWithList pats txt
+    termsInText pats txt = DL.nub
+                         $ DL.concat
+                         $ map (map unwords)
+                         $ extractTermsWithList pats txt
     --------------------------------------
 
 
@@ -199,7 +191,7 @@ main = do
     P.Left err -> putStrLn err
     P.Right conf -> do
 
-      termList <- csvGraphTermList (listPath conf)
+      termList <- csvMapTermList (listPath conf)
 
       corpus <- parse (corpusType conf) (limit conf) (corpusPath conf) termList
 

bin/gargantext-server/Main.hs

@@ -11,23 +11,20 @@ Script to start gargantext with different modes (Dev, Prod, Mock).
 
 -}
 
-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StandaloneDeriving #-}
 {-# LANGUAGE TypeOperators     #-}
 {-# LANGUAGE Strict             #-}
 
 module Main where
 
-
-import Options.Generic
+import Data.Version (showVersion)
 import Data.Text (unpack)
+import qualified Paths_gargantext as PG -- cabal magic build module
+import Options.Generic
+import System.Exit (exitSuccess)
 
 import Gargantext.Prelude
-import Gargantext.API (startGargantext) -- , startGargantextMock)
+import Gargantext.API (startGargantext, Mode(..)) -- , startGargantextMock)
 
 --------------------------------------------------------
 -- Graph Tests
@@ -37,13 +34,10 @@ import Gargantext.API (startGargantext) -- , startGargantextMock)
 --import qualified Gargantext.Graph.Distances.Matrice        as M
 --------------------------------------------------------
 
-data Mode = Dev | Mock | Prod 
-       deriving (Show, Read, Generic)
 instance ParseRecord Mode
 instance ParseField  Mode
 instance ParseFields Mode
 
-
 data MyOptions w =
   MyOptions { run  :: w ::: Mode
                         <?> "Possible modes: Dev | Mock | Prod"
@@ -51,6 +45,8 @@ data MyOptions w =
                         <?> "By default: 8008"
             , ini  :: w ::: Maybe Text
                         <?> "Ini-file path of gargantext.ini"
+            , version :: w ::: Bool
+                        <?> "Show version number and exit"
             }
    deriving (Generic)
 
@@ -60,22 +56,26 @@ deriving instance Show (MyOptions Unwrapped)
 
 main :: IO ()
 main = do
-  MyOptions myMode myPort myIniFile  <- unwrapRecord
+  MyOptions myMode myPort myIniFile myVersion  <- unwrapRecord
           "Gargantext server"
 
+  if myVersion then do
+    putStrLn $ "Version: " <> showVersion PG.version
+    System.Exit.exitSuccess
+  else
+    return ()
+
   let myPort' = case myPort of
         Just p  -> p
         Nothing -> 8008
 
   let start = case myMode of
-        Prod -> startGargantext myPort' (unpack myIniFile')
+        Mock -> panic "[ERROR] Mock mode unsupported"
+        _ -> startGargantext myMode myPort' (unpack myIniFile')
             where
               myIniFile' = case myIniFile of
                   Nothing -> panic "[ERROR] gargantext.ini needed"
                   Just i  -> i
-        Dev -> panic "[ERROR] Dev mode unsupported"
-        Mock -> panic "[ERROR] Mock mode unsupported"
-        -- _ -> startGargantextMock myPort'
 
   putStrLn $ "Starting with " <> show myMode <> " mode."
   start

bin/ghci

+stack ghci --profile

bin/install

+#!/bin/bash
+
+stack install #--profile # --test --haddock

build-shell.nix

-{ghc}:
-with (import <nixpkgs> {});
-
-haskell.lib.buildStackProject {
-  inherit ghc;
-  name = "gargantext";
-  buildInputs = [
-    docker-compose
-    blas
+{ ghc
+, pkgs ? import ./pinned.nix {}
+}:
+let
+  buildInputs = with pkgs; [
     bzip2
-    #gfortran
-    gfortran.cc.lib
-    glibc
+    git
     gmp
     gsl
     igraph
     liblapack
+    lzma
     pcre
     postgresql
-    #stack
     xz
     zlib
+    blas
+    gfortran7
+    gfortran7.cc.lib
   ];
+
+  libraryPaths = pkgs.lib.makeLibraryPath buildInputs;
+in
+pkgs.haskell.lib.buildStackProject rec {
+  inherit ghc;
+  inherit buildInputs;
+  name = "gargantext";
+
+  shellHook = ''
+    export LD_LIBRARY_PATH="${libraryPaths}"
+    export LIBRARY_PATH="${libraryPaths}"
+  '';
 }

devops/debian/debianPkgs

@@ -5,14 +5,14 @@ if git --version;
 then 
   echo "git installed, ok"
 else
-  sudo apt update && sudo apt install git
+  sudo apt update && sudo apt install -y git
 fi
 
 sudo apt update
-sudo apt install liblzma-dev libpcre3-dev libblas-dev liblapack-dev pkg-config libgsl-dev libbz2-dev postgresql nginx libpq-dev libigraph0-dev
+sudo apt install -y liblzma-dev libpcre3-dev libblas-dev liblapack-dev pkg-config libgsl-dev libbz2-dev postgresql nginx libpq-dev libigraph0-dev
 
 # Phylo management
-sudo apt install graphviz
+sudo apt install -y graphviz
 
-sudo apt install postgresql-server-dev-11
+sudo apt install -y postgresql-server-dev-11
 

devops/debian/deps

@@ -6,7 +6,7 @@ cd clustering-louvain-cplusplus
 cd ..
 
 
-sudo apt install default-jdk
+sudo apt install -y default-jdk
 wget https://dl.gargantext.org/coreNLP.tar.bz2
 tar xvjf coreNLP.tar.bz2
 # ./startServer.sh

devops/debian/install

@@ -26,7 +26,7 @@ tmux
 # Open Stack only: attach volumes
 # attach the volume created (OS interface or API)
 sudo fdisk -l
-sudo fisk /dev/vdb (n,p,t,83,w)
+sudo fdisk /dev/vdb (n,p,t,83,w)
 sudo mkfs.ext4 /dev/vdb1
 
 sudo blkid
@@ -42,7 +42,7 @@ sudo apt dist-upgrade
 ########################################################################
 
 sudo apt update
-sudo apt install liblzma-dev libpcre3-dev libblas-dev liblapack-dev pkg-config libgsl-dev libbz2-dev postgresql postgresql-server-dev-11 nginx libigraph0-dev
+sudo apt install liblzma-dev libpcre3-dev libblas-dev liblapack-dev pkg-config libgsl-dev libbz2-dev postgresql postgresql-server-dev-11 nginx libigraph0-dev libgfortran-8-dev
 
 
 sudo apt install git

devops/docker/Dockerfile

-from fpco/stack-build:lts-14.6
+from fpco/stack-build:lts-14.27
 
 RUN apt-get update && \
     apt-get install -y git libigraph0-dev && \

devops/docker/docker-compose.yaml

@@ -3,6 +3,7 @@ version: '3'
 services:
   postgres:
     image: 'postgres:latest'
+    network_mode: host
     ports:
       - 5432:5432
     environment:
@@ -15,6 +16,17 @@ services:
       - ../dbs:/dbs
       - ../postgres/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
 
+  pgadmin:
+    image: 'dpage/pgadmin4'
+    ports:
+      - 8081:80
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin
+      PGADMIN_DEFAULT_PASSWORD: admin
+
+    depends_on:
+      - postgres
+
   corenlp:
     image: 'cgenie/corenlp-garg'
     ports:

gargantext.ini

-[gargantext]
-MASTER_USER = gargantua
-
-[django]
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = %4{Vs(Pc!GU-]@OaAl0)(*4/yERwU<ct`ncV{1)O%32$6q
-# Space-separated list of hosts
-ALLOWED_HOSTS = localhost
-# Time-zone, possible values here: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-TIME_ZONE = Europe/Paris
-# PostgreSQL access
-DB_HOST = 127.0.0.1
-DB_PORT = 5432
-DB_NAME = gargandbV5
-DB_USER = gargantua
-DB_PASS = C8kdcUrAQy66U
-# Logs
-LOG_FILE = /var/log/gargantext/backend/django.log
-LOG_LEVEL = DEBUG
-LOG_FORMATTER = verbose
-# Pidfile of django backend test server
-TESTSERVER_PIDFILE = /tmp/gargantext_testserver.pid
-# Celery
-CELERYD_PID_FILE = /tmp/celery.pid
-CELERYD_LOG_FILE = /var/log/gargantext/backend/celery.log
-CELERYD_LOG_LEVEL = DEBUG
-
-[uwsgi]
-
-# See: http://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html
-# And: http://uwsgi-docs.readthedocs.io/en/latest/articles/TheArtOfGracefulReloading.html
-
-# Tip from: https://serverfault.com/questions/411361/uwsgi-ini-configuration-for-python-apps
-if-env         = VIRTUAL_ENV
-print          = [uWSGI] launched from virtualenv %(_)
-virtualenv     = %(_)
-endif          =
-
-# needed to run uwsgi when it was not installed with pipenv
-plugins        = python35
-# needed to run uwsgi outside of pipenv shell, because of this virtualenv bug:
-# https://github.com/kennethreitz/pipenv/issues/829
-home           = /home/alexandre/.local/share/virtualenvs/gargantext-ykNZD4Cw
-# unix socket is better than TCP one, there is less overhead
-socket         = /tmp/gargantext.sock
-chmod-socket   = 664
-# user running uwsgi MUST be a member of www-data group
-chown-socket   = %U:www-data
-# wsgi django module
-module         = gargantext.backend.wsgi:application
-# about master: http://uwsgi-docs.readthedocs.io/en/latest/Management.html?highlight=master#reloading-the-server
-master         = True
-# can do: uwsgi --stop /tmp/gargantext.pid
-pidfile        = /tmp/gargantext.pid
-# clear environment on exit
-vacuum         = True
-max-requests   = 5000
-# background the process & log
-daemonize      = /var/log/gargantext/uwsgi/@(exec://date +%%Y-%%m-%%d).log
-logfile-chmod  = 644
-# one log file per day
-log-reopen     = true
-
-# touch /tmp/gargantext.reload to reload configuration (after git pull for instance)
-touch-reload   = /tmp/gargantext.reload

gargantext.ini_toModify

+[gargantext]
+MASTER_USER = gargantua
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY      = PASSWORD_TO_CHANGE
+
+FRAME_WRITE_URL = URL_TO_CHANGE
+FRAME_CALC_URL  = URL_TO_CHANGE
+
+[database]
+# PostgreSQL access
+DB_HOST = 127.0.0.1
+DB_PORT = 5432
+DB_NAME = gargandbV5
+DB_USER = gargantua
+DB_PASS = PASSWORD_TO_CHANGE
+# Logs
+LOG_FILE = /var/log/gargantext/backend.log
+LOG_LEVEL = DEBUG
+LOG_FORMATTER = verbose

package.yaml

 name: gargantext
-version: '0.0.0.4'
+version: '0.0.1.6.8'
 synopsis: Search, map, share
 description: Please see README.md
 category: Data
 author: Gargantext Team
 maintainer: team@gargantext.org
 copyright:
-    - ! 'Copyright: (c) 2017-2018: see git logs and README'
-license: BSD3
+    - ! 'Copyright: (c) 2017-Present: see git logs and README'
+license: AGPL-3
 homepage: https://gargantext.org
 ghc-options: -Wall
+extra-libraries:
+  - gfortran
 dependencies:
-- extra
-- text
+  - extra
+  - text
+default-extensions:
+  - DataKinds
+  - DeriveGeneric
+  - FlexibleContexts
+  - FlexibleInstances
+  - GeneralizedNewtypeDeriving
+  - MultiParamTypeClasses
+  - NoImplicitPrelude
+  - OverloadedStrings
+  - RankNTypes
 library:
   source-dirs: src
   ghc-options:
@@ -22,35 +34,32 @@ library:
   - -Wunused-binds
   - -Wunused-imports
   - -Werror
+  - -freduction-depth=300
   exposed-modules:
   - Gargantext
   - Gargantext.API
-  - Gargantext.API.Auth
-  - Gargantext.API.Count
-  - Gargantext.API.FrontEnd
-  - Gargantext.API.Ngrams
+  - Gargantext.API.HashedResponse
   - Gargantext.API.Node
-    #  - Gargantext.API.Orchestrator
-  - Gargantext.API.Search
-  - Gargantext.API.Settings
-  - Gargantext.API.Types
+  - Gargantext.API.Admin.Settings
+  - Gargantext.API.Prelude
   - Gargantext.Core
   - Gargantext.Core.Types
+  - Gargantext.Core.Types.Individu
   - Gargantext.Core.Types.Main
   - Gargantext.Core.Utils.Prefix
-  - Gargantext.Database
-  - Gargantext.Database.Init
-  - Gargantext.Database.Config
-  - Gargantext.Database.Flow
-  - Gargantext.Database.Schema.Node
-  - Gargantext.Database.Tree
-  - Gargantext.Database.Types.Node
-  - Gargantext.Database.Utils
-  - Gargantext.Database.Schema.User
+  - Gargantext.Database.Action.Flow
+  - Gargantext.Database.Query.Table.User
+  - Gargantext.Database.Query.Table.Node
+  - Gargantext.Database.Prelude
+  - Gargantext.Database.Admin.Trigger.Init
+  - Gargantext.Database.Admin.Config
+  - Gargantext.Database.Admin.Types.Hyperdata
+  - Gargantext.Database.Admin.Types.Node
   - Gargantext.Prelude
   - Gargantext.Text
   - Gargantext.Text.Context
   - Gargantext.Text.Corpus.Parsers
+  - Gargantext.Text.Corpus.Parsers.Date.Parsec
   - Gargantext.Text.Corpus.API
   - Gargantext.Text.Corpus.Parsers.CSV
   - Gargantext.Text.Examples
@@ -81,14 +90,20 @@ library:
   - Gargantext.Viz.Phylo.LevelMaker
   - Gargantext.Viz.Phylo.View.Export
   - Gargantext.Viz.Phylo.View.ViewMaker
+  - Gargantext.Viz.Types
   dependencies:
-  - array
+  - HSvm
+  - KMP
+  - MonadRandom
   - QuickCheck
+  - SHA
+  - Unique
   - accelerate
   - aeson
   - aeson-lens
   - aeson-pretty
-  - argon2
+  - password
+  - array
   - async
   - attoparsec
   - auto-update
@@ -101,53 +116,51 @@ library:
   - bytestring
   - case-insensitive
   - cassava
-  #- charsetdetect-ae  # detect charset
+  - cereal     # (IGraph)
   - clock
   - clustering-louvain
   - conduit
   - conduit-extra
   - containers
   - contravariant
-  - crawlerPubMed
-  - crawlerIsidore
   - crawlerHAL
   - crawlerISTEX
+  - crawlerIsidore
+  - crawlerPubMed
   - data-time-segment
   - deepseq
   - directory
   - duckling
   - exceptions
-  - filepath
-  - formatting
-  - fullstop
+  - fast-logger
   - fclabels
   - fgl
-  - fast-logger
   - filelock
+  - filepath
+  - formatting
   - full-text-search
+  - fullstop
   - graphviz
   - haskell-igraph
+  - hlcm
+  - hsparql
+  - hstatistics
+  - http-api-data
   - http-client
   - http-client-tls
   - http-conduit
   - http-media
-  - http-api-data
   - http-types
-  - hsparql
-  - hstatistics
-  - HSvm
   - hxt
-  - hlcm
   - ini
   - insert-ordered-containers
   - jose
-    #  - kmeans-vector
   - json-stream
-  - KMP
   - lens
   - located-base
   - logging-effect
   - matrix
+  - monad-control
   - monad-logger
   - mtl
   - natural-transformation
@@ -166,36 +179,34 @@ library:
   - profunctors
   - protolude
   - pureMD5
-  - random-shuffle
-  - MonadRandom
-  - SHA
-  - simple-reflect
-  - cereal     # (IGraph)
-  - singletons # (IGraph)
   - quickcheck-instances
-  - random
   - rake
+  - random
+  - random-shuffle
+  - rdf4h
   - regex-compat
+  - resource-pool
   - resourcet
-  - rdf4h
   - safe
   - semigroups
+  - serialise
   - servant
   - servant-auth
   - servant-auth-server >= 0.4.4.0
   - servant-auth-swagger
   - servant-blaze
+  - servant-cassava
   - servant-client
-  - servant-flatten
   - servant-job
   - servant-mock
   - servant-multipart
   - servant-server
+  - servant-static-th
   - servant-swagger
   - servant-swagger-ui
-  - servant-static-th
-  - servant-cassava
-  - serialise
+  - servant-xml
+  - simple-reflect
+  - singletons # (IGraph)
   - split
   - stemmer
   - string-conversions
@@ -209,7 +220,6 @@ library:
   - transformers
   - transformers-base
   - unordered-containers
-  - Unique
   - uuid
   - validity
   - vector
@@ -220,9 +230,12 @@ library:
   - wreq
   - xml-conduit
   - xml-types
+  - xmlbf
   - yaml
   - zip
   - zlib
+    #  - kmeans-vector
+  #- charsetdetect-ae  # detect charset
 #  - utc
 #  API external connections
 
@@ -237,6 +250,7 @@ executables:
     - -rtsopts
     - -threaded
     - -with-rtsopts=-N
+    - -fprof-auto
     dependencies:
       - base
       - containers
@@ -351,36 +365,58 @@ executables:
 
 
 tests:
-#   garg-test:
-#     main: Main.hs
-#     source-dirs: src-test
-#     ghc-options:
-#     - -threaded
-#     - -rtsopts
-#     - -with-rtsopts=-N
-#     dependencies:
-#     - base
-#     - gargantext
-#     - hspec
-#     - QuickCheck
-#     - quickcheck-instances 
-#     - time
-#     - parsec
-#     - duckling
-#     - text
-  garg-doctest:
-    main: Main.hs
-    source-dirs: src-doctest
-    ghc-options:
-    - -O2
-    - -Wcompat
-    - -Wmissing-signatures
-    - -rtsopts
-    - -threaded
-    - -with-rtsopts=-N
-    dependencies:
-    - doctest
-    - Glob
-    - QuickCheck
-    - base
-    - gargantext
+   garg-test:
+     main: Main.hs
+     source-dirs: src-test
+     default-extensions:
+     - DataKinds
+     - DeriveGeneric
+     - FlexibleContexts
+     - FlexibleInstances
+     - GeneralizedNewtypeDeriving
+     - MultiParamTypeClasses
+     - NoImplicitPrelude
+     - OverloadedStrings
+     - RankNTypes
+     ghc-options:
+     - -threaded
+     - -rtsopts
+     - -with-rtsopts=-N
+     dependencies:
+     - base
+     - gargantext
+     - hspec
+     - QuickCheck
+     - quickcheck-instances
+     - time
+     - parsec
+     - duckling
+     - text
+#  garg-doctest:
+#    main: Main.hs
+#    source-dirs: src-doctest
+#    ghc-options:
+#    - -O2
+#    - -Wcompat
+#    - -Wmissing-signatures
+#    - -rtsopts
+#    - -threaded
+#    - -with-rtsopts=-N
+#    dependencies:
+#    - doctest
+#    - Glob
+#    - QuickCheck
+#    - base
+#    - gargantext
+#    default-extensions:
+#    - DataKinds
+#    - DeriveGeneric
+#    - FlexibleContexts
+#    - FlexibleInstances
+#    - GeneralizedNewtypeDeriving
+#    - MultiParamTypeClasses
+#    - NoImplicitPrelude
+#    - OverloadedStrings
+#    - RankNTypes
+#
+

pinned-19.09.nix

+# this version of nixpkgs contains liblapack at 3.8.0
+# this version of nixpkgs contains gsl at 2.5.0
+import (
+  builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/19.09.tar.gz";
+    sha256 = "0mhqhq21y5vrr1f30qd2bvydv4bbbslvyzclhw0kdxmkgg3z4c92";
+  }
+)

pinned.nix

+# this version of nixpkgs contains liblapack at 3.8.0
+# this version of nixpkgs contains gsl at 2.5.0
+import (
+  builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/18.09.tar.gz";
+    sha256 = "1ib96has10v5nr6bzf7v8kw7yzww8zanxgw2qi1ll1sbv6kj6zpd";
+  }
+)

server

+~/.local/bin/gargantext-server --ini gargantext.ini --run Dev +RTS # -p

default.nix → shell.nix

-with (import <nixpkgs> {});
-stdenv.mkDerivation rec {
-  name = "env";
-  env = buildEnv {
-    name = name;
-    paths = buildInputs;
-  };
-  buildInputs = [
+{ pkgs ? import ./pinned-19.09.nix {} }:
+
+pkgs.mkShell {
+  buildInputs = with pkgs; [
     docker-compose
     #glibc
     #gmp
@@ -16,8 +12,4 @@ stdenv.mkDerivation rec {
     #stack
     #xz
   ];
-  builder = builtins.toFile "builder.sh" ''
-    source $stdenv/setup
-    touch $out
-  '';
 }

src-doctest/Main.hs

 import System.FilePath.Glob
 import Test.DocTest
+import Gargantext.Prelude
 
 main :: IO ()
 main = glob "src/Gargantext/" >>= doctest

src-test/Main.hs

@@ -8,20 +8,21 @@ Stability   : experimental
 Portability : POSIX
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
 --import qualified Ngrams.Lang.Fr as Fr
-import qualified Ngrams.Lang as Lang
+--import qualified Ngrams.Lang as Lang
 import qualified Ngrams.Lang.Occurrences as Occ
 import qualified Ngrams.Metrics as Metrics
 import qualified Parsers.Date as PD
+import qualified Graph.Distance as GD
 
 main :: IO ()
 main = do
-    Occ.parsersTest
-    Lang.ngramsExtractionTest FR
-    Lang.ngramsExtractionTest EN
-    Metrics.main
+--    Occ.parsersTest
+--    Lang.ngramsExtractionTest FR
+--    Lang.ngramsExtractionTest EN
+--    Metrics.main
     PD.testFromRFC3339
+    GD.test

src-test/Ngrams/Lang.hs

@@ -12,10 +12,10 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module Ngrams.Lang where
 
+{-
 import Gargantext.Prelude (IO())
 
 import Gargantext.Core (Lang(..))
@@ -25,4 +25,4 @@ import qualified Ngrams.Lang.En as En
 ngramsExtractionTest :: Lang -> IO ()
 ngramsExtractionTest FR = Fr.ngramsExtractionTest
 ngramsExtractionTest EN = En.ngramsExtractionTest
-
+-}

src-test/Ngrams/Lang/En.hs

@@ -11,12 +11,11 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE NoImplicitPrelude   #-}
 
 module Ngrams.Lang.En where
 
+{-
 import Data.List ((!!))
 import Data.Text (Text)
 
@@ -24,8 +23,11 @@ import Test.Hspec
 
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
-import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
 
+-- TODO this import is not used anymore
+import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
+-- use instead
+-- import Gargantext.Text.Terms (extractNgramsT)
 
 ngramsExtractionTest :: IO ()
 ngramsExtractionTest = hspec $ do
@@ -45,4 +47,4 @@ ngramsExtractionTest = hspec $ do
             t2 <- map (selectNgrams EN) <$> extractNgrams EN t
             t2 `shouldBe` [[("Donald Trump","NNP","PERSON"),("president of the United-States of America","NN","LOCATION")]]
 
-
+-}

src-test/Ngrams/Lang/Fr.hs

@@ -11,18 +11,19 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE NoImplicitPrelude   #-}
 
 module Ngrams.Lang.Fr where
 
+{-
 import Test.Hspec
 
 import Gargantext.Prelude
 import Gargantext.Core (Lang(..))
+-- TODO this import is not used anymore
 import Gargantext.Text.Ngrams.PosTagging.Parser (extractNgrams, selectNgrams)
-
+-- use instead
+-
 ngramsExtractionTest :: IO ()
 ngramsExtractionTest = hspec $ do
     describe "Behavioral tests: ngrams extraction in French Language" $ do
@@ -63,4 +64,4 @@ ngramsExtractionTest = hspec $ do
             let textFr1 = "L'heure d'arrivée des coureurs dépend de la météo du jour."
             testFr1 <- map (selectNgrams FR) <$> (extractNgrams FR) textFr1
             testFr1 `shouldBe` [[("heure d' arrivée des coureurs","NC","O"),("météo du jour","NC","O")]]
-
+-}

src-test/Ngrams/Lang/Occurrences.hs

@@ -11,12 +11,11 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE NoImplicitPrelude   #-}
-{-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
 module Ngrams.Lang.Occurrences where
 
+{-
 import Test.Hspec
 
 import Data.Either (Either(Right))
@@ -61,4 +60,4 @@ parsersTest = hspec $ do
 --  describe "Parser for nodes" $ do
 --    it "returns the result of one parsing after space" $ do
 --      occOfCorpus 249509 "sciences" `shouldReturn` 7
-
+-}

src-test/Ngrams/Metrics.hs

@@ -14,11 +14,11 @@ commentary with @some markup@.
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
 {-# LANGUAGE CPP                  #-}
-{-# LANGUAGE NoImplicitPrelude    #-}
-{-# LANGUAGE OverloadedStrings    #-}
 
-module Ngrams.Metrics (main) where
+--module Ngrams.Metrics (main) where
+module Ngrams.Metrics  where
 
+{-
 import           Data.Text (Text)
 import qualified Data.Text as T
 import           Data.Ratio
@@ -141,3 +141,5 @@ testPair :: (Eq a, Show a)
   -> SpecWith ()
 testPair f a b r = it ("‘" <> T.unpack a <> "’ and ‘" <> T.unpack b <> "’") $
   f a b `shouldBe` r
+
+-}

src-test/Parsers/Date.hs

@@ -11,7 +11,6 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module Parsers.Date where
 
@@ -29,7 +28,7 @@ import Duckling.Time.Types (toRFC3339)
 
 -----------------------------------------------------------
 import Gargantext.Prelude
-import Gargantext.Text.Parsers.Date (fromRFC3339)
+import Gargantext.Text.Corpus.Parsers.Date.Parsec (fromRFC3339)
 import Parsers.Types
 -----------------------------------------------------------
 

src-test/Parsers/Types.hs

@@ -13,7 +13,6 @@ commentary with @some markup@.
 
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
-{-# LANGUAGE NoImplicitPrelude    #-}
 {-# LANGUAGE StandaloneDeriving   #-}
 
 module Parsers.Types where

src-test/Parsers/WOS.hs

@@ -11,6 +11,5 @@ Here is a longer description of this module, containing some
 commentary with @some markup@.
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module Parsers.WOS where

src/Gargantext.hs

@@ -10,7 +10,6 @@ Portability : POSIX
 @Gargantext@: search, map, share
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module Gargantext ( module Gargantext.API
                   , module Gargantext.Core

src/Gargantext/API/Auth.hs → src/Gargantext/API/Admin/Auth.hs

 {-|
-Module      : Gargantext.API.Auth
+Module      : Gargantext.API.Admin.Auth
 Description : Server API Auth Module
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -20,47 +20,41 @@ TODO-ACCESS Critical
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude   #-}
-{-# LANGUAGE DeriveGeneric       #-}
-{-# LANGUAGE DataKinds           #-}
-{-# LANGUAGE FlexibleContexts    #-}
-{-# LANGUAGE OverloadedStrings   #-}
-{-# LANGUAGE RankNTypes          #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TemplateHaskell     #-}
 
-module Gargantext.API.Auth
+module Gargantext.API.Admin.Auth
       where
 
 import Control.Lens (view)
-import Control.Monad.IO.Class (liftIO)
 import Data.Aeson.TH (deriveJSON)
-import Data.List (elem)
 import Data.Swagger
-import Data.Text (Text, reverse)
+import Data.Text (Text)
 import Data.Text.Lazy (toStrict)
 import Data.Text.Lazy.Encoding (decodeUtf8)
 import GHC.Generics (Generic)
-import Servant
-import Servant.Auth.Server
+import Gargantext.API.Admin.Settings
+import Gargantext.API.Prelude (HasJoseError(..), joseError, HasServerError, GargServerC)
+import Gargantext.Core.Types.Individu (User(..), Username, GargPassword(..), arbitraryUsername, arbitraryPassword)
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.API.Settings
-import Gargantext.API.Types (HasJoseError(..), joseError, HasServerError, GargServerC)
---import Gargantext.API.Types (HasJoseError(..), joseError, HasServerError, serverError, GargServerC)
-import Gargantext.Database.Root (getRoot)
-import Gargantext.Database.Tree (isDescendantOf, isIn)
-import Gargantext.Database.Types.Node (NodePoly(_node_id), NodeId(..), UserId, ListId, DocId)
-import Gargantext.Database.Utils (Cmd', CmdM, HasConnection)
+import Gargantext.Database.Query.Tree (isDescendantOf, isIn)
+import Gargantext.Database.Query.Tree.Root (getRoot)
+import Gargantext.Database.Schema.Node (NodePoly(_node_id)) 
+import Gargantext.Database.Admin.Types.Node (NodeId(..), UserId, ListId, DocId)
+import Gargantext.Database.Prelude (Cmd', CmdM, HasConnectionPool, HasConfig)
 import Gargantext.Prelude hiding (reverse)
+import Gargantext.Database.Query.Table.User
+import Servant
+import Servant.Auth.Server
 import Test.QuickCheck (elements, oneof)
 import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
-import Gargantext.Core.Types.Individu (Username, Password, arbitraryUsername, arbitraryPassword)
+import qualified Gargantext.Core.Auth as Auth
 
 ---------------------------------------------------
 
 -- | Main types for AUTH API
 data AuthRequest = AuthRequest { _authReq_username :: Username
-                               , _authReq_password :: Password
+                               , _authReq_password :: GargPassword
                                }
   deriving (Generic)
 
@@ -91,25 +85,31 @@ makeTokenForUser :: (HasSettings env, HasJoseError err)
                  => NodeId -> Cmd' env err Token
 makeTokenForUser uid = do
   jwtS <- view $ settings . jwtSettings
-  e <- liftIO $ makeJWT (AuthenticatedUser uid) jwtS Nothing
+  e <- liftBase $ makeJWT (AuthenticatedUser uid) jwtS Nothing
   -- TODO-SECURITY here we can implement token expiration ^^.
   either joseError (pure . toStrict . decodeUtf8) e
   -- TODO not sure about the encoding...
 
-checkAuthRequest :: (HasSettings env, HasConnection env, HasJoseError err)
-                 => Username -> Password -> Cmd' env err CheckAuth
-checkAuthRequest u p
-  | not (u `elem` arbitraryUsername) = pure InvalidUser
-  | u /= reverse p = pure InvalidPassword
-  | otherwise = do
-      muId <- head <$> getRoot u
-      case _node_id <$> muId of
-        Nothing  -> pure InvalidUser
-        Just uid -> do
-          token <- makeTokenForUser uid
-          pure $ Valid token uid
-
-auth :: (HasSettings env, HasConnection env, HasJoseError err)
+checkAuthRequest :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env)
+                 => Username
+                 -> GargPassword
+                 -> Cmd' env err CheckAuth
+checkAuthRequest u (GargPassword p) = do
+  candidate <- head <$> getUsersWith u
+  case candidate of
+    Nothing -> pure InvalidUser
+    Just (UserLight _id _u _email h) ->
+      case Auth.checkPassword (Auth.mkPassword p) (Auth.PasswordHash h) of
+        Auth.PasswordCheckFail    -> pure InvalidPassword
+        Auth.PasswordCheckSuccess -> do
+          muId <- head <$> getRoot (UserName u)
+          case _node_id <$> muId of
+            Nothing  -> pure InvalidUser
+            Just uid -> do
+              token <- makeTokenForUser uid
+              pure $ Valid token uid
+
+auth :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env)
      => AuthRequest -> Cmd' env err AuthResponse
 auth (AuthRequest u p) = do
   checkAuthRequest' <- checkAuthRequest u p
@@ -123,8 +123,10 @@ newtype AuthenticatedUser = AuthenticatedUser
   } deriving (Generic)
 
 $(deriveJSON (unPrefix "_authUser_") ''AuthenticatedUser)
+
 instance ToSchema AuthenticatedUser where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_authUser_")
+
 instance ToJWT AuthenticatedUser
 instance FromJWT AuthenticatedUser
 

src/Gargantext/API/FrontEnd.hs → src/Gargantext/API/Admin/FrontEnd.hs

 {-|
-Module      : Gargantext.API.FrontEnd
+Module      : Gargantext.API.Admin.FrontEnd
 Description : Server FrontEnd API
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -11,12 +11,10 @@ Loads all static file for the front-end.
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE DataKinds            #-}
 {-# LANGUAGE TypeOperators        #-}
 
 ---------------------------------------------------------------------
-module Gargantext.API.FrontEnd where
+module Gargantext.API.Admin.FrontEnd where
 
 import Servant
 import Servant.Server.StaticFiles (serveDirectoryFileServer)

src/Gargantext/API/Orchestrator.hs → src/Gargantext/API/Admin/Orchestrator.hs

-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE FlexibleContexts  #-}
-{-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE RankNTypes        #-}
+{-|
+Module      : Gargantext.API.Admin.Orchestrator
+Description : Jobs Orchestrator
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 {-# LANGUAGE TemplateHaskell   #-}
 {-# LANGUAGE TypeOperators     #-}
 
-module Gargantext.API.Orchestrator where
+module Gargantext.API.Admin.Orchestrator where
 
-import Gargantext.Prelude
-import Gargantext.API.Settings
-import Gargantext.API.Orchestrator.Types
-import Gargantext.API.Orchestrator.Scrapy.Schedule
 import Control.Lens hiding (elements)
 import Data.Aeson
-import qualified Data.ByteString.Lazy.Char8 as LBS
+import Data.Text
+import Gargantext.API.Admin.Orchestrator.Scrapy.Schedule
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Admin.Settings
+import Gargantext.Prelude
 import Servant
 import Servant.Job.Async
 import Servant.Job.Client
 import Servant.Job.Server
 import Servant.Job.Utils (extendBaseUrl)
+import qualified Data.ByteString.Lazy.Char8 as LBS
 
 callJobScrapy :: (ToJSON e, FromJSON e, FromJSON o, MonadClientJob m)
               => JobServerURL e Schedule o
@@ -37,7 +43,7 @@ callJobScrapy jurl schedule = do
 logConsole :: ToJSON a => a -> IO ()
 logConsole = LBS.putStrLn . encode
 
-callScraper :: MonadClientJob m => URL -> ScraperInput -> m ScraperStatus
+callScraper :: MonadClientJob m => URL -> ScraperInput -> m JobLog
 callScraper url input =
   callJobScrapy jurl $ \cb ->
     Schedule
@@ -57,11 +63,11 @@ callScraper url input =
           ,("callback",     [toUrlPiece cb])]
       }
   where
-    jurl :: JobServerURL ScraperStatus Schedule ScraperStatus
+    jurl :: JobServerURL JobLog Schedule JobLog
     jurl = JobServerURL url Callback
 
 pipeline :: FromJSON e => URL -> ClientEnv -> ScraperInput
-                       -> (e -> IO ()) -> IO ScraperStatus
+                       -> (e -> IO ()) -> IO JobLog
 pipeline scrapyurl client_env input log_status = do
   e <- runJobMLog client_env log_status $ callScraper scrapyurl input
   either (panic . cs . show) pure e -- TODO throwError
@@ -73,7 +79,7 @@ pipeline scrapyurl client_env input log_status = do
 scrapyOrchestrator :: Env -> IO (Server (WithCallbacks ScraperAPI))
 scrapyOrchestrator env = do
   apiWithCallbacksServer (Proxy :: Proxy ScraperAPI)
-    defaultSettings (extendBaseUrl ("scraper" :: String) $ env ^. env_self_url)
+    defaultSettings (extendBaseUrl ("scraper" :: Text) $ env ^. env_self_url)
     (env ^. env_manager) (LogEvent logConsole) $
     simpleServeJobsAPI (env ^. env_scrapers) .
       simpleJobFunction . pipeline (URL $ env ^. env_settings . scrapydUrl)

src/Gargantext/API/Orchestrator/Scrapy/Schedule.hs → src/Gargantext/API/Admin/Orchestrator/Scrapy/Schedule.hs

-{-# LANGUAGE DataKinds #-}
-{-# LANGUAGE DeriveGeneric #-}
-{-# LANGUAGE OverloadedStrings #-}
+{-|
+Module      : Gargantext.API.Admin.Orchestartor.Scrapy.Schedule
+Description : Server API Auth Module
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
 {-# LANGUAGE TypeOperators #-}
-module Gargantext.API.Orchestrator.Scrapy.Schedule where
+
+module Gargantext.API.Admin.Orchestrator.Scrapy.Schedule
+  where
 
 import Control.Lens
 import Data.Aeson
-import qualified Data.HashMap.Strict as H
 import Data.Text (Text)
 import GHC.Generics
+import Protolude
 import Servant
-import Servant.Job.Utils (jsonOptions)
 import Servant.Client
+import Servant.Job.Utils (jsonOptions)
 import Web.FormUrlEncoded hiding (parseMaybe)
+import qualified Data.HashMap.Strict as H
+
+------------------------------------------------------------------------
 
 data Schedule = Schedule
   { s_project :: !Text

src/Gargantext/API/Orchestrator/Types.hs → src/Gargantext/API/Admin/Orchestrator/Types.hs

 {-# OPTIONS_GHC -fno-warn-orphans #-}
-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE FlexibleContexts  #-}
-{-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE RankNTypes        #-}
 {-# LANGUAGE TemplateHaskell   #-}
 {-# LANGUAGE TypeOperators     #-}
 
-module Gargantext.API.Orchestrator.Types where
+module Gargantext.API.Admin.Orchestrator.Types
+  where
 
-import Gargantext.Prelude
 import Control.Lens hiding (elements)
 import Data.Aeson
 import Data.Proxy
-import Data.Text (Text)
 import Data.Swagger hiding (URL, url, port)
+import Data.Text (Text)
 import GHC.Generics hiding (to)
+import Gargantext.Core.Types (TODO(..))
+import Gargantext.Prelude
 import Servant.Job.Async
 import Servant.Job.Types
 import Servant.Job.Utils (jsonOptions)
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary
-import Gargantext.Core.Types (TODO(..))
+
+------------------------------------------------------------------------
 instance Arbitrary a => Arbitrary (JobStatus 'Safe a) where
   arbitrary = panic "TODO"
 
@@ -30,18 +27,12 @@ instance Arbitrary a => Arbitrary (JobOutput a) where
   arbitrary = JobOutput <$> arbitrary
 
 -- | Main Types
+-- TODO IsidoreAuth
 data ExternalAPIs = All
                   | PubMed
-
-                  | HAL_EN
-                  | HAL_FR
-
-                  | IsTex_EN
-                  | IsTex_FR
-
-                  | Isidore_EN
-                  | Isidore_FR
-                  -- | IsidoreAuth
+                  | HAL
+                  | IsTex
+                  | Isidore
   deriving (Show, Eq, Enum, Bounded, Generic)
 
 
@@ -85,7 +76,7 @@ data ScraperEvent = ScraperEvent
   , _scev_level   :: !(Maybe Text)
   , _scev_date    :: !(Maybe Text)
   }
-  deriving Generic
+  deriving (Show, Generic)
 
 instance Arbitrary ScraperEvent where
   arbitrary = ScraperEvent <$> elements [Nothing, Just "test message"]
@@ -98,28 +89,30 @@ instance ToJSON ScraperEvent where
 instance FromJSON ScraperEvent where
   parseJSON = genericParseJSON $ jsonOptions "_scev_"
 
-data ScraperStatus = ScraperStatus
+
+
+data JobLog = JobLog
   { _scst_succeeded :: !(Maybe Int)
   , _scst_failed    :: !(Maybe Int)
   , _scst_remaining :: !(Maybe Int)
   , _scst_events    :: !(Maybe [ScraperEvent])
   }
-  deriving Generic
+  deriving (Show, Generic)
 
-instance Arbitrary ScraperStatus where
-  arbitrary = ScraperStatus
+instance Arbitrary JobLog where
+  arbitrary = JobLog
            <$> arbitrary
            <*> arbitrary
            <*> arbitrary
            <*> arbitrary
 
-instance ToJSON ScraperStatus where
+instance ToJSON JobLog where
   toJSON = genericToJSON $ jsonOptions "_scst_"
 
-instance FromJSON ScraperStatus where
+instance FromJSON JobLog where
   parseJSON = genericParseJSON $ jsonOptions "_scst_"
 
-instance ToSchema ScraperStatus -- TODO _scst_ prefix
+instance ToSchema JobLog -- TODO _scst_ prefix
 
 instance ToSchema ScraperInput  -- TODO _scin_ prefix
 instance ToSchema ScraperEvent  -- TODO _scev_ prefix
@@ -130,6 +123,6 @@ instance ToParamSchema Offset -- where
 instance ToParamSchema Limit -- where
   -- toParamSchema = panic "TODO"
 
-type ScrapersEnv = JobEnv ScraperStatus ScraperStatus
+type ScrapersEnv = JobEnv JobLog JobLog
 
-type ScraperAPI  = AsyncJobsAPI ScraperStatus ScraperInput  ScraperStatus
+type ScraperAPI  = AsyncJobsAPI JobLog ScraperInput  JobLog

src/Gargantext/API/Utils.hs → src/Gargantext/API/Admin/Utils.hs

 {-|
-Module      : Gargantext.API.Utils
+Module      : Gargantext.API.Admin.Utils
 Description : Server API main Types
 Copyright   : (c) CNRS, 2017-Present
 License     : BSD3
@@ -11,10 +11,8 @@ Mainly copied from Servant.Job.Utils (Thanks)
 
 -}
 
-{-# LANGUAGE OverloadedStrings  #-}
-{-# LANGUAGE NoImplicitPrelude  #-}
 
-module Gargantext.API.Utils
+module Gargantext.API.Admin.Utils
   where
 
 import Gargantext.Prelude

src/Gargantext/API/Count.hs

@@ -12,13 +12,9 @@ Count API part of Gargantext.
 
 {-# OPTIONS_GHC -fno-warn-name-shadowing #-}
 
-{-# LANGUAGE NoImplicitPrelude  #-}
-{-# LANGUAGE DataKinds          #-}
 {-# LANGUAGE TemplateHaskell    #-}
 {-# LANGUAGE TypeOperators      #-}
-{-# LANGUAGE DeriveGeneric      #-}
 {-# LANGUAGE DeriveAnyClass     #-}
-{-# LANGUAGE OverloadedStrings  #-}
 
 module Gargantext.API.Count
       where

src/Gargantext/API/HashedResponse.hs

+module Gargantext.API.HashedResponse where
+
+import Data.Aeson
+import Data.Swagger
+import qualified Data.Digest.Pure.MD5 as DPMD5
+import GHC.Generics (Generic)
+import Protolude
+
+data HashedResponse a = HashedResponse { md5 :: Text, value :: a }
+  deriving (Generic)
+
+instance ToSchema a => ToSchema (HashedResponse a)
+instance ToJSON a => ToJSON (HashedResponse a) where
+  toJSON = genericToJSON defaultOptions
+
+constructHashedResponse :: ToJSON a => a -> HashedResponse a
+constructHashedResponse v = HashedResponse { md5 = md5', value = v }
+  where
+    md5' = show $ DPMD5.md5 $ encode v
\ No newline at end of file

src/Gargantext/API/Ngrams/List.hs

+{-|
+Module      : Gargantext.API.Ngrams.List
+Description : Get Ngrams (lists)
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+{-# LANGUAGE TemplateHaskell   #-}
+{-# LANGUAGE TypeOperators     #-}
+
+module Gargantext.API.Ngrams.List
+  where
+
+import Control.Lens hiding (elements)
+import Data.Aeson
+import Data.List (zip)
+import Data.Map (Map, toList, fromList)
+import Data.Swagger (ToSchema, declareNamedSchema, genericDeclareNamedSchema)
+import Data.Text (Text, concat, pack)
+import GHC.Generics (Generic)
+import Gargantext.API.Node.Corpus.New
+import Gargantext.API.Node.Corpus.New.File (FileType(..))
+import Gargantext.API.Ngrams
+import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.API.Prelude (GargServer)
+import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
+import Gargantext.Database.Action.Flow (FlowCmdM)
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Schema.Ngrams (NgramsType(..), ngramsTypes)
+import Gargantext.Prelude
+import Network.HTTP.Media ((//), (/:))
+import Servant
+import Servant.Job.Async
+import Servant.Job.Utils (jsonOptions)
+import Web.FormUrlEncoded (FromForm)
+
+------------------------------------------------------------------------
+type NgramsList = (Map NgramsType (Versioned NgramsTableMap))
+------------------------------------------------------------------------
+type API =  Get '[JSON, HTML] (Headers '[Header "Content-Disposition" Text] NgramsList)
+       -- :<|> ReqBody '[JSON] NgramsList :> Post '[JSON] Bool
+       :<|> PostAPI
+
+api :: ListId -> GargServer API
+api l = get l :<|> postAsync l
+
+data HTML
+instance Accept HTML where
+  contentType _ = "text" // "html" /: ("charset", "utf-8")
+instance ToJSON a => MimeRender HTML a where
+  mimeRender _ = encode
+
+------------------------------------------------------------------------
+
+get :: RepoCmdM env err m
+        => ListId -> m (Headers '[Header "Content-Disposition" Text] NgramsList)
+get lId = do
+  lst <- get' lId
+  let (NodeId id) = lId
+  return $ addHeader (concat [ "attachment; filename=GarganText_NgramsList-"
+                             , pack $ show id
+                             , ".json"
+                             ]
+                     ) lst
+
+get' :: RepoCmdM env err m
+    => ListId -> m NgramsList
+get' lId = fromList
+       <$> zip ngramsTypes
+       <$> mapM (getNgramsTableMap lId) ngramsTypes
+
+------------------------------------------------------------------------
+
+-- TODO : purge list
+post :: FlowCmdM env err m
+    => ListId
+    -> NgramsList
+    -> m Bool
+post l m  = do
+  -- TODO check with Version for optim
+  _ <- mapM (\(nt, Versioned _v ns) -> setListNgrams l nt ns) $ toList m
+  -- TODO reindex
+  pure True
+
+------------------------------------------------------------------------
+------------------------------------------------------------------------
+
+type PostAPI = Summary "Update List"
+        :> "add"
+        :> "form"
+        :> "async"
+        :> AsyncJobs JobLog '[FormUrlEncoded] WithFile JobLog
+
+postAsync :: ListId -> GargServer PostAPI
+postAsync lId =
+  serveJobsAPI $
+    JobFunction (\f  log' -> postAsync' lId f (liftBase . log'))
+
+postAsync' :: FlowCmdM env err m
+          => ListId
+          -> WithFile
+          -> (JobLog -> m ())
+          -> m JobLog
+postAsync' l (WithFile _ m _) logStatus = do
+
+  logStatus JobLog { _scst_succeeded = Just 0
+                          , _scst_failed    = Just 0
+                          , _scst_remaining = Just 1
+                          , _scst_events    = Just []
+                          }
+  _r <- post l m
+
+  pure JobLog { _scst_succeeded = Just 1
+                     , _scst_failed    = Just 0
+                     , _scst_remaining = Just 0
+                     , _scst_events    = Just []
+                     }
+
+data WithFile = WithFile
+  { _wf_filetype :: !FileType
+  , _wf_data     :: !NgramsList
+  , _wf_name     :: !Text
+  } deriving (Eq, Show, Generic)
+
+makeLenses ''WithFile
+instance FromForm WithFile
+instance FromJSON WithFile where
+  parseJSON = genericParseJSON $ jsonOptions "_wf_"
+instance ToSchema WithFile where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wf_")
+

src/Gargantext/API/Ngrams/NTree.hs

@@ -9,11 +9,6 @@ Portability : POSIX
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE RankNTypes        #-}
-{-# LANGUAGE DeriveGeneric        #-}
-{-# LANGUAGE GeneralizedNewtypeDeriving #-}
 {-# LANGUAGE TemplateHaskell   #-}
 
 module Gargantext.API.Ngrams.NTree

src/Gargantext/API/Ngrams/Tools.hs

@@ -9,9 +9,6 @@ Portability : POSIX
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE RankNTypes        #-}
 
 module Gargantext.API.Ngrams.Tools
   where
@@ -30,13 +27,12 @@ import Gargantext.Prelude
 import qualified Data.Map.Strict as Map
 import qualified Data.Set as Set
 
-
 type RootTerm = Text
 
 getRepo :: RepoCmdM env err m => m NgramsRepo
 getRepo = do
   v <- view repoVar
-  liftIO $ readMVar v
+  liftBase $ readMVar v
 
 listNgramsFromRepo :: [ListId] -> NgramsType
                    -> NgramsRepo -> Map Text NgramsRepoElement
@@ -83,9 +79,9 @@ filterListWithRoot :: ListType -> Map Text (ListType, Maybe Text)
                       -> Map Text (Maybe RootTerm)
 filterListWithRoot lt m = Map.fromList
                     $ map (\(t,(_,r)) -> (t,r))
-                    $ filter isGraphTerm (Map.toList m)
+                    $ filter isMapTerm (Map.toList m)
   where
-    isGraphTerm (_t,(l, maybeRoot)) = case maybeRoot of
+    isMapTerm (_t,(l, maybeRoot)) = case maybeRoot of
       Nothing -> l == lt
       Just  r -> case Map.lookup r m of
         Nothing -> panic $ "Garg.API.Ngrams.Tools: filterWithRoot, unknown key: " <> r

src/Gargantext/API/Annuaire.hs → src/Gargantext/API/Node/Corpus/Annuaire.hs

 {-|
-Module      : Gargantext.API.Annuaire
+Module      : Gargantext.API.Node.Corpus.Annuaire
 Description : New annuaire API
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -8,16 +8,10 @@ Stability   : experimental
 Portability : POSIX
 -}
 
-{-# LANGUAGE NoImplicitPrelude  #-}
 {-# LANGUAGE TemplateHaskell    #-}
-{-# LANGUAGE DeriveGeneric      #-}
-{-# LANGUAGE DataKinds          #-}
 {-# LANGUAGE TypeOperators      #-}
-{-# LANGUAGE OverloadedStrings  #-}
-{-# LANGUAGE FlexibleContexts   #-}
-{-# LANGUAGE RankNTypes         #-}
 
-module Gargantext.API.Annuaire
+module Gargantext.API.Node.Corpus.Annuaire
       where
 
 import Control.Lens hiding (elements)
@@ -25,19 +19,18 @@ import Data.Aeson
 import Data.Swagger
 import Data.Text (Text)
 import GHC.Generics (Generic)
-import qualified Gargantext.API.Corpus.New.File as NewFile
-import Gargantext.API.Orchestrator.Types
+import Gargantext.API.Admin.Orchestrator.Types
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
-import Gargantext.Database.Flow (FlowCmdM)  -- flowAnnuaire
-import Gargantext.Database.Types.Node (AnnuaireId)
+import Gargantext.Database.Action.Flow (FlowCmdM)  -- flowAnnuaire
+import Gargantext.Database.Admin.Types.Node (AnnuaireId)
 import Gargantext.Prelude
 import Servant
-import Servant.API.Flatten (Flat)
 import Servant.Job.Core
 import Servant.Job.Types
 import Servant.Job.Utils (jsonOptions)
-import Web.FormUrlEncoded          (FromForm)
+import Web.FormUrlEncoded (FromForm)
+import qualified Gargantext.API.Node.Corpus.New.File as NewFile
 
 
 type Api = Summary "New Annuaire endpoint"
@@ -45,22 +38,22 @@ type Api = Summary "New Annuaire endpoint"
 
 ------------------------------------------------------------------------
 ------------------------------------------------------------------------
-data WithForm = WithForm
+data AnnuaireWithForm = AnnuaireWithForm
   { _wf_filetype :: !NewFile.FileType
   , _wf_data     :: !Text
   , _wf_lang     :: !(Maybe Lang)
   } deriving (Eq, Show, Generic)
 
-makeLenses ''WithForm
-instance FromForm WithForm
-instance FromJSON WithForm where
+makeLenses ''AnnuaireWithForm
+instance FromForm AnnuaireWithForm
+instance FromJSON AnnuaireWithForm where
   parseJSON = genericParseJSON $ jsonOptions "_wf_"
-instance ToSchema WithForm where
+instance ToSchema AnnuaireWithForm where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wf_")
 
 ------------------------------------------------------------------------
 type AsyncJobs event ctI input output =
-  Flat (AsyncJobsAPI' 'Unsafe 'Safe ctI '[JSON] Maybe event input output)
+  AsyncJobsAPI' 'Unsafe 'Safe ctI '[JSON] Maybe event input output
 ------------------------------------------------------------------------
 
 type AddWithForm = Summary "Add with FormUrlEncoded to annuaire endpoint"
@@ -69,15 +62,15 @@ type AddWithForm = Summary "Add with FormUrlEncoded to annuaire endpoint"
    :> "add"
    :> "form"
    :> "async"
-   :> AsyncJobs ScraperStatus '[FormUrlEncoded] WithForm ScraperStatus
+   :> AsyncJobs JobLog '[FormUrlEncoded] AnnuaireWithForm JobLog
 
 ------------------------------------------------------------------------
 addToAnnuaireWithForm :: FlowCmdM env err m
                     => AnnuaireId
-                    -> WithForm
-                    -> (ScraperStatus -> m ())
-                    -> m ScraperStatus
-addToAnnuaireWithForm _cid (WithForm ft _d _l) logStatus = do
+                    -> AnnuaireWithForm
+                    -> (JobLog -> m ())
+                    -> m JobLog
+addToAnnuaireWithForm _cid (AnnuaireWithForm ft _d _l) logStatus = do
 
   printDebug "ft" ft
 
@@ -88,12 +81,12 @@ addToAnnuaireWithForm _cid (WithForm ft _d _l) logStatus = do
     --   WOS       -> Parser.parseFormat Parser.WOS
     --   PresseRIS -> Parser.parseFormat Parser.RisPresse
 
-  -- docs <- liftIO
+  -- docs <- liftBase
   --       $ splitEvery 500
   --      <$> take 1000000
   --      <$> parse (cs d)
 
-  logStatus ScraperStatus { _scst_succeeded = Just 1
+  logStatus JobLog { _scst_succeeded = Just 1
                           , _scst_failed    = Just 0
                           , _scst_remaining = Just 1
                           , _scst_events    = Just []
@@ -105,7 +98,7 @@ addToAnnuaireWithForm _cid (WithForm ft _d _l) logStatus = do
 
   -- printDebug "cid'" cid'
 
-  pure      ScraperStatus { _scst_succeeded = Just 2
+  pure      JobLog { _scst_succeeded = Just 2
                           , _scst_failed    = Just 0
                           , _scst_remaining = Just 0
                           , _scst_events    = Just []

src/Gargantext/API/Export.hs → src/Gargantext/API/Node/Corpus/Export.hs

 {-|
-Module      : Gargantext.API.Export
+Module      : Gargantext.API.Node.Corpus.Export
 Description : Get Metrics from Storage (Database like)
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -11,48 +11,44 @@ Main exports of Gargantext:
 - corpus
 - document and ngrams
 - lists
-
 -}
 
-
-{-# LANGUAGE DataKinds         #-}
-{-# LANGUAGE DeriveGeneric     #-}
-{-# LANGUAGE FlexibleContexts  #-}
-{-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE RankNTypes        #-}
 {-# LANGUAGE TemplateHaskell   #-}
 {-# LANGUAGE TypeOperators     #-}
 
-module Gargantext.API.Export
+module Gargantext.API.Node.Corpus.Export
   where
 
 import Data.Aeson.TH (deriveJSON)
+import qualified Data.List as List
+import qualified Data.Map as Map
 import Data.Map (Map)
-import Data.Set (Set)
 import Data.Maybe (fromMaybe)
+import Data.Set (Set)
+import qualified Data.Set as Set
 import Data.Swagger
 import Data.Text (Text)
 import GHC.Generics (Generic)
+import Servant
+
 import Gargantext.API.Ngrams
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, mapTermListRoot, getRepo)
-import Gargantext.API.Types (GargNoServer)
+import Gargantext.API.Prelude (GargNoServer)
 import Gargantext.Core.Types --
 import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Config (userMaster)
-import Gargantext.Database.Metrics.NgramsByNode (getNgramsByNodeOnlyUser)
-import Gargantext.Database.Node.Select (selectNodesWithUsername)
+import Gargantext.Database.Action.Metrics.NgramsByNode (getNgramsByNodeOnlyUser)
+import Gargantext.Database.Admin.Config (userMaster)
+import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Node (Node, NodeId, ListId, CorpusId)
+import Gargantext.Database.Prelude (Cmd)
+import Gargantext.Database.Query.Table.Node
+import Gargantext.Database.Query.Table.Node.Select (selectNodesWithUsername)
+import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
+import Gargantext.Database.Query.Table.NodeNode (selectDocNodes)
+import Gargantext.Database.Schema.Node (_node_id, _node_hyperdata)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))
-import Gargantext.Database.Schema.Node (defaultList, HasNodeError)
-import Gargantext.Database.Schema.NodeNode (selectDocNodes)
-import Gargantext.Database.Types.Node (Node, HyperdataDocument(..), NodeId, ListId, CorpusId)
-import Gargantext.Database.Utils (Cmd)
 import Gargantext.Prelude
 import Gargantext.Prelude.Utils (sha)
-import Servant
-import qualified Data.Map as Map
-import qualified Data.Set as Set
-import qualified Data.List as List
 
 
 -- Corpus Export
@@ -141,7 +137,7 @@ getNodeNgrams cId lId' nt repo = do
     Just  l -> pure l
 
   lIds <- selectNodesWithUsername NodeList userMaster
-  let ngs = filterListWithRoot GraphTerm $ mapTermListRoot [lId] nt repo
+  let ngs = filterListWithRoot MapTerm $ mapTermListRoot [lId] nt repo
   r <- getNgramsByNodeOnlyUser cId (lIds <> [lId]) nt (Map.keys ngs)
   pure r
 

src/Gargantext/API/Corpus/New/File.hs → src/Gargantext/API/Node/Corpus/New/File.hs

 {-|
-Module      : Gargantext.API.Corpus.New.File
+Module      : Gargantext.API.Node.Corpus.New.File
 Description : Server API
 Copyright   : (c) CNRS, 2017-Present
 License     : AGPL + CECILL v3
@@ -11,32 +11,24 @@ Portability : POSIX
 
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
-{-# LANGUAGE DataKinds            #-}
-{-# LANGUAGE DeriveGeneric        #-}
-{-# LANGUAGE FlexibleContexts     #-}
-{-# LANGUAGE FlexibleInstances    #-}
-{-# LANGUAGE NoImplicitPrelude    #-}
-{-# LANGUAGE OverloadedStrings    #-}
-{-# LANGUAGE RankNTypes           #-}
 {-# LANGUAGE ScopedTypeVariables  #-}
 {-# LANGUAGE TemplateHaskell      #-}
 {-# LANGUAGE TypeOperators        #-}
 
-module Gargantext.API.Corpus.New.File
+module Gargantext.API.Node.Corpus.New.File
   where
 
 import Control.Lens ((.~), (?~))
 import Control.Monad (forM)
-import Control.Monad.IO.Class (liftIO)
-import Data.Maybe
 import Data.Aeson
+import Data.Maybe
 import Data.Monoid (mempty)
 import Data.Swagger
 import Data.Text (Text())
 import GHC.Generics (Generic)
 import Gargantext.API.Ngrams (TODO)
-import Gargantext.Database.Types.Node
-import Gargantext.Database.Utils -- (Cmd, CmdM)
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Prelude -- (Cmd, CmdM)
 import Gargantext.Prelude
 import Gargantext.Prelude.Utils (sha)
 import Servant
@@ -100,18 +92,18 @@ postUpload :: NodeId
            -> Cmd err [Hash]
 postUpload _ Nothing _ = panic "fileType is a required parameter"
 postUpload _ (Just fileType) multipartData = do
-  putStrLn $ "File Type: " <> (show fileType)
-  is <- liftIO $ do
-    putStrLn ("Inputs:" :: Text)
+  printDebug "File Type: " fileType
+  is <- liftBase $ do
+    printDebug "Inputs:" ()
     forM (inputs multipartData) $ \input -> do
-      putStrLn $ ("iName  " :: Text) <> (iName input)
-               <> ("iValue " :: Text) <> (iValue input)
+      printDebug "iName  " (iName input)
+      printDebug "iValue " (iValue input)
       pure $ iName input
 
   _ <- forM (files multipartData) $ \file -> do
     let content = fdPayload file
-    putStrLn $ ("XXX " :: Text) <> (fdFileName file)
-    putStrLn $ ("YYY " :: Text) <>  cs content
+    printDebug "XXX " (fdFileName file)
+    printDebug "YYY " content
     --pure $ cs content
   -- is <- inputs multipartData
 

src/Gargantext/API/Node/Get.hs

+{-|
+Module      : Gargantext.API.Node.Get
+Description :
+Copyright   : (c) CNRS, 2017-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+Polymorphic Get Node API
+
+-}
+
+{-# LANGUAGE TemplateHaskell    #-}
+{-# LANGUAGE TypeOperators      #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
+module Gargantext.API.Node.Get
+      where
+
+-- import Gargantext.API.Admin.Settings (HasSettings)
+-- import Servant.Job.Async (JobFunction(..), serveJobsAPI)
+-- import Test.QuickCheck (elements)
+-- import Gargantext.Database.Action.Flow.Types (FlowCmdM)
+import Data.Aeson
+import Data.Swagger
+import GHC.Generics (Generic)
+import Gargantext.API.Prelude
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Query.Table.Node (JSONB{-, getNodeWith-})
+import Gargantext.Prelude
+import Servant
+import Test.QuickCheck.Arbitrary
+
+------------------------------------------------------------------------
+type API a = Summary "Polymorphic Get Node Endpoint"
+         :> ReqBody '[JSON] GetNodeParams
+         :> Get '[JSON] (Node a)
+
+------------------------------------------------------------------------
+data GetNodeParams = GetNodeParams { node_id  :: NodeId 
+                                   , nodetype :: NodeType
+                                   }
+    deriving (Generic)
+
+----------------------------------------------------------------------
+api :: forall proxy a.
+    ( JSONB a
+    , FromJSON a
+    , ToJSON   a
+    ) => proxy a -> UserId -> NodeId -> GargServer (API a)
+api _p _uId _nId (GetNodeParams _nId' _nt) = undefined
+
+------------------------------------------------------------------------
+instance FromJSON  GetNodeParams where
+  parseJSON = genericParseJSON (defaultOptions { sumEncoding = ObjectWithSingleField })
+
+instance ToJSON    GetNodeParams where
+  toJSON = genericToJSON (defaultOptions { sumEncoding = ObjectWithSingleField })
+  
+instance ToSchema  GetNodeParams
+instance Arbitrary GetNodeParams where
+  arbitrary = GetNodeParams <$> arbitrary <*> arbitrary
+
+------------------------------------------------------------------------

src/Gargantext/API/Search.hs

@@ -12,34 +12,28 @@ Count API part of Gargantext.
 
 {-# OPTIONS_GHC -fno-warn-name-shadowing #-}
 
-{-# LANGUAGE FlexibleContexts   #-}
-{-# LANGUAGE NoImplicitPrelude  #-}
-{-# LANGUAGE DataKinds          #-}
 {-# LANGUAGE TemplateHaskell    #-}
 {-# LANGUAGE TypeOperators      #-}
-{-# LANGUAGE DeriveGeneric      #-}
 {-# LANGUAGE DeriveAnyClass     #-}
-{-# LANGUAGE OverloadedStrings  #-}
-{-# LANGUAGE RankNTypes         #-}
 
 module Gargantext.API.Search
       where
 
-import GHC.Generics (Generic)
-import Data.Time (UTCTime)
 import Data.Aeson.TH (deriveJSON)
 import Data.Swagger
 import Data.Text (Text)
+import Data.Time (UTCTime)
+import GHC.Generics (Generic)
+import Gargantext.API.Prelude (GargServer)
+import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
+import Gargantext.Database.Query.Facet
+import Gargantext.Database.Action.Search
+import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument)
+import Gargantext.Database.Admin.Types.Node
+import Gargantext.Prelude
 import Servant
-import Test.QuickCheck.Arbitrary
 import Test.QuickCheck (elements)
--- import Control.Applicative ((<*>))
-import Gargantext.API.Types (GargServer)
-import Gargantext.Prelude
-import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
-import Gargantext.Database.Types.Node
-import Gargantext.Database.TextSearch
-import Gargantext.Database.Facet
+import Test.QuickCheck.Arbitrary
 
 -----------------------------------------------------------------------
 data SearchQuery = SearchQuery

src/Gargantext/Core.hs

@@ -9,10 +9,7 @@ Portability : POSIX
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude  #-}
-{-# LANGUAGE DeriveGeneric      #-}
 {-# LANGUAGE TemplateHaskell    #-}
-{-# LANGUAGE OverloadedStrings #-}
 
 module Gargantext.Core
   where

src/Gargantext/Core/Statistics.hs

@@ -9,7 +9,6 @@ Portability : POSIX
 
 -}
 
-{-# LANGUAGE NoImplicitPrelude #-}
 
 module Gargantext.Core.Statistics
   where