Merge branch 'dev' into dev-sources-chart-sort

.gitlab-ci.yml

@@ -2,7 +2,8 @@
 # https://vadosware.io/post/zero-to-continuous-integrated-testing-a-haskell-project-with-gitlab/
 #
 #
-image: cgenie/stack-build:lts-17.13-garg
+image: cgenie/stack-build:lts-18.18-garg
+#image: cgenie/nixos-stack:latest
 
 variables:
   STACK_ROOT: "${CI_PROJECT_DIR}/.stack-root"
@@ -13,16 +14,31 @@ variables:
   #- apt-get install make xz-utils
 
 stages:
+  - deps
   - docs
   - test
 
+deps:
+  cache:
+    # cache per branch name
+    # key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - .stack
+      - .stack-root/
+      - .stack-work/
+      - target
+  script:
+    - stack build --no-terminal --haddock --no-haddock-deps --only-dependencies --fast
+
 docs:
   cache:
     # cache per branch name
     # key: ${CI_COMMIT_REF_SLUG}
     paths:
+      - .stack
       - .stack-root/
       - .stack-work/
+      - target
   script:
     - stack build --no-terminal --haddock --no-haddock-deps --fast
     - cp -R "$(stack path --local-install-root)"/doc ./output
@@ -36,39 +52,12 @@ test:
     # cache per branch name
     # key: ${CI_COMMIT_REF_SLUG}
     paths:
+      - .stack
       - .stack-root/
       - .stack-work/
+      - target
   script:
     - stack test --no-terminal --fast
 
       # TOOO
-#unit-test:
-#  stage: test
-#  script:
-#    - make test-unit
-#
-#int-test:
-#  stage: test
-#  script:
-#    - make test-int
-#
-#e2e-test:
-#  stage: test
-#  script:
-#    - make test-e2e
-#
-
-# If you find yourself with a non-sensical build error when you know your project should be building just fine, this fragment should help:
-#
-#build:
-#  stage: build
-#  script:
-#    # Clear out cache files
-#    - rm -rf .stack
-#    - rm -rf .stack-work
-#    - stack setup --system-ghc
-#    - stack install --local-bin-path target --system-ghc
-
-
-
 

CHANGELOG.md

+## Version 0.0.4.9.9.4
+* [FEAT] Corpus docs download
+
+## Version 0.0.4.9.9.3
+* [BACK] Graph update with force option
+
+## Version 0.0.4.9.9.2
+* [BACK] Opaleye Upgrade
+
+## Version 0.0.4.9.9.1
+* [FRONT] 350-dev-graph-search-in-forms-not-labels
+* [FRONT] 359-dev-input-with-autocomplete
+
+## Version 0.0.4.9.9
+* [FIX] Continuous Integration (CI)
+
+## Version 0.0.4.9.8
+* [FEAT] All backend routes with clients functions
+
+## Version 0.0.4.9.7
+* [FEAT] Searx API done (needs a fix for language selection)
+
+## Version 0.0.4.9.6
+* [UX] GT.query forces trees reload for async tasks
+
+## Version 0.0.4.9.5
+* [FEAT] Order 2 fixed with filtered edges
+
+## Version 0.0.4.9.4
+* [FEAT] Order 1 similarity validated and optimized
+
+## Version 0.0.4.9.3
+* [FIX] Node Calc import + more flexible delimiter for CSV parser
+
+## Version 0.0.4.9.2
+* [FEAT] Node Calc Parsing added (in tests)
+
+## Version 0.0.4.9.1
+* [FIX] Graph Screenshot
+
+## Version 0.0.4.9
+* [FEAT] Graph with order 1 and order 2 and node size
+
+## Version 0.0.4.8.9
+* BACKEND: fix psql function util without sensitive data
+* FRONTEND: fix folder navigation (up link)
+
+## Version 0.0.4.8.8
+* FIX for CI
+
+## Version 0.0.4.8.7
+* FIX the graph generation (automatic/default, renewal, any distance)
+
+## Version 0.0.4.8.6
+* FIX the ngrams grouping
+
+## Version 0.0.4.8.5
+* Unary document insertion: Doc table is reloaded after upload
+
+## Version 0.0.4.8.4
+* Migration: instance dev is now dev.sub.gargantext.org
 
 ## Version 0.0.4.1
 * Refact/code design better syntax for DataType fields

README.md

@@ -196,3 +196,36 @@ To build documentation, run:
 stack --docker build --haddock --no-haddock-deps --fast
 ```
 
+## GraphQL
+
+Some introspection information.
+
+Playground is located at http://localhost:8008/gql
+
+### List all GraphQL types in the Playground
+
+```
+{
+  __schema {
+    types {
+      name
+    }
+  }
+}
+```
+
+### List details about a type in GraphQL
+
+```
+{
+  __type(name:"User") {
+  	fields {
+    	name
+      description
+      type {
+        name
+      }
+  	}
+	}
+}
+```

bin/gargantext-cbor2json/Main.hs

-import Prelude (IO, id, (.))
+
+import System.Environment (getArgs)
+import Prelude (IO, id, (.), ($))
 import Data.Aeson (encode)
 import Codec.Serialise (deserialise)
 import qualified Data.ByteString.Lazy as L
-
-import Gargantext.API.Ngrams.Types (NgramsRepo)
+import Gargantext.Core.NodeStory (NodeListStory)
 
 main :: IO ()
-main = L.interact (encode . (id :: NgramsRepo -> NgramsRepo) . deserialise)
+main = L.interact (encode . (id :: NodeListStory -> NodeListStory) . deserialise)

bin/gargantext-import/Main.hs

@@ -47,13 +47,13 @@ main = do
     tt = (Multi EN)
     format = CsvGargV3 -- CsvHal --WOS
     corpus :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    corpus = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt  format corpusPath Nothing
+    corpus = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt  format corpusPath Nothing (\_ -> pure ())
 
     corpusCsvHal :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    corpusCsvHal = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt CsvHal corpusPath Nothing
+    corpusCsvHal = flowCorpusFile (UserName $ cs user) (Left (cs name :: Text)) (read limit :: Int) tt CsvHal corpusPath Nothing (\_ -> pure ())
 
     annuaire :: forall m. FlowCmdM DevEnv GargError m => m CorpusId
-    annuaire = flowAnnuaire (UserName $ cs user) (Left "Annuaire") (Multi EN) corpusPath
+    annuaire = flowAnnuaire (UserName $ cs user) (Left "Annuaire") (Multi EN) corpusPath (\_ -> pure ())
 
   {-
   let debatCorpus :: forall m. FlowCmdM DevEnv GargError m => m CorpusId

bin/install

 #!/bin/bash
 
-stack install --nix --profile --test --fast --no-install-ghc --skip-ghc-check
+#stack install --nix --profile --test --fast --no-install-ghc --skip-ghc-check
+stack install --nix --test --no-install-ghc --skip-ghc-check

bin/psql

 #!/bin/bash
 
-psql postgresql://gargantua:C8kdcUrAQy66U12341@localhost/gargandbV5
+INIFILE=$1
+
+getter () {
+  grep $1 $INIFILE | sed "s/^.*= //"
+}
+
+connect () {
+USER=$(getter "DB_USER")
+NAME=$(getter "DB_NAME")
+PASS=$(getter "DB_PASS")
+HOST=$(getter "DB_HOST")
+PORT=$(getter "DB_PORT")
+psql "postgresql://${USER}:${PASS}@${HOST}:${PORT}/${NAME}"
+}
+
+if [[ $1 == "" ]]
+then echo "USAGE : ./psql gargantext.ini"
+else connect $INIFILE
+fi
+
+
 

cabal.project

 packages: .
-
-allow-newer: base, accelerate
+          -- ../servant-job
+          -- ../ekg-json
+          -- ../../../code/servant/servant
+          -- ../../../code/servant/servant-server
+          -- ../../../code/servant/servant-client-core
+          -- ../../../code/servant/servant-client
+          -- ../../../code/servant/servant-auth/servant-auth
+          -- ../../../code/servant/servant-auth/servant-auth-client
+          -- ../../../code/servant/servant-auth/servant-auth-server
+
+allow-newer: base, accelerate, servant, time
 
 -- Patches
+source-repository-package
+  type: git
+  location: https://github.com/alpmestan/servant-job.git
+  tag: ceb251b91e8ec1804198422a3cdbdab08d843b79
+
+source-repository-package
+  type: git
+  location: https://github.com/alpmestan/ekg-json.git
+  tag: c7bde4851a7cd41b3f3debf0c57f11bbcb11d698
+
+source-repository-package
+  type: git
+  location: https://github.com/haskell-servant/servant.git
+  tag: c2af6e775d1d36f2011d43aff230bb502f8fba63
+  subdir: servant/
+          servant-server/
+          servant-client-core/
+          servant-client/
+          servant-auth/servant-auth/
+          servant-auth/servant-auth-client/
+          servant-auth/servant-auth-server/
+
 source-repository-package
   type: git
   location: https://github.com/delanoe/patches-map.git
@@ -13,6 +44,11 @@ source-repository-package
   location: https://gitlab.iscpif.fr/gargantext/patches-class.git
   tag: 271ba32d6c940029dc653354dd7974a819f48e77
 
+source-repository-package
+  type: git
+  location: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude.git
+  tag: 6bfdb29e9a576472c7fd7ebe648ad101e5b3927f
+
 -- External Data API connectors
 source-repository-package
   type: git
@@ -67,16 +103,16 @@ source-repository-package
   location: https://github.com/delanoe/servant-static-th.git
   tag: 8cb8aaf2962ad44d319fcea48442e4397b3c49e8
 
-source-repository-package
-  type: git
-  location: https://github.com/alpmestan/servant-job.git
-  tag: e9a4c57ca3ddee450627ed251df942effb27e4be
+-- source-repository-package
+--   type: git
+--   location: https://github.com/alpmestan/servant-job.git
+--   tag: e9a4c57ca3ddee450627ed251df942effb27e4be
 
 -- Database libraries
 source-repository-package
   type: git
   location: https://github.com/delanoe/haskell-opaleye.git
-  tag: 63ee65d974e9d20eaaf17a2e83652175988cbb79
+  tag: d3ab7acd5ede737478763630035aa880f7e34444
 
 source-repository-package
   type: git
@@ -100,4 +136,14 @@ source-repository-package
   location: https://gitlab.iscpif.fr/anoe/accelerate-utility.git
   tag: 83ada76e78ac10d9559af8ed6bd4064ec81308e4
 
-constraints: unordered-containers==0.2.13.*
\ No newline at end of file
+
+-- Wikidata 
+source-repository-package
+  type: git
+  location: https://github.com/rspeer/wikiparsec.git
+  tag: 9637a82344bb70f7fa8f02e75db3c081ccd434ce
+
+
+constraints: unordered-containers==0.2.14.*,
+             servant-ekg==0.3.1,
+             time==1.9.3

devops/docker/Dockerfile

-FROM fpco/stack-build:lts-18.12
+FROM fpco/stack-build:lts-18.18
 
 #RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 8B1DA6120C2BF624
 RUN apt-get update && \
-    apt-get install -y git libigraph0-dev && \
+    apt-get install -y ca-certificates git libigraph0-dev && \
     rm -rf /var/lib/apt/lists/*
 

devops/postgres/tools/removeUnusedDocuments.sql

+
+WITH repeated AS
+  ( select nn.node2_id AS id, count(*) AS c
+    FROM nodes_nodes nn 
+    GROUP BY nn.node2_id
+  )
+
+DELETE FROM nodes n 
+USING repeated r
+WHERE
+n.id = r.id
+AND r.c <= 1
+AND n.typename = 4
+; 
+

devops/postgres/tools/removeUnusedNgrams.sql

+
+WITH listed AS
+  ( select nn.ngrams_id AS id, count(*) AS c
+    FROM node_node_ngrams nn 
+    GROUP BY nn.ngrams_id
+  )
+
+--SELECT count(*) from listed l
+-- WHERE
+--l.c <= 1
+
+
+DELETE FROM ngrams n 
+  USING listed l
+  WHERE
+  n.id = l.id
+  AND l.c <= 1
+; 
+
+
+

ekg-assets/index.html

+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <link rel="stylesheet" href="bootstrap-1.4.0.min.css">
+    <link rel="stylesheet" href="monitor.css" type="text/css">
+    <script type="text/javascript" src="jquery-1.6.4.min.js"></script>
+    <script type="text/javascript" src="jquery.flot.min.js"></script>
+    <title>ekg</title>
+  </head>
+  <body>
+    <div class="topbar">
+      <div class="topbar-inner">
+        <div class="container-fluid">
+          <span class="brand">ekg</span>
+          <p class="pull-right">Polling interval:
+            <select id="updateInterval" class="small">
+              <option value="100">100 ms</option>
+              <option value="200">200 ms</option>
+              <option value="500">500 ms</option>
+              <option value="1000" selected="selected">1 s</option>
+              <option value="2000">2 s</option>
+              <option value="5000">5 s</option>
+              <option value="10000">10 s</option>
+            </select> |
+            <button id="pause-ui" class="btn">Pause UI</button>
+          </p>
+        </div>
+      </div>
+    </div>
+
+    <div class="container">
+      <div class="row">
+        <div class="alert-message error fade in hide" data-alert="alert">
+          <p>Lost connection to server.</p>
+        </div>
+      </div>
+
+      <div class="row">
+        <div id="plots" class="span11">
+          <div id="current-bytes-used-plot" class="plot-container">
+            <h3>Current residency</h3>
+            <div class="plot"></div>
+          </div>
+
+          <div id="allocation-rate-plot" class="plot-container">
+            <h3>Allocation rate</h3>
+            <div class="plot"></div>
+          </div>
+
+          <div id="productivity-plot" class="plot-container">
+            <h3>Productivity</h3>
+            <div class="plot"></div>
+          </div>
+        </div>
+
+        <div class="span5">
+          <h3>GC and memory statistics</h3>
+          <table class="condensed-table">
+            <thead>
+              <tr>
+                <th>Statistic</th>
+                <th>Value</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td>Maximum residency</td>
+                <td id="max-bytes-used" class="span3 value">0</td>
+              </tr>
+              <tr>
+                <td>Current residency</td>
+                <td id="current-bytes-used" class="value">0</td>
+              </tr>
+              <tr>
+                <td>Maximum slop</td>
+                <td id="max-bytes-slop" class="value">0</td>
+              </tr>
+              <tr>
+                <td>Current slop</td>
+                <td id="current-bytes-slop" class="value">0</td>
+              </tr>
+              <tr>
+                <td>Productivity (wall clock time)</td>
+                <td id="productivity-wall" class="value">0</td>
+              </tr>
+              <tr>
+                <td>Productivity (cpu time)</td>
+                <td id="productivity-cpu" class="value">0</td>
+              </tr>
+              <tr>
+                <td>Allocation rate</td>
+                <td id="allocation-rate" class="value">0</td>
+              </tr>
+            </tbody>
+          </table>
+
+          <h3>Metrics</h3>
+          <table id="metric-table" class="condensed-table">
+            <thead>
+              <tr>
+                <th class="span4">Name</th>
+                <th class="span1">Value</th>
+              </tr>
+            </thead>
+            <tbody>
+            </tbody>
+          </table>
+
+        </div>
+      </div>
+    </div>
+
+    <script type="text/javascript" src="monitor.js"></script>
+  </body>
+</html>

ekg-assets/monitor.css

+/**
+ * Blueprint/flot compatibility
+ *
+ * Resets some styles back to the browser default.
+ */
+
+.plot table {
+    width: auto;
+    border-spacing: 2px;
+}
+
+.plot th,
+.plot td,
+.plot caption {
+    padding: 0;
+}
+
+/**
+ * Body margin
+ */
+
+body {
+    padding-top: 60px;
+}
+
+/**
+ * Plots
+ */
+
+.plot {
+    width: 600px;
+    height: 300px;
+    margin-bottom: 1.5em;
+}
+
+.close-button {
+    float: right;
+    cursor: pointer;
+}
+
+/**
+ * Table
+ */
+
+.value {
+    text-align: right;
+}
+
+.string {
+    text-align: left;
+}
+
+.graph-button {
+    cursor: pointer;
+    vertical-align: middle;
+}

gargantext.ini_toModify

@@ -67,3 +67,12 @@ DB_PASS = PASSWORD_TO_CHANGE
 LOG_FILE = /var/log/gargantext/backend.log
 LOG_LEVEL = LevelDebug
 LOG_FORMATTER = verbose
+
+[mail]
+MAIL_PORT = 25
+MAIL_HOST = localhost
+MAIL_USER = gargantext
+MAIL_PASSWORD = 
+MAIL_FROM     = 
+# NoAuth | Normal | SSL | TLS | STARTTLS
+MAIL_LOGIN_TYPE = Normal

nix/pkgs.nix

@@ -26,6 +26,8 @@ rec {
     blas
     gfortran7
     #    gfortran7.cc.lib
+    expat
+    icu
   ];
   libPaths = pkgs.lib.makeLibraryPath nonhsBuildInputs;
   shellHook = ''

package.yaml

 name: gargantext
-version: '0.0.4.4'
+version: '0.0.4.9.9.4'
 synopsis: Search, map, share
 description: Please see README.md
 category: Data
@@ -25,6 +25,15 @@ default-extensions:
   - OverloadedStrings
   - RankNTypes
   - RecordWildCards
+data-files:
+  - ekg-assets/index.html
+  - ekg-assets/monitor.js
+  - ekg-assets/monitor.css
+  - ekg-assets/jquery.flot.min.js
+  - ekg-assets/jquery-1.6.4.min.js
+  - ekg-assets/bootstrap-1.4.0.min.css
+  - ekg-assets/chart_line_add.png
+  - ekg-assets/cross.png
 library:
   source-dirs: src
   ghc-options:
@@ -49,6 +58,7 @@ library:
   - Gargantext.API.Admin.EnvTypes
   - Gargantext.API.Admin.Types
   - Gargantext.API.Prelude
+  - Gargantext.API.Client
   - Gargantext.Core
   - Gargantext.Core.NodeStory
   - Gargantext.Core.Methods.Distances
@@ -137,6 +147,8 @@ library:
   - deepseq
   - directory
   - duckling
+  - ekg-core
+  - ekg-json
   - exceptions
   - fast-logger
   - fclabels
@@ -147,6 +159,7 @@ library:
   - full-text-search
   - fullstop
   - gargantext-prelude
+    #  - gargantext-graph >= 0.1.0.0
   - graphviz
   - hashable
   - haskell-igraph
@@ -170,6 +183,10 @@ library:
   - matrix
   - monad-control
   - monad-logger
+  - morpheus-graphql
+  - morpheus-graphql-app
+  - morpheus-graphql-core
+  - morpheus-graphql-subscriptions
   - mtl
   - natural-transformation
   - opaleye
@@ -201,11 +218,14 @@ library:
   - serialise
   - servant
   - servant-auth
+  - servant-auth-client
   - servant-auth-server >= 0.4.4.0
   - servant-auth-swagger
   - servant-blaze
   - servant-cassava
   - servant-client
+  - servant-ekg
+  - servant-flatten
   - servant-job
   - servant-mock
   - servant-multipart
@@ -219,15 +239,18 @@ library:
   - split
   - stemmer
   - swagger2
+  - taggy-lens
   - tagsoup
   - template-haskell
   - temporary
+  - text-conversions
   - text-metrics
   - time
   - time-locale-compat
   - timezone-series
   - transformers
   - transformers-base
+  - tuple
   - unordered-containers
   - utf8-string
   - uuid
@@ -237,7 +260,10 @@ library:
   - wai-app-static
   - wai-cors
   - wai-extra
+  - wai-websockets
   - warp
+  - wikiparsec
+  - websockets
   - wreq
   - xml-conduit
   - xml-types
@@ -257,6 +283,7 @@ executables:
     - -rtsopts
     - -threaded
     - -with-rtsopts=-N
+    - -with-rtsopts=-T
     - -fprof-auto
     dependencies:
       - base
@@ -352,19 +379,19 @@ executables:
       - gargantext-prelude
       - base
 
-  gargantext-upgrade:
-    main: Main.hs
-    source-dirs: bin/gargantext-upgrade
-    ghc-options:
-    - -threaded
-    - -rtsopts
-    - -with-rtsopts=-N
-    - -O2
-    - -Wmissing-signatures
-    dependencies:
-      - gargantext
-      - gargantext-prelude
-      - base
+#  gargantext-upgrade:
+#    main: Main.hs
+#    source-dirs: bin/gargantext-upgrade
+#    ghc-options:
+#    - -threaded
+#    - -rtsopts
+#    - -with-rtsopts=-N
+#    - -O2
+#    - -Wmissing-signatures
+#    dependencies:
+#      - gargantext
+#      - gargantext-prelude
+#      - base
 
   gargantext-admin:
     main: Main.hs
@@ -380,6 +407,7 @@ executables:
       - gargantext-prelude
       - base
 
+
   gargantext-cbor2json:
     main: Main.hs
     source-dirs: bin/gargantext-cbor2json

shell.nix

+{ pkgs ? import ./nix/pkgs.nix {} }:
+let 
+  myBuildInputs = [
+    pkgs.pkgs.docker-compose
+    pkgs.pkgs.haskell-language-server
+    pkgs.pkgs.stack
+  ];
+in  
+pkgs.pkgs.mkShell {
+  name = pkgs.shell.name;
+  shellHook = pkgs.shell.shellHook;
+  buildInputs = pkgs.shell.buildInputs ++ myBuildInputs;
+}

src/Gargantext/API.hs

@@ -27,7 +27,7 @@ Pouillard (who mainly made it).
 -}
 
 {-# LANGUAGE ScopedTypeVariables  #-}
-
+{-# LANGUAGE TypeOperators        #-}
 module Gargantext.API
       where
 
@@ -43,6 +43,7 @@ import GHC.Generics (Generic)
 import Gargantext.API.Admin.Auth.Types (AuthContext)
 import Gargantext.API.Admin.Settings (newEnv)
 import Gargantext.API.Admin.Types (FireWall(..), PortNumber, cookieSettings, jwtSettings, settings)
+import Gargantext.API.EKG
 import Gargantext.API.Ngrams (saveNodeStory)
 import Gargantext.API.Prelude
 import Gargantext.API.Routes
@@ -54,9 +55,9 @@ import Network.Wai
 import Network.Wai.Handler.Warp hiding (defaultSettings)
 import Network.Wai.Middleware.Cors
 import Network.Wai.Middleware.RequestLogger
+import Paths_gargantext (getDataDir)
 import Servant
-import System.IO (FilePath)
-
+import System.FilePath
 
 data Mode = Dev | Mock | Prod
   deriving (Show, Read, Generic)
@@ -191,8 +192,14 @@ serverGargAdminAPI =  roots
 --gargMock :: Server GargAPI
 --gargMock = mock apiGarg Proxy
 ---------------------------------------------------------------------
-makeApp :: EnvC env => env -> IO Application
-makeApp env = serveWithContext api cfg <$> server env
+
+makeApp :: (Typeable env, EnvC env) => env -> IO Application
+makeApp env = do
+  serv <- server env
+  (ekgStore, ekgMid) <- newEkgStore api
+  ekgDir <- (</> "ekg-assets") <$> getDataDir
+  return $ ekgMid $ serveWithContext apiWithEkg cfg
+    (ekgServer ekgDir ekgStore :<|> serv)
   where
     cfg :: Servant.Context AuthContext
     cfg = env ^. settings . jwtSettings
@@ -206,6 +213,9 @@ makeApp env = serveWithContext api cfg <$> server env
 api :: Proxy API
 api  = Proxy
 
+apiWithEkg :: Proxy (EkgAPI :<|> API)
+apiWithEkg = Proxy
+
 apiGarg :: Proxy GargAPI
 apiGarg  = Proxy
 ---------------------------------------------------------------------

src/Gargantext/API/Admin/Auth.hs

@@ -20,6 +20,7 @@ TODO-ACCESS Critical
 
 -}
 
+{-# LANGUAGE MonoLocalBinds      #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
 module Gargantext.API.Admin.Auth
@@ -35,17 +36,18 @@ import Servant
 import Servant.Auth.Server
 import qualified Gargantext.Prelude.Crypto.Auth as Auth
 
-import Gargantext.API.Admin.Types
 import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Types
 import Gargantext.API.Prelude (HasJoseError(..), joseError, HasServerError, GargServerC)
+import Gargantext.Core.Mail.Types (HasMail)
 import Gargantext.Core.Types.Individu (User(..), Username, GargPassword(..))
+import Gargantext.Database.Admin.Types.Node (NodeId(..), UserId)
+import Gargantext.Database.Prelude (Cmd', CmdM, HasConnectionPool, HasConfig)
+import Gargantext.Database.Query.Table.User
 import Gargantext.Database.Query.Tree (isDescendantOf, isIn)
 import Gargantext.Database.Query.Tree.Root (getRoot)
 import Gargantext.Database.Schema.Node (NodePoly(_node_id))
-import Gargantext.Database.Admin.Types.Node (NodeId(..), UserId)
-import Gargantext.Database.Prelude (Cmd', CmdM, HasConnectionPool, HasConfig)
 import Gargantext.Prelude hiding (reverse)
-import Gargantext.Database.Query.Table.User
 
 ---------------------------------------------------
 
@@ -60,7 +62,7 @@ makeTokenForUser uid = do
   either joseError (pure . toStrict . decodeUtf8) e
   -- TODO not sure about the encoding...
 
-checkAuthRequest :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env)
+checkAuthRequest :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env, HasMail env)
                  => Username
                  -> GargPassword
                  -> Cmd' env err CheckAuth
@@ -79,7 +81,7 @@ checkAuthRequest u (GargPassword p) = do
               token <- makeTokenForUser uid
               pure $ Valid token uid
 
-auth :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env)
+auth :: (HasSettings env, HasConnectionPool env, HasJoseError err, HasConfig env, HasMail env)
      => AuthRequest -> Cmd' env err AuthResponse
 auth (AuthRequest u p) = do
   checkAuthRequest' <- checkAuthRequest u p

src/Gargantext/API/Admin/EnvTypes.hs

@@ -14,24 +14,25 @@ import Servant.Job.Async (HasJobEnv(..), Job)
 import System.Log.FastLogger
 import qualified Servant.Job.Core
 
-import Gargantext.API.Ngrams.Types (HasRepoVar(..), HasRepoSaver(..), HasRepo(..), RepoEnv(..))
 import Gargantext.API.Admin.Types
 import Gargantext.API.Admin.Orchestrator.Types
+import Gargantext.Core.NodeStory
+import Gargantext.Core.Mail.Types (HasMail, mailSettings)
 import Gargantext.Database.Prelude (HasConnectionPool(..), HasConfig(..))
 import Gargantext.Prelude
 import Gargantext.Prelude.Config (GargConfig(..))
-import Gargantext.Core.NodeStory
+import Gargantext.Prelude.Mail.Types (MailConfig)
 
 data Env = Env
   { _env_settings  :: !Settings
   , _env_logger    :: !LoggerSet
   , _env_pool      :: !(Pool Connection)
-  , _env_repo      :: !RepoEnv
   , _env_nodeStory :: !NodeStoryEnv
   , _env_manager   :: !Manager
   , _env_self_url  :: !BaseUrl
   , _env_scrapers  :: !ScrapersEnv
   , _env_config    :: !GargConfig
+  , _env_mail      :: !MailConfig
   }
   deriving (Generic)
 
@@ -55,15 +56,8 @@ instance HasNodeStorySaver Env where
 instance HasSettings Env where
   settings = env_settings
 
--- Specific to Repo
-instance HasRepoVar Env where
-  repoVar = repoEnv . repoVar
-instance HasRepoSaver Env where
-  repoSaver = repoEnv . repoSaver
-instance HasRepo Env where
-  repoEnv = env_repo
-
-
+instance HasMail Env where
+  mailSettings = env_mail
 
 
 instance Servant.Job.Core.HasEnv Env (Job JobLog JobLog) where
@@ -83,10 +77,10 @@ makeLenses ''MockEnv
 
 data DevEnv = DevEnv
   { _dev_env_settings  :: !Settings
-  , _dev_env_repo      :: !RepoEnv
   , _dev_env_config    :: !GargConfig
   , _dev_env_pool      :: !(Pool Connection)
   , _dev_env_nodeStory :: !NodeStoryEnv
+  , _dev_env_mail      :: !MailConfig
   }
 
 makeLenses ''DevEnv
@@ -110,12 +104,5 @@ instance HasNodeStoryVar DevEnv where
 instance HasNodeStorySaver DevEnv where
   hasNodeStorySaver = hasNodeStory . nse_saver
 
-
-instance HasRepoVar DevEnv where
-  repoVar = repoEnv . repoVar
-instance HasRepoSaver DevEnv where
-  repoSaver = repoEnv . repoSaver
-instance HasRepo DevEnv where
-  repoEnv = dev_env_repo
-
-
+instance HasMail DevEnv where
+  mailSettings = dev_env_mail

src/Gargantext/API/Admin/Orchestrator/Types.hs

@@ -7,6 +7,9 @@ module Gargantext.API.Admin.Orchestrator.Types
 
 import Control.Lens hiding (elements)
 import Data.Aeson
+import Data.Morpheus.Types
+  ( GQLType
+  , typeOptions )
 import Data.Proxy
 import Data.Swagger hiding (URL, url, port)
 import Data.Text (Text)
@@ -18,6 +21,7 @@ import Servant.Job.Utils (jsonOptions)
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Arbitrary
 
+import qualified Gargantext.API.GraphQL.Utils as GQLU
 import Gargantext.Core.Types (TODO(..))
 import Gargantext.Prelude
 
@@ -84,13 +88,13 @@ instance Arbitrary ScraperEvent where
   arbitrary = ScraperEvent <$> elements [Nothing, Just "test message"]
                            <*> elements [Nothing, Just "INFO", Just "WARN"]
                            <*> elements [Nothing, Just "2018-04-18"]
-
 instance ToJSON ScraperEvent where
   toJSON = genericToJSON $ jsonOptions "_scev_"
-
 instance FromJSON ScraperEvent where
   parseJSON = genericParseJSON $ jsonOptions "_scev_"
-
+instance ToSchema ScraperEvent  -- TODO _scev_ prefix
+instance GQLType ScraperEvent where
+  typeOptions _ = GQLU.unPrefix "_scev_"
 
 
 data JobLog = JobLog
@@ -109,17 +113,15 @@ instance Arbitrary JobLog where
            <*> arbitrary
            <*> arbitrary
            <*> arbitrary
-
 instance ToJSON JobLog where
   toJSON = genericToJSON $ jsonOptions "_scst_"
-
 instance FromJSON JobLog where
   parseJSON = genericParseJSON $ jsonOptions "_scst_"
-
 instance ToSchema JobLog -- TODO _scst_ prefix
+instance GQLType JobLog where
+  typeOptions _ = GQLU.unPrefix "_scst_"
 
 instance ToSchema ScraperInput  -- TODO _scin_ prefix
-instance ToSchema ScraperEvent  -- TODO _scev_ prefix
 
 instance ToParamSchema Offset -- where
   -- toParamSchema = panic "TODO"

src/Gargantext/API/Admin/Settings.hs

@@ -18,9 +18,8 @@ TODO-SECURITY: Critical
 module Gargantext.API.Admin.Settings
     where
 
-import Codec.Serialise (Serialise(), serialise, deserialise)
-import Control.Concurrent
-import Control.Debounce (mkDebounce, defaultDebounceSettings, debounceFreq, debounceAction)
+-- import Control.Debounce (mkDebounce, defaultDebounceSettings, debounceFreq, debounceAction)
+import Codec.Serialise (Serialise(), serialise)
 import Control.Lens
 import Control.Monad.Logger
 import Control.Monad.Reader
@@ -34,7 +33,7 @@ import Servant.Auth.Server (defaultJWTSettings, CookieSettings(..), XsrfCookieSe
 import Servant.Client (parseBaseUrl)
 import Servant.Job.Async (newJobEnv, defaultSettings)
 import System.Directory
-import System.FileLock (tryLockFile, unlockFile, SharedExclusive(Exclusive))
+-- import System.FileLock (tryLockFile, unlockFile, SharedExclusive(Exclusive))
 import System.IO (FilePath, hClose)
 import System.IO.Temp (withTempFile)
 import System.Log.FastLogger
@@ -43,10 +42,11 @@ import qualified Data.ByteString.Lazy as L
 
 import Gargantext.API.Admin.EnvTypes
 import Gargantext.API.Admin.Types
-import Gargantext.API.Ngrams.Types (NgramsRepo, HasRepo(..), RepoEnv(..), r_version, initRepo, renv_var, renv_lock)
-import Gargantext.Database.Prelude (databaseParameters, HasConfig(..))
+-- import Gargantext.API.Ngrams.Types (NgramsRepo, HasRepo(..), RepoEnv(..), r_version, initRepo, renv_var, renv_lock)
+import Gargantext.Database.Prelude (databaseParameters)
 import Gargantext.Prelude
-import Gargantext.Prelude.Config (gc_repofilepath)
+-- import Gargantext.Prelude.Config (gc_repofilepath)
+import qualified Gargantext.Prelude.Mail as Mail
 
 devSettings :: FilePath -> IO Settings
 devSettings jwkFile = do
@@ -113,7 +113,7 @@ repoSaverAction repoDir a = do
 
 
 
---{-
+{-
 -- The use of mkDebounce makes sure that repoSaverAction is not called too often.
 -- If repoSaverAction start taking more time than the debounceFreq then it should
 -- be increased.
@@ -133,6 +133,8 @@ mkRepoSaver repoDir repo_var = mkDebounce settings'
                    -- Add a new MVar just for saving.
                  }
 
+-}
+{-
 readRepoEnv :: FilePath -> IO RepoEnv
 readRepoEnv repoDir = do
   -- Does file exist ? :: Bool
@@ -178,27 +180,27 @@ newEnv port file = do
   self_url_env  <- parseBaseUrl $ "http://0.0.0.0:" <> show port
   dbParam       <- databaseParameters file
   pool          <- newPool dbParam
-  repo          <- readRepoEnv (_gc_repofilepath config_env)
   nodeStory_env <- readNodeStoryEnv (_gc_repofilepath config_env)
   scrapers_env  <- newJobEnv defaultSettings manager_env
   logger        <- newStderrLoggerSet defaultBufSize
+  config_mail   <- Mail.readConfig file
 
   pure $ Env
     { _env_settings  = settings'
     , _env_logger    = logger
     , _env_pool      = pool
-    , _env_repo      = repo
     , _env_nodeStory = nodeStory_env
     , _env_manager   = manager_env
     , _env_scrapers  = scrapers_env
     , _env_self_url  = self_url_env
     , _env_config    = config_env
+    , _env_mail      = config_mail
     }
 
 newPool :: ConnectInfo -> IO (Pool Connection)
 newPool param = createPool (connect param) close 1 (60*60) 8
 
---{-
+{-
 cleanEnv :: (HasConfig env, HasRepo env) => env -> IO ()
 cleanEnv env = do
   r <- takeMVar (env ^. repoEnv . renv_var)

src/Gargantext/API/Dev.hs

@@ -23,6 +23,7 @@ import Gargantext.Core.NodeStory
 import Gargantext.Database.Prelude
 import Gargantext.Prelude
 import Gargantext.Prelude.Config (GargConfig(..), readConfig)
+import qualified Gargantext.Prelude.Mail as Mail
 import Servant
 import System.IO (FilePath)
 
@@ -31,7 +32,7 @@ type IniPath  = FilePath
 withDevEnv :: IniPath -> (DevEnv -> IO a) -> IO a
 withDevEnv iniPath k = do
   env <- newDevEnv
-  k env `finally` cleanEnv env
+  k env -- `finally` cleanEnv env
 
   where
     newDevEnv = do
@@ -39,14 +40,14 @@ withDevEnv iniPath k = do
       dbParam <- databaseParameters iniPath
       nodeStory_env <- readNodeStoryEnv (_gc_repofilepath cfg)
       pool    <- newPool            dbParam
-      repo    <- readRepoEnv        (_gc_repofilepath cfg)
       setts   <- devSettings devJwkFile
+      mail    <- Mail.readConfig iniPath
       pure $ DevEnv
         { _dev_env_pool     = pool
-        , _dev_env_repo     = repo
         , _dev_env_nodeStory  = nodeStory_env
         , _dev_env_settings = setts
         , _dev_env_config   = cfg
+        , _dev_env_mail      = mail
         }
 
 -- | Run Cmd Sugar for the Repl (GHCI)

src/Gargantext/API/GraphQL/Utils.hs

+module Gargantext.API.GraphQL.Utils where
+
+import Data.Morpheus.Types (GQLTypeOptions, fieldLabelModifier)
+import qualified Data.Text as T
+import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix)
+import Gargantext.Prelude
+
+unPrefix :: T.Text -> GQLTypeOptions -> GQLTypeOptions
+unPrefix prefix options = options { fieldLabelModifier = nflm }
+  where
+    nflm label = unCapitalize $ dropPrefix (T.unpack prefix) $ ( fieldLabelModifier options ) label