Merge branch 'dev-dockerfile-ca-certificates-fix' into 74-dev-frame-calc-csv-import

CHANGELOG.md

+## Version 0.0.4.8.9
+* BACKEND: fix psql function util without sensitive data
+* FRONTEND: fix folder navigation (up link)
+
+## Version 0.0.4.8.8
+* FIX for CI
+
+## Version 0.0.4.8.7
+* FIX the graph generation (automatic/default, renewal, any distance)
+
+## Version 0.0.4.8.6
+* FIX the ngrams grouping
+
 ## Version 0.0.4.8.5
 * Unary document insertion: Doc table is reloaded after upload
 

bin/gargantext-cbor2json/Main.hs

-import Prelude (IO, id, (.))
+
+import System.Environment (getArgs)
+import Prelude (IO, id, (.), ($))
 import Data.Aeson (encode)
 import Codec.Serialise (deserialise)
 import qualified Data.ByteString.Lazy as L
-
-import Gargantext.API.Ngrams.Types (NgramsRepo)
+import Gargantext.Core.NodeStory (NodeListStory)
 
 main :: IO ()
-main = L.interact (encode . (id :: NgramsRepo -> NgramsRepo) . deserialise)
+main = L.interact (encode . (id :: NodeListStory -> NodeListStory) . deserialise)

bin/psql

 #!/bin/bash
 
-psql postgresql://gargantua:C8kdcUrAQy66U12341@localhost/gargandbV5
+INIFILE=$1
+
+getter () {
+  grep $1 $INIFILE | sed "s/^.*= //"
+}
+
+connect () {
+USER=$(getter "DB_USER")
+NAME=$(getter "DB_NAME")
+PASS=$(getter "DB_PASS")
+HOST=$(getter "DB_HOST")
+PORT=$(getter "DB_PORT")
+psql "postgresql://${USER}:${PASS}@${HOST}:${PORT}/${NAME}"
+}
+
+if [[ $1 == "" ]]
+then echo "USAGE : ./psql gargantext.ini"
+else connect $INIFILE
+fi
+
+
 

devops/docker/Dockerfile

-FROM fpco/stack-build:lts-18.16
+FROM fpco/stack-build:lts-18.18
 
 #RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 8B1DA6120C2BF624
 RUN apt-get update && \
-    apt-get install -y git libigraph0-dev && \
+    apt-get install -y ca-certificates git libigraph0-dev && \
     rm -rf /var/lib/apt/lists/*
 

package.yaml

 name: gargantext
-version: '0.0.4.8.5'
+version: '0.0.4.8.9'
 synopsis: Search, map, share
 description: Please see README.md
 category: Data
@@ -158,6 +158,7 @@ library:
   - full-text-search
   - fullstop
   - gargantext-prelude
+    #  - gargantext-graph >= 0.1.0.0
   - graphviz
   - hashable
   - haskell-igraph
@@ -235,6 +236,7 @@ library:
   - split
   - stemmer
   - swagger2
+  - taggy-lens
   - tagsoup
   - template-haskell
   - temporary
@@ -255,6 +257,7 @@ library:
   - wai-extra
   - wai-websockets
   - warp
+  - wikiparsec
   - websockets
   - wreq
   - xml-conduit
@@ -400,22 +403,22 @@ executables:
       - base
 
 
-#  gargantext-cbor2json:
-#    main: Main.hs
-#    source-dirs: bin/gargantext-cbor2json
-#    ghc-options:
-#    - -threaded
-#    - -rtsopts
-#    - -with-rtsopts=-N
-#    - -O2
-#    - -Wmissing-signatures
-#    dependencies:
-#      - gargantext
-#      - gargantext-prelude
-#      - base
-#      - bytestring
-#      - aeson
-#      - serialise
+  gargantext-cbor2json:
+    main: Main.hs
+    source-dirs: bin/gargantext-cbor2json
+    ghc-options:
+    - -threaded
+    - -rtsopts
+    - -with-rtsopts=-N
+    - -O2
+    - -Wmissing-signatures
+    dependencies:
+      - gargantext
+      - gargantext-prelude
+      - base
+      - bytestring
+      - aeson
+      - serialise
 
 
 tests:

shell.nix

+{ pkgs ? import ./nix/pkgs.nix {} }:
+let 
+  myBuildInputs = [
+    pkgs.pkgs.docker-compose
+    pkgs.pkgs.haskell-language-server
+    pkgs.pkgs.stack
+  ];
+in  
+pkgs.pkgs.mkShell {
+  name = pkgs.shell.name;
+  shellHook = pkgs.shell.shellHook;
+  buildInputs = pkgs.shell.buildInputs ++ myBuildInputs;
+}

src/Gargantext/Core/Mail.hs

@@ -69,7 +69,7 @@ email_to' (NewUser u m _) = (m,u)
 
 ------------------------------------------------------------------------
 bodyWith :: ServerAddress -> MailModel -> [Text]
-bodyWith server (Invitation u) = [ "Congratulation, you have been granted a beta user account to test the"
+bodyWith server (Invitation u) = [ "Congratulation, you have been granted a user account to test the"
                                  , "new GarganText platform!"
                                  ] <> (email_credentials server u)
 

src/Gargantext/Core/Text/Corpus/Parsers.hs

@@ -25,9 +25,8 @@ module Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), clean, parseFile, cl
 
 import "zip" Codec.Archive.Zip (withArchive, getEntry, getEntries)
 import Control.Concurrent.Async as CCA (mapConcurrently)
-import Control.Monad (join, sequence)
-import Control.Monad.IO.Class (liftIO)
 import Data.Attoparsec.ByteString (parseOnly, Parser)
+import Control.Monad (join)
 import Data.Either(Either(..))
 import Data.Either.Extra (partitionEithers)
 import Data.List (concat, lookup)
@@ -98,13 +97,11 @@ parseFormat WOS bs = do
           $ [runParser'  WOS bs]
   pure $ Right docs
 parseFormat ZIP bs = do
-  path <- emptySystemTempFile "parsed.zip"
+  path <- emptySystemTempFile "parsed-zip"
   DB.writeFile path bs
-  withArchive path $ do
-    files <- DM.keys <$> getEntries
-    filesContents <- mapM getEntry files
-    ddocs <- liftIO $ mapM (parseFormat CsvGargV3) filesContents
-    pure $ concat <$> sequence ddocs
+  parsedZip <- withArchive path $ do
+    DM.keys <$> getEntries
+  pure $ Left $ "Not implemented for ZIP, parsedZip" <> show parsedZip
 parseFormat _ _ = undefined
 
 -- | Parse file into documents

src/Gargantext/Core/Text/Corpus/Parsers/Date.hs

@@ -16,11 +16,13 @@ DGP.parseDateRaw DGP.FR "12 avril 2010" == "2010-04-12T00:00:00.000+00:00"
 
 {-# LANGUAGE TypeFamilies #-}
 
-module Gargantext.Core.Text.Corpus.Parsers.Date {-(parse, parseRaw, dateSplit, Year, Month, Day)-} where
+module Gargantext.Core.Text.Corpus.Parsers.Date
+{-(parse, parseRaw, dateSplit, Year, Month, Day)-}
+  where
 
 import Data.Aeson (toJSON, Value)
 import Data.HashMap.Strict as HM hiding (map)
-import Data.Text (Text, unpack, splitOn, pack)
+import Data.Text (Text, unpack, splitOn)
 import Data.Time (parseTimeOrError, defaultTimeLocale, toGregorian)
 import Data.Time.Clock (UTCTime(..), getCurrentTime)
 import Data.Time.LocalTime (utc)
@@ -69,12 +71,21 @@ parse lang s = parseDate' "%Y-%m-%dT%T" "0-0-0T0:0:0" lang s
 type DateFormat  = Text
 type DateDefault = Text
 
-parseDate' :: DateFormat -> DateDefault -> Lang -> Text -> IO UTCTime
+parseDate' :: DateFormat
+           -> DateDefault
+           -> Lang
+           -> Text
+           -> IO UTCTime
 parseDate' format def lang s = do
   dateStr' <- parseRaw lang s
-  let dateStr = unpack $ maybe def identity
-                       $ head $ splitOn "." dateStr'
-  pure $ parseTimeOrError True defaultTimeLocale (unpack format) dateStr
+  if dateStr' == ""
+    then getCurrentTime
+    else do
+      let dateStr = unpack
+                  $ maybe def identity
+                  $ head
+                  $ splitOn "." dateStr'
+      pure $ parseTimeOrError True defaultTimeLocale (unpack format) dateStr
 
 
 -- TODO add Paris at Duckling.Locale Region datatype
@@ -91,24 +102,28 @@ parserLang _  = panic "not implemented"
 -- currentContext lang = localContext lang <$> utcToDucklingTime <$> getCurrentTime
 -- parseRaw :: Context -> Text -> SomeErrorHandling Text
 
--- TODO error handling
 parseRaw :: Lang -> Text -> IO Text
 parseRaw lang text = do -- case result
-    maybeResult <- extractValue <$> getTimeValue <$> parseDateWithDuckling lang text (Options True)
+    maybeResult <- extractValue <$> getTimeValue
+                                <$> parseDateWithDuckling lang text (Options True)
     case maybeResult of
       Just result -> pure result
-      Nothing     -> panic $ "[G.C.T.C.P.D.parseRaw] ERROR" <> (pack . show) lang <> " " <> text
-
+      Nothing     -> do
+        printDebug ("[G.C.T.C.P.D.parseRaw] ERROR " <> (cs . show) lang)
+                   text
+        pure ""
 
-getTimeValue :: [ResolvedToken] -> Value
+getTimeValue :: [ResolvedToken] -> Maybe Value
 getTimeValue rt = case head rt of
-  Nothing -> panic "error"
+  Nothing -> do
+    Nothing
   Just x  -> case rval x of
-    RVal Time t -> toJSON t
-    _  -> panic "error2"
+    RVal Time t -> Just $ toJSON t
+    _  -> do
+      Nothing
 
-extractValue :: Value -> Maybe Text
-extractValue (Json.Object object) =
+extractValue :: Maybe Value -> Maybe Text
+extractValue (Just (Json.Object object)) =
   case HM.lookup "value" object of
     Just (Json.String date) -> Just date
     _                  -> Nothing

src/Gargantext/Core/Text/Corpus/Parsers/Wikidata.hs

+{-|
+Module      : Gargantext.Core.Text.Corpus.Parsers.Wikidata
+<<<<<<< HEAD
+Description : To query Wikidata 
+=======
+Description : To query Wikidata
+>>>>>>> dev-clustering
+Copyright   : (c) CNRS, 2019-Present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+{-# LANGUAGE TemplateHaskell     #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Gargantext.Core.Text.Corpus.Parsers.Wikidata where
+
+import Control.Lens (makeLenses, (^.) )
+import Data.Maybe (catMaybes)
+import Data.Text (Text, concat)
+import Database.HSparql.Connection
+import Gargantext.Core (Lang(..))
+import Gargantext.Core.Text.Corpus.Parsers.Isidore (unbound)
+import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
+import Gargantext.Prelude
+import Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
+import Prelude (String)
+import qualified Data.List as List
+import Gargantext.Core.Text.Corpus.Parsers.Date (dateSplit)
+
+
+
+data WikiResult = WikiResult { _wr_cid         :: Maybe Text
+                             , _wr_title       :: Maybe Text
+                             , _wr_url         :: Maybe Text
+                             , _wr_yearStart   :: Maybe Text
+                             , _wr_yearEnd     :: Maybe Text
+                             , _wr_yearFlorish :: Maybe Text
+                             } deriving (Show, Eq)
+$(makeLenses ''WikiResult)
+
+type NumberOfSections = Int
+
+wikidataGet :: Int -> NumberOfSections -> IO [HyperdataDocument]
+wikidataGet n m = do
+  results <- wikidataSelect n
+  mapM (wikiPageToDocument m) results
+
+
+wikiPageToDocument :: NumberOfSections -> WikiResult ->  IO HyperdataDocument
+wikiPageToDocument m wr = do
+  sections <- case wr ^. wr_url of
+    Nothing -> pure []
+    Just  u -> crawlPage u
+
+  let bdd    = Just "wikidata"
+      doi    = Nothing
+      url    = (wr ^. wr_url)
+      uniqId    = Nothing
+      uniqIdBdd = Nothing
+      page      = Nothing
+      title     = (wr ^. wr_title)
+      authors    = Nothing
+      institutes = Nothing
+      source     = Nothing
+      abstract   = Just $ concat $ take m sections
+
+  (date, (year, month, day))
+    <- dateSplit EN $ head
+                    $ catMaybes
+                    [ wr ^. wr_yearStart
+                    , wr ^. wr_yearEnd
+                    , wr ^. wr_yearFlorish
+                    , head sections
+                    ]
+
+  let hour = Nothing
+      minute = Nothing
+      second = Nothing
+      iso2   = Just $ cs $ show EN
+
+  pure $ HyperdataDocument bdd doi url uniqId uniqIdBdd
+                           page title authors institutes source
+                           abstract ((cs . show) <$> date) year month day hour minute second iso2
+
+
+wikidataSelect :: Int -> IO [WikiResult]
+wikidataSelect n = do
+  result <- selectQueryRaw wikidataRoute (wikidataQuery n)
+  case result of
+    Nothing      -> pure []
+    Just result' -> pure $ map toWikiResult $ unbound' EN result'
+
+
+unbound' :: Lang -> [[BindingValue]] -> [[Maybe Text]]
+unbound' l = map (map (unbound l))
+
+toWikiResult :: [Maybe Text] -> WikiResult
+toWikiResult (c:t:u:ys:ye:yf:_) = WikiResult c t u ys ye yf
+toWikiResult _                  = panic "[G.C.T.C.Parsers.Wikidata.toWikiResult] error"
+
+wikidataRoute :: EndPoint
+wikidataRoute = "https://query.wikidata.org/sparql"
+
+wikidataQuery :: Int -> String
+wikidataQuery n = List.unlines
+      ["     PREFIX wd:       <http://www.wikidata.org/entity/>"
+      ,"     PREFIX wdt:      <http://www.wikidata.org/prop/direct/>"
+      ,"     PREFIX schema:   <http://schema.org/>"
+      ,"     PREFIX wikibase: <http://wikiba.se/ontology#>"
+      ,"     SELECT DISTINCT "
+      ,"      ?cid"
+      ,"      ?title"
+      ,"      ?url"
+      ,"      (year(xsd:dateTime(?dateStart))   as ?yearStart)"
+      ,"      (year(xsd:dateTime(?dateEnd))     as ?yearEnd)"
+      ,"      (year(xsd:dateTime(?dateFlorish)) as ?yearFlorish) "
+      ,"     WHERE {"
+      ,"       ?cid wdt:P31 wd:Q968159 ."
+      ,"       ?cid rdfs:label ?title filter (lang(?title) = \"en\") ."
+      ,"      "
+      ,"       ?url schema:about ?cid ."
+      ,"       ?url schema:inLanguage \"en\" ."
+      ,"       FILTER (SUBSTR(str(?url), 1, 25) = \"https://en.wikipedia.org/\")"
+      ,"       OPTIONAL {?cid (wdt:P580) ?dateStart   .}"
+      ,"       OPTIONAL {?cid (wdt:P582) ?dateEnd     .}"
+      ,"       OPTIONAL {?cid (wdt:P571) ?dateFlorish .}"
+      ,"     }"
+      ,"       LIMIT " <> (cs $ show n)
+      ]
+

src/Gargantext/Core/Text/Corpus/Parsers/Wikidata/Crawler.hs

+{-|
+Module      : Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
+Description : Some utils to parse dates
+Copyright   : (c) CNRS 2017-present
+License     : AGPL + CECILL v3
+Maintainer  : team@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+Thx to Alp Well Typed for the first version.
+
+-}
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
+  where
+
+import Control.Lens hiding (element, elements, children)
+import Data.ByteString.Lazy (ByteString)
+import Data.Text (Text, unpack)
+import Data.Text.Encoding.Error (lenientDecode)
+import Data.Text.Lazy.Encoding (decodeUtf8With)
+import Gargantext.Prelude
+import Network.HTTP.Client (Response)
+import Network.Wreq (responseBody, get)
+import Text.Taggy.Lens
+
+
+
+type WikipediaUrlPage = Text
+crawlPage :: WikipediaUrlPage -> IO [Text]
+crawlPage url = do
+  datas <- get (unpack url)
+  pure $ sectionsOf datas
+
+
+sectionsOf :: Response ByteString -> [Text]
+sectionsOf resp =
+  resp ^.. responseBody
+         . to (decodeUtf8With lenientDecode)
+         . html
+         . allAttributed (ix "class" . only "mw-parser-output")
+         . allNamed (only "p")
+         . to paragraphText
+
+paragraphText :: Element -> Text
+paragraphText p = collectTextN (p ^. children)
+  where collectTextN (NodeContent t : ns) = t <> collectTextN ns
+        collectTextN (NodeElement elt : ns) = collectTextE elt <> collectTextN ns
+        collectTextN [] = ""
+
+        collectTextE (Element _ _ ns) = collectTextN ns
+
+

src/Gargantext/Core/Viz/Graph/API.hs

@@ -90,10 +90,11 @@ getGraph _uId nId = do
   let
     graph  = nodeGraph ^. node_hyperdata . hyperdataGraph
     camera = nodeGraph ^. node_hyperdata . hyperdataCamera
-    cId = maybe (panic "[G.V.G.API] Node has no parent")
-                  identity
-                  $ nodeGraph ^. node_parent_id
+    
+  mcId <- getClosestParentIdByType nId NodeCorpus
+  let cId = maybe (panic "[G.V.G.API] Node has no parent") identity mcId
 
+  printDebug "[getGraph] getting list for cId" cId
   listId <- defaultList cId
   repo <- getRepo' [listId]
 
@@ -130,15 +131,13 @@ recomputeGraph _uId nId maybeDistance = do
     graphMetric   = case maybeDistance of
                       Nothing -> graph ^? _Just . graph_metadata . _Just . gm_metric
                       _       -> maybeDistance
-
-  let
-    cId = maybe (panic "[G.C.V.G.API.recomputeGraph] Node has no parent")
-                  identity
-                  $ nodeGraph ^. node_parent_id
     similarity = case graphMetric of
                    Nothing -> withMetric Order1
                    Just m  -> withMetric m
 
+  mcId <- getClosestParentIdByType nId NodeCorpus
+  let cId = maybe (panic "[G.V.G.API] Node has no parent") identity mcId
+
   listId  <- defaultList cId
   repo <- getRepo' [listId]
   let v   = repo ^. unNodeStory . at listId . _Just . a_version
@@ -190,8 +189,10 @@ computeGraph cId d nt repo = do
 
   listNgrams <- getListNgrams [lId] nt
 
+  -- graph <- liftBase $ cooc2graphWith Bac d 0 myCooc
   graph <- liftBase $ cooc2graphWith Spinglass d 0 myCooc
   -- saveAsFileDebug "debug/graph" graph
+
   pure $ mergeGraphNgrams graph (Just listNgrams)
 
 

src/Gargantext/Core/Viz/Graph/Bridgeness.hs

@@ -24,7 +24,9 @@ import Data.Maybe (catMaybes)
 import Data.Ord (Down(..))
 import Gargantext.Prelude
 import qualified Data.Map as DM
-import Gargantext.Core.Viz.Graph.Tools.IGraph (ClusterNode(..))
+import Gargantext.Core.Viz.Graph.Types (ClusterNode(..))
+
+
 
 ----------------------------------------------------------------------
 type Partitions a = Map (Int, Int) Double -> IO [a]

src/Gargantext/Core/Viz/Graph/Tools.hs

@@ -27,8 +27,10 @@ import Gargantext.Core.Statistics
 import Gargantext.Core.Viz.Graph
 import Gargantext.Core.Viz.Graph.Bridgeness (bridgeness, Partitions, ToComId(..))
 import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex, map2mat, mat2map, Index, MatrixShape(..))
-import Gargantext.Core.Viz.Graph.Tools.IGraph (mkGraphUfromEdges, spinglass, ClusterNode)
+import Gargantext.Core.Viz.Graph.Tools.IGraph (mkGraphUfromEdges, spinglass)
+import Gargantext.Core.Viz.Graph.Types (ClusterNode)
 import Gargantext.Prelude
+-- import qualified Graph.BAC.ProxemyOptim as BAC
 import IGraph.Random -- (Gen(..))
 import qualified Data.HashMap.Strict      as HashMap
 import qualified Data.List                as List
@@ -40,12 +42,11 @@ import qualified IGraph.Algorithms.Layout as Layout
 
 
 -------------------------------------------------------------
-
 defaultClustering :: Map (Int, Int) Double -> IO [ClusterNode]
-defaultClustering = spinglass 1
+-- defaultClustering x = pure $ BAC.defaultClustering x
+defaultClustering x = spinglass 1 x
 
 -------------------------------------------------------------
-
 type Threshold = Double
 
 cooc2graph' :: Ord t => Distance
@@ -67,7 +68,7 @@ cooc2graph' distance threshold myCooc
         myCooc' = toIndex ti myCooc
 
 
-data PartitionMethod = Louvain | Spinglass | Bac
+data PartitionMethod = Louvain | Spinglass -- | Bac
 
 -- | coocurrences graph computation
 cooc2graphWith :: PartitionMethod
@@ -75,9 +76,9 @@ cooc2graphWith :: PartitionMethod
                -> Threshold
                -> HashMap (NgramsTerm, NgramsTerm) Int
                -> IO Graph
-cooc2graphWith Louvain   = undefined -- TODO use IGraph bindings
+cooc2graphWith Louvain   = undefined
 cooc2graphWith Spinglass = cooc2graphWith' (spinglass 1)
-cooc2graphWith Bac       = undefined -- cooc2graphWith' BAC.defaultClustering
+-- cooc2graphWith Bac       = cooc2graphWith' (\x -> pure $ BAC.defaultClustering x)
 
 cooc2graph'' :: Ord t => Distance
                       -> Double
@@ -109,13 +110,16 @@ filterByNeighbours threshold distanceMap = filteredMap
                                        $ Map.filter (> 0)
                                        $ Map.filterWithKey (\(from,_) _ -> idx == from) distanceMap
                            in List.take (round threshold) selected
-                      ) indexes                 
+                      ) indexes
 
 
 doDistanceMap :: Distance
-               -> Threshold
-               -> HashMap (NgramsTerm, NgramsTerm) Int
-               -> (Map (Int,Int) Double, Map (Index, Index) Int, Map NgramsTerm Index)
+              -> Threshold
+              -> HashMap (NgramsTerm, NgramsTerm) Int
+              -> ( Map (Int,Int) Double
+                 , Map (Index, Index) Int
+                 , Map NgramsTerm Index
+                 )
 doDistanceMap distance threshold myCooc = (distanceMap, myCooc', ti)
   where
     -- TODO remove below
@@ -125,9 +129,10 @@ doDistanceMap distance threshold myCooc = (distanceMap, myCooc', ti)
     (ti, _) = createIndices theMatrix
     tiSize  = Map.size ti
     myCooc' = toIndex ti theMatrix
+
     matCooc = case distance of  -- Shape of the Matrix
                 Conditional    -> map2mat Triangle 0 tiSize
-                Distributional -> map2mat Square     0 tiSize
+                Distributional -> map2mat Square   0 tiSize
             $ case distance of   -- Removing the Diagonal ?
                 Conditional     -> Map.filterWithKey (\(a,b) _ -> a /= b)
                 Distributional  -> identity
@@ -136,7 +141,8 @@ doDistanceMap distance threshold myCooc = (distanceMap, myCooc', ti)
     similarities = measure distance matCooc
     links = round (let n :: Double = fromIntegral tiSize in n * log n)
 
-    distanceMap = Map.fromList  $ List.take links
+    distanceMap = Map.fromList
+                $ List.take links
                 $ List.sortOn snd
                 $ Map.toList
                 $ case distance of

src/Gargantext/Core/Viz/Graph/Tools/IGraph.hs

@@ -20,6 +20,8 @@ import Data.Singletons (SingI)
 import IGraph hiding (mkGraph, neighbors, edges, nodes, Node, Graph)
 import Protolude
 import Gargantext.Core.Viz.Graph.Index
+-- import Graph.Types
+import Gargantext.Core.Viz.Graph.Types
 import qualified Data.List                   as List
 import qualified IGraph                      as IG
 import qualified IGraph.Algorithms.Clique    as IG
@@ -82,10 +84,6 @@ partitions_spinglass' s g = do
   IG.findCommunity g Nothing Nothing IG.spinglass gen
 
 
-data ClusterNode = ClusterNode { cl_node_id :: Int
-                               , cl_community_id :: Int
-                               }
-
 toClusterNode :: [[Int]] -> [ClusterNode]
 toClusterNode ns = List.concat
                  $ map (\(cId, ns') -> map (\n -> ClusterNode n cId) ns')

src/Gargantext/Core/Viz/Graph/Types.hs

+{-| Module      : Graph.Types
+Description :
+Copyright   : (c) CNRS, Alexandre Delanoë
+License     : AGPL + CECILL v3
+Maintainer  : contact@gargantext.org
+Stability   : experimental
+Portability : POSIX
+
+-}
+
+module Gargantext.Core.Viz.Graph.Types where
+
+import qualified Data.Graph.Inductive.PatriciaTree as DGIP
+-- import Data.IntMap (IntMap)
+-- import qualified Eigen.Matrix as DenseMatrix
+-- import Eigen.SparseMatrix (SparseMatrix)
+
+--import qualified Data.Matrix.Sparse.Static    as Sparse
+-- import qualified Data.Vector.Unboxed          as VU
+-- import qualified Numeric.LinearAlgebra.Static as Dense
+import Protolude hiding (sum, natVal)
+
+-- | Main Types use in this libray
+
+type Dict = IntMap
+
+-- | Use the optimized version of Graph
+type Graph a b = DGIP.Gr a b
+
+-- | Type for Matrix computation optimizations (with Eigen)
+-- type MatrixD          n = Dense.L n n
+-- type MatrixS          n = Sparse.Matrix n n Double
+
+
+data ClusterNode = ClusterNode
+  { cl_node_id      :: Int
+  , cl_community_id :: Int
+  } deriving Show
+
+

src/Gargantext/Database/Query/Table/Node.hs

@@ -386,7 +386,6 @@ defaultList cId =
 defaultListMaybe :: (HasNodeError err, HasDBid NodeType) => CorpusId -> Cmd err (Maybe NodeId)
 defaultListMaybe cId = headMay <$> map (view node_id ) <$> getListsWithParentId cId
 
-
 getListsWithParentId :: HasDBid NodeType => NodeId -> Cmd err [Node HyperdataList]
 getListsWithParentId n = runOpaQuery $ selectNodesWith' n (Just NodeList)
 

stack.yaml

 resolver:
- url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/16.yaml 
+ url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/18.yaml 
 flags: {}
 extra-package-dbs: []
 packages:
 - .
-#- 'deps/patches-class'
+  #- 'deps/gargantext-graph'
 #- 'deps/patches-map'
 #- 'deps/accelerate'
 #- 'deps/accelerate-utility'
@@ -12,7 +12,7 @@ packages:
 docker:
   enable: false
   #enable: true
-  repo: 'cgenie/stack-build:lts-18.16-garg'
+  repo: 'cgenie/stack-build:lts-18.18-garg'
   run-args:
     - '--publish=8008:8008'
 
@@ -28,7 +28,9 @@ allow-newer: true
 
 extra-deps:
 - git: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude.git
-  commit: 6bfdb29e9a576472c7fd7ebe648ad101e5b3927f
+  commit: 220f32810f988a5a121f110a7d557fc7d0721712
+#- git: ssh://gitolite3@delanoe.org/gargantext-graph
+#  commit: 294887a220460bd0c114638fff9ea53306cd2f18
   # Data Mining Libs
 - git: https://github.com/delanoe/data-time-segment.git
   commit: 10a416b9f6c443866b36479c3441ebb3bcdeb7ef
@@ -47,7 +49,7 @@ extra-deps:
 - git: https://github.com/delanoe/haskell-opaleye.git
   commit: d3ab7acd5ede737478763630035aa880f7e34444
 - git: https://github.com/delanoe/hsparql.git
-  commit: 308c74b71a1abb0a91546fa57d353131248e3a7f
+  commit: 2acbbc55ac9bbd4bf1a713c586b8b8e8b82892eb
 - git: https://github.com/robstewart57/rdf4h.git
   commit: 4fd2edf30c141600ffad6d730cc4c1c08a6dbce4
 
@@ -84,8 +86,21 @@ extra-deps:
 - git: https://gitlab.iscpif.fr/anoe/accelerate-utility.git
   commit: 83ada76e78ac10d9559af8ed6bd4064ec81308e4
 - accelerate-arithmetic-1.0.0.1@sha256:555639232aa5cad411e89247b27871d09352b987a754230a288c690b6de6d888,2096
+- git: https://github.com/rspeer/wikiparsec.git
+  commit: 9637a82344bb70f7fa8f02e75db3c081ccd434ce
 
-  # Others dependencies (using stack resolver)
+  # Gargantext-graph
+- eigen-3.3.7.0@sha256:7c24a86426b238e02ba0ac2179cc44997518b443a13828ed7a791fe49a6dffa5,82060
+- git: https://github.com/alpmestan/sparse-linear.git
+  commit: bc6ca8058077b0b5702ea4b88bd4189cfcad267a
+  subdirs:
+    - sparse-linear
+- git: https://github.com/alpmestan/hmatrix.git
+  commit: b9fca8beee0f23c17a6b2001ec834d071709e6e7
+  subdirs:
+    - packages/base
+
+      # Others dependencies (using stack resolver)
 - constraints-extras-0.3.1.0@sha256:12016ebb91ad5ed2c82bf7e48c6bd6947d164d33c9dca5ac3965de1bb6c780c0,1777
 - KMP-0.2.0.0@sha256:6dfbac03ef00ebd9347234732cb86a40f62ab5a80c0cc6bedb8eb51766f7df28,2562
 - Unique-0.4.7.8@sha256:9661f45aa31dde119a2114566166ea38b011a45653337045ee4ced75636533c0,2067
@@ -111,4 +126,18 @@ extra-deps:
 
 # need Vector.uncons
 - vector-0.12.3.0@sha256:0ae2c1ba86f0077910be242ec6802cc3d7725fe7b2bea6987201aa3737b239b5,7953
+
+# needed for wikiparsec
+- fast-tagsoup-utf8-only-1.0.5@sha256:9292c8ff275c08b88b6013ccc410182552f180904214a07ad4db932ab462aaa1,1651
+# wikipedia crawl
+- taggy-lens-0.1.2@sha256:091ca81d02bd3d7fb493dce0148e1a38f25eb178a1ebd751043a23239e5e3265,3009
+- taggy-0.2.1@sha256:7bc55ddba178971dc6052163597f0445a0a2b5b0ca0e84ce651d53d722e3c265,4662
+
 - servant-ekg-0.3.1@sha256:19bd9dc3943983da8e79d6f607614c68faea4054fb889d508c8a2b67b6bdd448,2203
+
+
+# For the graph clustering
+ghc-options:
+  hmatrix: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal  -fno-state-hack
+  sparse-linear: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal  -fno-state-hack
+  gargantext-graph: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal  -fno-state-hack