[FEAT][PHYLO] preparing integration to backend

package.yaml

@@ -100,7 +100,7 @@ library:
   - Gargantext.Core.Viz.Graph.Tools
   - Gargantext.Core.Viz.Graph.Tools.IGraph
   - Gargantext.Core.Viz.Graph.Index
-  - Gargantext.Core.Viz.AdaptativePhylo
+  - Gargantext.Core.Viz.Phylo
   - Gargantext.Core.Viz.Phylo.PhyloMaker
   - Gargantext.Core.Viz.Phylo.PhyloTools
   - Gargantext.Core.Viz.Phylo.PhyloExport
@@ -322,9 +322,9 @@ executables:
       - unordered-containers
       - full-text-search
 
-  gargantext-adaptative-phylo:
+  gargantext-phylo:
     main: Main.hs
-    source-dirs: bin/gargantext-adaptative-phylo
+    source-dirs: bin/gargantext-phylo
     ghc-options:
     - -threaded
     - -rtsopts

src/Gargantext/Core/Methods/Graph/MaxClique.hs

@@ -64,7 +64,7 @@ import Gargantext.Core.Viz.Graph.FGL (Graph_Undirected, degree, neighbors, mkGra
 import Gargantext.Core.Viz.Graph.Tools (cooc2graph',cooc2graph'', Threshold)
 import Gargantext.Core.Methods.Distances (Distance)
 import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex)
-import Gargantext.Core.Viz.AdaptativePhylo
+import Gargantext.Core.Viz.Phylo
 -- import Debug.Trace (trace)
 type Graph = Graph_Undirected
 type Neighbor = Node

src/Gargantext/Core/Viz/AdaptativePhylo.hs → src/Gargantext/Core/Viz/Phylo.hs

@@ -24,31 +24,26 @@ one 8, e54847.
 {-# LANGUAGE DeriveAnyClass   #-}
 {-# LANGUAGE TemplateHaskell   #-}
 
-module Gargantext.Core.Viz.AdaptativePhylo where
+module Gargantext.Core.Viz.Phylo where
 
+import Control.DeepSeq (NFData)
+import Control.Lens (makeLenses)
 import Data.Aeson
 import Data.Aeson.TH (deriveJSON)
+import Data.Map (Map)
 import Data.Text   (Text, pack)
 import Data.Vector (Vector)
-import Data.Map (Map)
-
-import Gargantext.Core.Utils.Prefix (unPrefix)
-import Gargantext.Prelude
-import Gargantext.Core.Text.Context (TermList)
-
 import GHC.Generics
 import GHC.IO (FilePath)
-import Control.DeepSeq (NFData)
-import Control.Lens (makeLenses)
-
+import Gargantext.Core.Text.Context (TermList)
+import Gargantext.Core.Utils.Prefix (unPrefix)
+import Gargantext.Prelude
 import qualified Data.Text.Lazy as TextLazy
 
-
 ----------------
 -- | Config | --
 ----------------
 
-
 data CorpusParser =
       Wos  {_wos_limit  :: Int}
     | Csv  {_csv_limit  :: Int}
@@ -182,34 +177,49 @@ defaultConfig =
 
 instance FromJSON Config
 instance ToJSON Config
+
 instance FromJSON CorpusParser
 instance ToJSON CorpusParser
+
 instance FromJSON Proximity
 instance ToJSON Proximity
+
 instance FromJSON SeaElevation
 instance ToJSON SeaElevation
+
 instance FromJSON TimeUnit
 instance ToJSON TimeUnit
+
 instance FromJSON CliqueFilter
 instance ToJSON CliqueFilter
+
 instance FromJSON Clique
 instance ToJSON Clique
+
 instance FromJSON PhyloLabel
 instance ToJSON PhyloLabel
+
 instance FromJSON Tagger
 instance ToJSON Tagger
+
 instance FromJSON Sort
 instance ToJSON Sort
+
 instance FromJSON Order
 instance ToJSON Order
+
 instance FromJSON Filter
 instance ToJSON Filter
+
 instance FromJSON SynchronyScope
 instance ToJSON SynchronyScope
+
 instance FromJSON SynchronyStrategy
 instance ToJSON SynchronyStrategy
+
 instance FromJSON Synchrony
 instance ToJSON Synchrony
+
 instance FromJSON Quality
 instance ToJSON Quality
 
@@ -252,9 +262,10 @@ type Ngrams = Text
 
 -- Document : a piece of Text linked to a Date
 -- date = computational date; date' = original string date yyyy-mm-dd
+-- Export Database to Document
 data Document = Document
-      { date    :: Date
-      , date'   :: Text
+      { date    :: Date   -- datatype Date {unDate :: Int}
+      , date'   :: Text   -- show date
       , text    :: [Ngrams]
       , weight  :: Maybe Double
       , sources :: [Text]
@@ -469,16 +480,22 @@ makeLenses ''PhyloBranch
 
 instance FromJSON Phylo
 instance ToJSON Phylo
+
 instance FromJSON PhyloSources
 instance ToJSON PhyloSources
+
 instance FromJSON PhyloParam
 instance ToJSON PhyloParam
+
 instance FromJSON PhyloPeriod
 instance ToJSON PhyloPeriod
+
 instance FromJSON PhyloLevel
 instance ToJSON PhyloLevel
+
 instance FromJSON Software
 instance ToJSON Software
+
 instance FromJSON PhyloGroup
 instance ToJSON PhyloGroup
 

src/Gargantext/Core/Viz/Phylo/PhyloExample.hs

@@ -15,23 +15,20 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.PhyloExample where
 
+import Control.Lens
+import Data.GraphViz.Types.Generalised (DotGraph)
 import Data.List (sortOn, nub, sort)
 import Data.Map (Map)
 import Data.Text (Text, toLower)
-
-import Gargantext.Prelude
 import Gargantext.Core.Text.Context (TermList)
 import Gargantext.Core.Text.Terms.Mono (monoTexts)
-import Gargantext.Core.Viz.AdaptativePhylo
-import Gargantext.Core.Viz.Phylo.PhyloTools
-import Gargantext.Core.Viz.Phylo.PhyloMaker
+import Gargantext.Core.Viz.Phylo
 import Gargantext.Core.Viz.Phylo.PhyloExport
-import Gargantext.Core.Viz.Phylo.TemporalMatching (adaptativeTemporalMatching, constanteTemporalMatching)
+import Gargantext.Core.Viz.Phylo.PhyloMaker
+import Gargantext.Core.Viz.Phylo.PhyloTools
 import Gargantext.Core.Viz.Phylo.SynchronicClustering (synchronicClustering)
-
-import Control.Lens
-import Data.GraphViz.Types.Generalised (DotGraph)
-
+import Gargantext.Core.Viz.Phylo.TemporalMatching (adaptativeTemporalMatching, constanteTemporalMatching)
+import Gargantext.Prelude
 import qualified Data.Vector as Vector
 
 ---------------------------------

src/Gargantext/Core/Viz/Phylo/PhyloExport.hs

@@ -12,30 +12,27 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.PhyloExport where
 
-import Data.Map (Map, fromList, empty, fromListWith, insert, (!), elems, unionWith, findWithDefault, toList, member)
-import Data.List ((++), sort, nub, null, concat, sortOn, groupBy, union, (\\), (!!), init, partition, notElem, unwords, nubBy, inits, elemIndex)
-import Data.Vector (Vector)
-
-import Prelude (writeFile)
-import Gargantext.Prelude
-import Gargantext.Core.Viz.AdaptativePhylo
-import Gargantext.Core.Viz.Phylo.PhyloTools
-import Gargantext.Core.Viz.Phylo.TemporalMatching (filterDocs, filterDiago, reduceDiagos, toProximity, getNextPeriods)
-
 import Control.Lens hiding (Level)
 import Control.Parallel.Strategies (parList, rdeepseq, using)
 import Data.GraphViz hiding (DotGraph, Order)
-import Data.GraphViz.Types.Generalised (DotGraph)
 import Data.GraphViz.Attributes.Complete hiding (EdgeType, Order) 
+import Data.GraphViz.Types.Generalised (DotGraph)
 import Data.GraphViz.Types.Monadic
+import Data.List ((++), sort, nub, null, concat, sortOn, groupBy, union, (\\), (!!), init, partition, notElem, unwords, nubBy, inits, elemIndex)
+import Data.Map (Map, fromList, empty, fromListWith, insert, (!), elems, unionWith, findWithDefault, toList, member)
 import Data.Text.Lazy (fromStrict, pack, unpack)
-import System.FilePath
+import Data.Vector (Vector)
 import Debug.Trace (trace)
-
+import Gargantext.Core.Viz.Phylo
+import Gargantext.Core.Viz.Phylo.PhyloTools
+import Gargantext.Core.Viz.Phylo.TemporalMatching (filterDocs, filterDiago, reduceDiagos, toProximity, getNextPeriods)
+import Gargantext.Prelude
+import Prelude (writeFile)
+import System.FilePath
+import qualified Data.GraphViz.Attributes.HTML as H
 import qualified Data.Text as Text
-import qualified Data.Vector as Vector
 import qualified Data.Text.Lazy as Lazy
-import qualified Data.GraphViz.Attributes.HTML as H
+import qualified Data.Vector as Vector
 
 --------------------
 -- | Dot export | --

src/Gargantext/Core/Viz/Phylo/PhyloMaker.hs

@@ -11,30 +11,29 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.PhyloMaker where
 
+
+import Control.DeepSeq (NFData)
+import Control.Lens hiding (Level)
+import Control.Parallel.Strategies (parList, rdeepseq, using)
 import Data.List (concat, nub, partition, sort, (++), group, intersect, null, sortOn, groupBy, tail)
 import Data.Map (Map, fromListWith, keys, unionWith, fromList, empty, toList, elems, (!), restrictKeys, foldlWithKey, insert)
-import Data.Vector (Vector)
 import Data.Text (Text)
+import Data.Vector (Vector)
+import Debug.Trace (trace)
 
-import Gargantext.Prelude
-import Gargantext.Core.Viz.AdaptativePhylo
-import Gargantext.Core.Viz.Phylo.PhyloTools
-import Gargantext.Core.Viz.Phylo.TemporalMatching (adaptativeTemporalMatching, constanteTemporalMatching, getNextPeriods, filterDocs, filterDiago, reduceDiagos, toProximity)
-import Gargantext.Core.Viz.Phylo.SynchronicClustering (synchronicClustering)
+import Gargantext.Core.Methods.Distances (Distance(Conditional))
+import Gargantext.Core.Methods.Graph.MaxClique (getMaxCliques)
 import Gargantext.Core.Text.Context (TermList)
 import Gargantext.Core.Text.Metrics.FrequentItemSet (fisWithSizePolyMap, fisWithSizePolyMap', Size(..))
-import Gargantext.Core.Methods.Graph.MaxClique (getMaxCliques)
-import Gargantext.Core.Methods.Distances (Distance(Conditional))
+import Gargantext.Core.Viz.Phylo
 import Gargantext.Core.Viz.Phylo.PhyloExport (toHorizon)
+import Gargantext.Core.Viz.Phylo.PhyloTools
+import Gargantext.Core.Viz.Phylo.SynchronicClustering (synchronicClustering)
+import Gargantext.Core.Viz.Phylo.TemporalMatching (adaptativeTemporalMatching, constanteTemporalMatching, getNextPeriods, filterDocs, filterDiago, reduceDiagos, toProximity)
+import Gargantext.Prelude
 
-
-import Control.DeepSeq (NFData)
-import Control.Parallel.Strategies (parList, rdeepseq, using)
-import Debug.Trace (trace)
-import Control.Lens hiding (Level)
-
-import qualified Data.Vector as Vector
 import qualified Data.Set as Set
+import qualified Data.Vector as Vector
 
 ------------------
 -- | To Phylo | --
@@ -162,6 +161,7 @@ indexDates' m = map (\docs ->
 
 
 -- To build the first phylo step from docs and terms
+-- QL: backend entre phyloBase et phyloClique
 toPhyloStep :: [Document] -> TermList -> Config -> Phylo
 toPhyloStep docs lst conf = case (getSeaElevation phyloBase) of 
     Constante  _ _ -> appendGroups cliqueToGroup 1 phyloClique (updatePeriods (indexDates' docs') phyloBase)
@@ -173,6 +173,7 @@ toPhyloStep docs lst conf = case (getSeaElevation phyloBase) of
         phyloClique =  toPhyloClique phyloBase docs'
         --------------------------------------
         docs' :: Map (Date,Date) [Document]
+        -- QL: Time Consuming here
         docs' =  groupDocsByPeriodRec date (getPeriodIds phyloBase) (sortOn date docs) empty
         --------------------------------------
         phyloBase :: Phylo

src/Gargantext/Core/Viz/Phylo/PhyloTools.hs

@@ -12,28 +12,23 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.PhyloTools where
 
-import Data.Vector (Vector, elemIndex)
+import Control.Lens hiding (Level)
 import Data.List (sort, concat, null, union, (++), tails, sortOn, nub, init, tail, partition, tails, nubBy, group)
-import Data.Set (Set, disjoint)
 import Data.Map (Map, elems, fromList, unionWith, keys, member, (!), filterWithKey, fromListWith, empty, restrictKeys)
+import Data.Set (Set, disjoint)
 import Data.String (String)
 import Data.Text (Text,unpack)
-
-import Prelude (floor,read)
-
+import Data.Vector (Vector, elemIndex)
+import Debug.Trace (trace)
+import Gargantext.Core.Viz.Phylo
 import Gargantext.Prelude
-import Gargantext.Core.Viz.AdaptativePhylo
+import Prelude (floor,read)
 import Text.Printf
-
-
-import Debug.Trace (trace)
-import Control.Lens hiding (Level)
-
-import qualified Data.Vector as Vector
 import qualified Data.List as List
-import qualified Data.Set as Set
 import qualified Data.Map as Map
+import qualified Data.Set as Set
 import qualified Data.Text as Text
+import qualified Data.Vector as Vector
 
 ------------
 -- | Io | --

src/Gargantext/Core/Viz/Phylo/SynchronicClustering.hs

@@ -11,20 +11,17 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.SynchronicClustering where
 
-import Gargantext.Prelude
-import Gargantext.Core.Viz.AdaptativePhylo
-import Gargantext.Core.Viz.Phylo.PhyloTools
-import Gargantext.Core.Viz.Phylo.TemporalMatching (weightedLogJaccard', filterDiago, reduceDiagos)
-import Gargantext.Core.Viz.Phylo.PhyloExport (processDynamics)
-
-import Data.List ((++), null, intersect, nub, concat, sort, sortOn, groupBy)
-import Data.Map  (Map, fromList, fromListWith, foldlWithKey, (!), insert, empty, restrictKeys, elems, mapWithKey, member)
-
+-- import Debug.Trace (trace)
 import Control.Lens hiding (Level)
-import Control.Parallel.Strategies (parList, rdeepseq, using)
 import Control.Monad (sequence)
--- import Debug.Trace (trace)
-
+import Control.Parallel.Strategies (parList, rdeepseq, using)
+import Data.List ((++), null, intersect, nub, concat, sort, sortOn, groupBy)
+import Data.Map  (Map, fromList, fromListWith, foldlWithKey, (!), insert, empty, restrictKeys, elems, mapWithKey, member)
+import Gargantext.Core.Viz.Phylo
+import Gargantext.Core.Viz.Phylo.PhyloExport (processDynamics)
+import Gargantext.Core.Viz.Phylo.PhyloTools
+import Gargantext.Core.Viz.Phylo.TemporalMatching (weightedLogJaccard', filterDiago, reduceDiagos)
+import Gargantext.Prelude
 import qualified Data.Map as Map
 
 
@@ -32,7 +29,6 @@ import qualified Data.Map as Map
 -- | New Level Maker | --
 -------------------------
 
-
 mergeGroups :: [Cooc] -> PhyloGroupId -> Map PhyloGroupId PhyloGroupId -> [PhyloGroup] -> PhyloGroup
 mergeGroups coocs id mapIds childs = 
     let ngrams = (sort . nub . concat) $ map _phylo_groupNgrams childs

src/Gargantext/Core/Viz/Phylo/TemporalMatching.hs

@@ -11,26 +11,21 @@ Portability : POSIX
 
 module Gargantext.Core.Viz.Phylo.TemporalMatching where
 
+import Control.Lens hiding (Level)
+import Control.Parallel.Strategies (parList, rdeepseq, using)
 import Data.List (concat, splitAt, tail, sortOn, (++), intersect, null, inits, groupBy, scanl, nub, nubBy, union, dropWhile, partition, or, sort, (!!))
 import Data.Map  (Map, fromList, elems, restrictKeys, unionWith, findWithDefault, keys, (!), (!?), filterWithKey, singleton, empty, mapKeys, adjust)
-
-import Gargantext.Prelude
-import Gargantext.Core.Viz.AdaptativePhylo
+import Debug.Trace (trace)
+import Gargantext.Core.Viz.Phylo
 import Gargantext.Core.Viz.Phylo.PhyloTools
-
+import Gargantext.Prelude
 import Prelude (floor,tan,pi)
-import Control.Lens hiding (Level)
-import Control.Parallel.Strategies (parList, rdeepseq, using)
-import Debug.Trace (trace)
-
 import Text.Printf
-
 import qualified Data.Map as Map
 import qualified Data.Set as Set
 import qualified Data.Vector as Vector
 
 
-
 -------------------
 -- | Proximity | --
 -------------------