[MERGE][GRAPH FIX] edges weight + confluence in graph.

package.yaml

@@ -47,6 +47,7 @@ library:
   - Gargantext.Prelude
   - Gargantext.Text
   - Gargantext.Text.Context
+  - Gargantext.Text.Crawlers
   - Gargantext.Text.Examples
   - Gargantext.Text.List.CSV
   - Gargantext.Text.Metrics
@@ -103,6 +104,7 @@ library:
   - contravariant
   - crawlerPubMed
   - crawlerIsidore
+  - crawlerHAL
   - data-time-segment
   - directory
   - duckling

src/Gargantext/Text/Crawlers.hs

@@ -15,7 +15,19 @@ Portability : POSIX
 module Gargantext.Text.Crawlers
     where
 
+{-
+import Data.Text (Text)
 --import Gargantext.Prelude
---import qualified PUBMED as PubMed
+import qualified PUBMED as PubMed
 
+data Crawler = PubMed | HAL | Isidore
 
+type Query = Text
+
+
+
+--{-
+crawl :: Crawler -> Query -> IO [PubMed.Doc]
+crawl Pubmed = PubMed.crawler
+--}
+-}

src/Gargantext/Viz/Graph.hs

@@ -64,6 +64,7 @@ instance ToSchema Node where
 data Edge = Edge { edge_source :: Text
                  , edge_target :: Text
                  , edge_weight :: Double
+                 , edge_confluence :: Double
                  , edge_id     :: Text
                  }
   deriving (Show, Generic)
@@ -119,7 +120,7 @@ instance Arbitrary Graph where
   arbitrary = elements $ [defaultGraph]
 
 defaultGraph :: Graph
-defaultGraph = Graph {_graph_nodes = [Node {node_x_coord=0, node_y_coord=0, node_size = 4, node_type = Terms, node_id = pack "0", node_label = pack "animal", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 3, node_type = Terms, node_id = pack "1", node_label = pack "bird", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "2", node_label = pack "boy", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "3", node_label = pack "dog", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "4", node_label = pack "girl", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 4, node_type = Terms, node_id = pack "5", node_label = pack "human body", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 3, node_type = Terms, node_id = pack "6", node_label = pack "object", node_attributes = Attributes {clust_default = 2}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "7", node_label = pack "pen", node_attributes = Attributes {clust_default = 2}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "8", node_label = pack "table", node_attributes = Attributes {clust_default = 2}}], _graph_edges = [Edge {edge_source = pack "0", edge_target = pack "0", edge_weight = 1.0, edge_id = pack "0"},Edge {edge_source = pack "1", edge_target = pack "0", edge_weight = 1.0, edge_id = pack "1"},Edge {edge_source = pack "1", edge_target = pack "1", edge_weight = 1.0, edge_id = pack "2"},Edge {edge_source = pack "2", edge_target = pack "2", edge_weight = 1.0, edge_id = pack "3"},Edge {edge_source = pack "2", edge_target = pack "5", edge_weight = 1.0, edge_id = pack "4"},Edge {edge_source = pack "3", edge_target = pack "0", edge_weight = 1.0, edge_id = pack "5"},Edge {edge_source = pack "3", edge_target = pack "1", edge_weight = 1.0, edge_id = pack "6"},Edge {edge_source = pack "3", edge_target = pack "3", edge_weight = 1.0, edge_id = pack "7"},Edge {edge_source = pack "4", edge_target = pack "4", edge_weight = 1.0, edge_id = pack "8"},Edge {edge_source = pack "4", edge_target = pack "5", edge_weight = 1.0, edge_id = pack "9"},Edge {edge_source = pack "5", edge_target = pack "5", edge_weight = 1.0, edge_id = pack "10"},Edge {edge_source = pack "6", edge_target = pack "6", edge_weight = 1.0, edge_id = pack "11"},Edge {edge_source = pack "7", edge_target = pack "6", edge_weight = 1.0, edge_id = pack "12"},Edge {edge_source = pack "7", edge_target = pack "7", edge_weight = 1.0, edge_id = pack "13"},Edge {edge_source = pack "8", edge_target = pack "6", edge_weight = 1.0, edge_id = pack "14"},Edge {edge_source = pack "8", edge_target = pack "7", edge_weight = 1.0, edge_id = pack "15"},Edge {edge_source = pack "8", edge_target = pack "8", edge_weight = 1.0, edge_id = pack "16"}], _graph_metadata = Nothing}
+defaultGraph = Graph {_graph_nodes = [Node {node_x_coord=0, node_y_coord=0, node_size = 4, node_type = Terms, node_id = pack "0", node_label = pack "animal", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 3, node_type = Terms, node_id = pack "1", node_label = pack "bird", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "2", node_label = pack "boy", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "3", node_label = pack "dog", node_attributes = Attributes {clust_default = 0}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "4", node_label = pack "girl", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 4, node_type = Terms, node_id = pack "5", node_label = pack "human body", node_attributes = Attributes {clust_default = 1}},Node {node_x_coord=0, node_y_coord=0, node_size = 3, node_type = Terms, node_id = pack "6", node_label = pack "object", node_attributes = Attributes {clust_default = 2}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "7", node_label = pack "pen", node_attributes = Attributes {clust_default = 2}},Node {node_x_coord=0, node_y_coord=0, node_size = 2, node_type = Terms, node_id = pack "8", node_label = pack "table", node_attributes = Attributes {clust_default = 2}}], _graph_edges = [Edge {edge_source = pack "0", edge_target = pack "0", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "0"},Edge {edge_source = pack "1", edge_target = pack "0", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "1"},Edge {edge_source = pack "1", edge_target = pack "1", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "2"},Edge {edge_source = pack "2", edge_target = pack "2", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "3"},Edge {edge_source = pack "2", edge_target = pack "5", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "4"},Edge {edge_source = pack "3", edge_target = pack "0", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "5"},Edge {edge_source = pack "3", edge_target = pack "1", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "6"},Edge {edge_source = pack "3", edge_target = pack "3", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "7"},Edge {edge_source = pack "4", edge_target = pack "4", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "8"},Edge {edge_source = pack "4", edge_target = pack "5", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "9"},Edge {edge_source = pack "5", edge_target = pack "5", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "10"},Edge {edge_source = pack "6", edge_target = pack "6", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "11"},Edge {edge_source = pack "7", edge_target = pack "6", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "12"},Edge {edge_source = pack "7", edge_target = pack "7", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "13"},Edge {edge_source = pack "8", edge_target = pack "6", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "14"},Edge {edge_source = pack "8", edge_target = pack "7", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "15"},Edge {edge_source = pack "8", edge_target = pack "8", edge_weight = 1.0, edge_confluence=0.5, edge_id = pack "16"}], _graph_metadata = Nothing}
 
 
 -----------------------------------------------------------
@@ -161,7 +162,7 @@ graphV3ToGraph (GraphV3 links nodes) = Graph (map nodeV32node nodes) (zipWith li
                 = Node no_s' Terms (cs $ show no_id') no_lb' 0 0 (Attributes cl')
 
     linkV32edge :: Int -> EdgeV3 -> Edge
-    linkV32edge n (EdgeV3 eo_s' eo_t' eo_w') = Edge (cs $ show eo_s') (cs $ show eo_t') ((T.read $ T.unpack eo_w') :: Double) (cs $ show n)
+    linkV32edge n (EdgeV3 eo_s' eo_t' eo_w') = Edge (cs $ show eo_s') (cs $ show eo_t') ((T.read $ T.unpack eo_w') :: Double) 0.5 (cs $ show n)
 
 
 graphV3ToGraphWithFiles :: FilePath -> FilePath -> IO ()

src/Gargantext/Viz/Graph/Proxemy.hs

@@ -32,6 +32,13 @@ type FalseReflexive = Bool
 type NeighborsFilter = Graph_Undirected -> Node -> [Node]
 type We = Bool
 
+confluence :: [(Node,Node)] -> Length -> FalseReflexive -> We -> Map (Node,Node) Double
+confluence ns l fr we = similarity_conf (mkGraphUfromEdges ns) l fr we
+
+similarity_conf :: Graph_Undirected -> Length -> FalseReflexive -> We -> Map (Node,Node) Double
+similarity_conf g l fr we = Map.fromList [ ((x,y), similarity_conf_x_y g (x,y) l fr we)
+                                         | x <- nodes g, y <- nodes g, x < y]
+
 similarity_conf_x_y :: Graph_Undirected -> (Node,Node) -> Length -> FalseReflexive -> We -> Double
 similarity_conf_x_y g (x,y) l r we = similarity
   where

src/Gargantext/Viz/Graph/Tools.hs

@@ -15,7 +15,7 @@ Portability : POSIX
 module Gargantext.Viz.Graph.Tools
   where
 
--- import Debug.Trace (trace)
+--import Debug.Trace (trace)
 import Data.Graph.Clustering.Louvain.CplusPlus (LouvainNode(..))
 import Data.Graph.Clustering.Louvain.CplusPlus (cLouvain)
 import Data.Map (Map)
@@ -23,10 +23,11 @@ import Data.Text (Text)
 import Gargantext.Prelude
 import Gargantext.Core.Statistics
 import Gargantext.Viz.Graph
---import Gargantext.Viz.Graph.Bridgeness (bridgeness)
+import Gargantext.Viz.Graph.Bridgeness (bridgeness)
 import Gargantext.Viz.Graph.Distances.Matrice (measureConditional)
 import Gargantext.Viz.Graph.Index (createIndices, toIndex, map2mat, mat2map)
 import Gargantext.Viz.Graph.IGraph (mkGraphUfromEdges)
+import Gargantext.Viz.Graph.Proxemy (confluence)
 import GHC.Float (sin, cos)
 import qualified IGraph as Igraph
 import qualified IGraph.Algorithms.Layout as Layout
@@ -40,29 +41,30 @@ cooc2graph myCooc = do
       myCooc4 = toIndex ti myCooc
       matCooc = map2mat (0) (Map.size ti) myCooc4
       distanceMat = measureConditional matCooc
-      distanceMap = Map.map (\_ -> 1) $ Map.filter (>0) $ mat2map distanceMat
+      distanceMap = Map.filter (>0.01) $ mat2map distanceMat
 
   partitions <- case Map.size distanceMap > 0 of
     True  -> cLouvain distanceMap
     False -> panic "Text.Flow: DistanceMap is empty"
 
-  let distanceMap' = distanceMap -- bridgeness 300 partitions distanceMap
+  let bridgeness' = bridgeness 300 partitions distanceMap
+  let confluence' = confluence (Map.keys bridgeness') 3 True False
 
-  data2graph (Map.toList ti) myCooc4 distanceMap' partitions
+  data2graph (Map.toList ti) myCooc4 bridgeness' confluence' partitions
 
 
 ----------------------------------------------------------
 -- | From data to Graph
--- FIXME: distance should not be a map since we just "toList" it (same as cLouvain)
 data2graph :: [(Text, Int)] -> Map (Int, Int) Int
+                            -> Map (Int, Int) Double
                             -> Map (Int, Int) Double
                             -> [LouvainNode]
               -> IO Graph
-data2graph labels coocs distance partitions = do
+data2graph labels coocs bridge conf partitions = do
     
     let community_id_by_node_id = Map.fromList [ (n, c) | LouvainNode n c <- partitions ]
 
-    nodes <- mapM (setCoord ForceAtlas labels distance)
+    nodes <- mapM (setCoord ForceAtlas labels bridge)
           [ (n, Node { node_size = maybe 0 identity (Map.lookup (n,n) coocs)
                    , node_type = Terms -- or Unknown
                    , node_id    = cs (show n)
@@ -78,9 +80,10 @@ data2graph labels coocs distance partitions = do
 
     let edges = [ Edge { edge_source = cs (show s)
                    , edge_target = cs (show t)
-                   , edge_weight = w
+                   , edge_weight = d
+                   , edge_confluence = maybe (panic "E: data2graph edges") identity $ Map.lookup (s,t) conf
                    , edge_id     = cs (show i) }
-            | (i, ((s,t), w)) <- zip ([0..]::[Integer]) (Map.toList distance) ]
+            | (i, ((s,t), d)) <- zip ([0..]::[Integer]) (Map.toList bridge) ]
 
     pure $ Graph nodes edges Nothing
 

stack.yaml

@@ -22,8 +22,10 @@ extra-deps:
   commit: 3fe28b683aba5ddf05e3b5f8eced0bd05c5a29f9
 - git: https://github.com/robstewart57/rdf4h.git
   commit: 4fd2edf30c141600ffad6d730cc4c1c08a6dbce4
-- git: https://gitlab.iscpif.fr/gargantext/crawlers/pubmed
+- git: https://gitlab.iscpif.fr/gargantext/crawlers/pubmed.git
   commit: dcaa0f5dd53f20648f4f5a615d29163582a4219c
+- git: https://gitlab.iscpif.fr/gargantext/crawlers/hal.git
+  commit: bf57642f6b66f554fdc0a38ac391cd8200dffcb3
 - git: https://gitlab.iscpif.fr/gargantext/patches-class
   commit: 746b4ce0af8f9e600d555ad7e5b2973a940cdad9
 - git: https://github.com/np/servant-job.git