Commit aaf48a73 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[Merge]

parents 1b1b4a07 d63bc5be
Pipeline #2044 failed with stage
in 10 minutes and 6 seconds
...@@ -158,6 +158,7 @@ library: ...@@ -158,6 +158,7 @@ library:
- full-text-search - full-text-search
- fullstop - fullstop
- gargantext-prelude - gargantext-prelude
- gargantext-graph >= 0.1.0.0
- graphviz - graphviz
- hashable - hashable
- haskell-igraph - haskell-igraph
......
...@@ -64,7 +64,7 @@ email_to' (NewUser u m _) = (m,u) ...@@ -64,7 +64,7 @@ email_to' (NewUser u m _) = (m,u)
------------------------------------------------------------------------ ------------------------------------------------------------------------
bodyWith :: ServerAddress -> MailModel -> [Text] bodyWith :: ServerAddress -> MailModel -> [Text]
bodyWith server (Invitation u) = [ "Congratulation, you have been granted a beta user account to test the" bodyWith server (Invitation u) = [ "Congratulation, you have been granted a user account to test the"
, "new GarganText platform!" , "new GarganText platform!"
] <> (email_credentials server u) ] <> (email_credentials server u)
......
...@@ -25,9 +25,8 @@ module Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), clean, parseFile, cl ...@@ -25,9 +25,8 @@ module Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), clean, parseFile, cl
import "zip" Codec.Archive.Zip (withArchive, getEntry, getEntries) import "zip" Codec.Archive.Zip (withArchive, getEntry, getEntries)
import Control.Concurrent.Async as CCA (mapConcurrently) import Control.Concurrent.Async as CCA (mapConcurrently)
import Control.Monad (join, sequence)
import Control.Monad.IO.Class (liftIO)
import Data.Attoparsec.ByteString (parseOnly, Parser) import Data.Attoparsec.ByteString (parseOnly, Parser)
import Control.Monad (join)
import Data.Either(Either(..)) import Data.Either(Either(..))
import Data.Either.Extra (partitionEithers) import Data.Either.Extra (partitionEithers)
import Data.List (concat, lookup) import Data.List (concat, lookup)
...@@ -98,13 +97,11 @@ parseFormat WOS bs = do ...@@ -98,13 +97,11 @@ parseFormat WOS bs = do
$ [runParser' WOS bs] $ [runParser' WOS bs]
pure $ Right docs pure $ Right docs
parseFormat ZIP bs = do parseFormat ZIP bs = do
path <- emptySystemTempFile "parsed.zip" path <- emptySystemTempFile "parsed-zip"
DB.writeFile path bs DB.writeFile path bs
withArchive path $ do parsedZip <- withArchive path $ do
files <- DM.keys <$> getEntries DM.keys <$> getEntries
filesContents <- mapM getEntry files pure $ Left $ "Not implemented for ZIP, parsedZip" <> show parsedZip
ddocs <- liftIO $ mapM (parseFormat CsvGargV3) filesContents
pure $ concat <$> sequence ddocs
parseFormat _ _ = undefined parseFormat _ _ = undefined
-- | Parse file into documents -- | Parse file into documents
......
...@@ -119,7 +119,7 @@ getTimeValue rt = case head rt of ...@@ -119,7 +119,7 @@ getTimeValue rt = case head rt of
Nothing Nothing
Just x -> case rval x of Just x -> case rval x of
RVal Time t -> Just $ toJSON t RVal Time t -> Just $ toJSON t
_ -> do _ -> do
Nothing Nothing
extractValue :: Maybe Value -> Maybe Text extractValue :: Maybe Value -> Maybe Text
......
{-| {-|
Module : Gargantext.Core.Text.Corpus.Parsers.Wikidata Module : Gargantext.Core.Text.Corpus.Parsers.Wikidata
<<<<<<< HEAD
Description : To query Wikidata Description : To query Wikidata
=======
Description : To query Wikidata
>>>>>>> dev-clustering
Copyright : (c) CNRS, 2019-Present Copyright : (c) CNRS, 2019-Present
License : AGPL + CECILL v3 License : AGPL + CECILL v3
Maintainer : team@gargantext.org Maintainer : team@gargantext.org
...@@ -29,7 +33,7 @@ import Gargantext.Core.Text.Corpus.Parsers.Date (dateSplit) ...@@ -29,7 +33,7 @@ import Gargantext.Core.Text.Corpus.Parsers.Date (dateSplit)
data WikiResult = WikiResult { _wr_cid :: Maybe Text data WikiResult = WikiResult { _wr_cid :: Maybe Text
, _wr_title :: Maybe Text , _wr_title :: Maybe Text
, _wr_url :: Maybe Text , _wr_url :: Maybe Text
, _wr_yearStart :: Maybe Text , _wr_yearStart :: Maybe Text
...@@ -48,10 +52,9 @@ wikidataGet n m = do ...@@ -48,10 +52,9 @@ wikidataGet n m = do
wikiPageToDocument :: NumberOfSections -> WikiResult -> IO HyperdataDocument wikiPageToDocument :: NumberOfSections -> WikiResult -> IO HyperdataDocument
wikiPageToDocument m wr = do wikiPageToDocument m wr = do
sections <- case wr ^. wr_url of sections <- case wr ^. wr_url of
Nothing -> pure [] Nothing -> pure []
Just u -> crawlPage u Just u -> crawlPage u
let bdd = Just "wikidata" let bdd = Just "wikidata"
doi = Nothing doi = Nothing
...@@ -129,5 +132,3 @@ wikidataQuery n = List.unlines ...@@ -129,5 +132,3 @@ wikidataQuery n = List.unlines
," LIMIT " <> (cs $ show n) ," LIMIT " <> (cs $ show n)
] ]
...@@ -24,7 +24,7 @@ import Data.Maybe (catMaybes) ...@@ -24,7 +24,7 @@ import Data.Maybe (catMaybes)
import Data.Ord (Down(..)) import Data.Ord (Down(..))
import Gargantext.Prelude import Gargantext.Prelude
import qualified Data.Map as DM import qualified Data.Map as DM
import Gargantext.Core.Viz.Graph.Tools.IGraph (ClusterNode(..)) import Graph.Types (ClusterNode(..))
---------------------------------------------------------------------- ----------------------------------------------------------------------
type Partitions a = Map (Int, Int) Double -> IO [a] type Partitions a = Map (Int, Int) Double -> IO [a]
......
...@@ -27,8 +27,10 @@ import Gargantext.Core.Statistics ...@@ -27,8 +27,10 @@ import Gargantext.Core.Statistics
import Gargantext.Core.Viz.Graph import Gargantext.Core.Viz.Graph
import Gargantext.Core.Viz.Graph.Bridgeness (bridgeness, Partitions, ToComId(..)) import Gargantext.Core.Viz.Graph.Bridgeness (bridgeness, Partitions, ToComId(..))
import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex, map2mat, mat2map, Index, MatrixShape(..)) import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex, map2mat, mat2map, Index, MatrixShape(..))
import Gargantext.Core.Viz.Graph.Tools.IGraph (mkGraphUfromEdges, spinglass, ClusterNode) import Gargantext.Core.Viz.Graph.Tools.IGraph (mkGraphUfromEdges, spinglass)
import Gargantext.Prelude import Gargantext.Prelude
import Graph.Types (ClusterNode)
import qualified Graph.BAC.ProxemyOptim as BAC
import IGraph.Random -- (Gen(..)) import IGraph.Random -- (Gen(..))
import qualified Data.HashMap.Strict as HashMap import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List import qualified Data.List as List
...@@ -40,12 +42,10 @@ import qualified IGraph.Algorithms.Layout as Layout ...@@ -40,12 +42,10 @@ import qualified IGraph.Algorithms.Layout as Layout
------------------------------------------------------------- -------------------------------------------------------------
defaultClustering :: Map (Int, Int) Double -> IO [ClusterNode] defaultClustering :: Map (Int, Int) Double -> IO [ClusterNode]
defaultClustering = spinglass 1 defaultClustering x = pure $ BAC.defaultClustering x
------------------------------------------------------------- -------------------------------------------------------------
type Threshold = Double type Threshold = Double
cooc2graph' :: Ord t => Distance cooc2graph' :: Ord t => Distance
......
...@@ -20,6 +20,7 @@ import Data.Singletons (SingI) ...@@ -20,6 +20,7 @@ import Data.Singletons (SingI)
import IGraph hiding (mkGraph, neighbors, edges, nodes, Node, Graph) import IGraph hiding (mkGraph, neighbors, edges, nodes, Node, Graph)
import Protolude import Protolude
import Gargantext.Core.Viz.Graph.Index import Gargantext.Core.Viz.Graph.Index
import Graph.Types
import qualified Data.List as List import qualified Data.List as List
import qualified IGraph as IG import qualified IGraph as IG
import qualified IGraph.Algorithms.Clique as IG import qualified IGraph.Algorithms.Clique as IG
...@@ -82,10 +83,6 @@ partitions_spinglass' s g = do ...@@ -82,10 +83,6 @@ partitions_spinglass' s g = do
IG.findCommunity g Nothing Nothing IG.spinglass gen IG.findCommunity g Nothing Nothing IG.spinglass gen
data ClusterNode = ClusterNode { cl_node_id :: Int
, cl_community_id :: Int
}
toClusterNode :: [[Int]] -> [ClusterNode] toClusterNode :: [[Int]] -> [ClusterNode]
toClusterNode ns = List.concat toClusterNode ns = List.concat
$ map (\(cId, ns') -> map (\n -> ClusterNode n cId) ns') $ map (\(cId, ns') -> map (\n -> ClusterNode n cId) ns')
......
...@@ -4,7 +4,7 @@ flags: {} ...@@ -4,7 +4,7 @@ flags: {}
extra-package-dbs: [] extra-package-dbs: []
packages: packages:
- . - .
#- 'deps/patches-class' #- 'deps/gargantext-graph'
#- 'deps/patches-map' #- 'deps/patches-map'
#- 'deps/accelerate' #- 'deps/accelerate'
#- 'deps/accelerate-utility' #- 'deps/accelerate-utility'
...@@ -30,6 +30,9 @@ extra-deps: ...@@ -30,6 +30,9 @@ extra-deps:
- #git: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude.git - #git: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude.git
git: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude.git git: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude.git
commit: 35b09629a658fc16cc9ff63e7591e58511cd98a7 commit: 35b09629a658fc16cc9ff63e7591e58511cd98a7
#git: ssh://git@gitlab.iscpif.fr:20022/gargantext/gargantext-graph.git
- git: ssh://gitolite3@delanoe.org/gargantext-graph
commit: 294887a220460bd0c114638fff9ea53306cd2f18
# Data Mining Libs # Data Mining Libs
- git: https://github.com/delanoe/data-time-segment.git - git: https://github.com/delanoe/data-time-segment.git
commit: 10a416b9f6c443866b36479c3441ebb3bcdeb7ef commit: 10a416b9f6c443866b36479c3441ebb3bcdeb7ef
...@@ -88,7 +91,18 @@ extra-deps: ...@@ -88,7 +91,18 @@ extra-deps:
- git: https://github.com/rspeer/wikiparsec.git - git: https://github.com/rspeer/wikiparsec.git
commit: 9637a82344bb70f7fa8f02e75db3c081ccd434ce commit: 9637a82344bb70f7fa8f02e75db3c081ccd434ce
# Others dependencies (using stack resolver) # Gargantext-graph
- eigen-3.3.7.0@sha256:7c24a86426b238e02ba0ac2179cc44997518b443a13828ed7a791fe49a6dffa5,82060
- git: https://github.com/alpmestan/sparse-linear.git
commit: bc6ca8058077b0b5702ea4b88bd4189cfcad267a
subdirs:
- sparse-linear
- git: https://github.com/alpmestan/hmatrix.git
commit: b9fca8beee0f23c17a6b2001ec834d071709e6e7
subdirs:
- packages/base
# Others dependencies (using stack resolver)
- constraints-extras-0.3.1.0@sha256:12016ebb91ad5ed2c82bf7e48c6bd6947d164d33c9dca5ac3965de1bb6c780c0,1777 - constraints-extras-0.3.1.0@sha256:12016ebb91ad5ed2c82bf7e48c6bd6947d164d33c9dca5ac3965de1bb6c780c0,1777
- KMP-0.2.0.0@sha256:6dfbac03ef00ebd9347234732cb86a40f62ab5a80c0cc6bedb8eb51766f7df28,2562 - KMP-0.2.0.0@sha256:6dfbac03ef00ebd9347234732cb86a40f62ab5a80c0cc6bedb8eb51766f7df28,2562
- Unique-0.4.7.8@sha256:9661f45aa31dde119a2114566166ea38b011a45653337045ee4ced75636533c0,2067 - Unique-0.4.7.8@sha256:9661f45aa31dde119a2114566166ea38b011a45653337045ee4ced75636533c0,2067
...@@ -122,3 +136,10 @@ extra-deps: ...@@ -122,3 +136,10 @@ extra-deps:
- taggy-0.2.1@sha256:7bc55ddba178971dc6052163597f0445a0a2b5b0ca0e84ce651d53d722e3c265,4662 - taggy-0.2.1@sha256:7bc55ddba178971dc6052163597f0445a0a2b5b0ca0e84ce651d53d722e3c265,4662
- servant-ekg-0.3.1@sha256:19bd9dc3943983da8e79d6f607614c68faea4054fb889d508c8a2b67b6bdd448,2203 - servant-ekg-0.3.1@sha256:19bd9dc3943983da8e79d6f607614c68faea4054fb889d508c8a2b67b6bdd448,2203
# For the graph clustering
ghc-options:
hmatrix: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal -fno-state-hack
sparse-linear: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal -fno-state-hack
gargantext-graph: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal -fno-state-hack
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment