Commit 37a16868 authored by Alexandre Delanoë's avatar Alexandre Delanoë

Merge remote-tracking branch 'origin/274-dev-more-protolude-in-prelude' into dev

parents 1d836ed3 855ad039
......@@ -30,5 +30,5 @@ main = do
withDevEnv iniPath $ \env -> do
x <- runCmdDev env ((newUsers $ map cs mails) :: Cmd'' DevEnv GargError [UserId])
putStrLn $ show x
putStrLn (show x :: Text)
pure ()
......@@ -14,17 +14,14 @@ compress the contexts around the main terms of the query.
module CleanCsvCorpus where
--import GHC.IO (FilePath)
import Data.Either (Either(..))
import Data.SearchEngine as S
import qualified Data.Set as S
import Data.SearchEngine qualified as S
import Data.Set qualified as S
import Data.Text (pack)
import Data.Vector (Vector)
import qualified Data.Vector as V
import Gargantext.Prelude
import Data.Vector qualified as V
import Gargantext.Core.Text.Corpus.Parsers.CSV qualified as CSV
import Gargantext.Core.Text.Search
import qualified Gargantext.Core.Text.Corpus.Parsers.CSV as CSV
import Gargantext.Prelude
------------------------------------------------------------------------
type Query = [S.Term]
......@@ -43,16 +40,16 @@ main = do
eDocs <- CSV.readCSVFile rPath
case eDocs of
Right (h, csvDocs) -> do
putStrLn $ "Number of documents before:" <> show (V.length csvDocs)
putStrLn $ "Mean size of docs:" <> show ( CSV.docsSize csvDocs)
putStrLn ("Number of documents before:" <> show (V.length csvDocs) :: Text)
putStrLn ("Mean size of docs:" <> show ( CSV.docsSize csvDocs) :: Text)
let docs = CSV.toDocs csvDocs
let engine = insertDocs docs initialDocSearchEngine
let engine = S.insertDocs docs initialDocSearchEngine
let docIds = S.query engine (map pack q)
let docs' = CSV.fromDocs $ filterDocs docIds (V.fromList docs)
putStrLn $ "Number of documents after:" <> show (V.length docs')
putStrLn $ "Mean size of docs:" <> show (CSV.docsSize docs')
putStrLn ("Number of documents after:" <> show (V.length docs') :: Text)
putStrLn ("Mean size of docs:" <> show (CSV.docsSize docs') :: Text)
CSV.writeFile wPath (h, docs')
Left e -> panic $ "Error: " <> e
......@@ -15,7 +15,6 @@ Import a corpus binary.
module Main where
import Data.Either (Either(..))
import Gargantext.API.Dev (withDevEnv, runCmdDev)
import Gargantext.API.Node () -- instances only
import Gargantext.API.Prelude (GargError)
......@@ -30,7 +29,6 @@ import Gargantext.Database.Query.Table.Node (getOrMkList)
import Gargantext.Database.Query.Table.User (insertNewUsers, )
import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(..), readConfig)
import Prelude (getLine)
import System.Environment (getArgs)
......@@ -42,10 +40,10 @@ main = do
then panic "USAGE: ./gargantext-init gargantext.ini"
else pure ()
putStrLn "Enter master user (gargantua) _password_ :"
putStrLn ("Enter master user (gargantua) _password_ :" :: Text)
password <- getLine
putStrLn "Enter master user (gargantua) _email_ :"
putStrLn ("Enter master user (gargantua) _email_ :" :: Text)
email <- getLine
cfg <- readConfig iniPath
......@@ -77,5 +75,5 @@ main = do
_ <- runCmdDev env createUsers
x <- runCmdDev env initMaster
_ <- runCmdDev env mkRoots
putStrLn $ show x
putStrLn (show x :: Text)
pure ()
......@@ -16,22 +16,23 @@ Adaptative Phylo binaries
module Main where
-- import Debug.Trace (trace)
import Control.Concurrent.Async (mapConcurrently)
import Crypto.Hash.SHA256 (hash)
import Data.Aeson
import Data.Either (Either(..), fromRight)
import Data.List (concat, nub, isSuffixOf,sort,tail)
import Data.ByteString.Char8 qualified as C8
import Data.List (nub, isSuffixOf, tail)
import Data.List.Split
import Data.Maybe (fromMaybe)
import Data.String (String)
import Data.Text (Text, unwords, unpack, replace, pack)
import Data.Maybe (fromJust)
import Data.Text (unwords, unpack, replace, pack)
import Data.Text qualified as T
import Data.Vector qualified as Vector
import GHC.IO (FilePath)
import Gargantext.API.Ngrams.Prelude (toTermList)
import Gargantext.API.Ngrams.Types
import Gargantext.Core.Text.Context (TermList)
import Gargantext.Core.Text.Corpus.Parsers (FileFormat(..), FileType(..), parseFile)
import Gargantext.Core.Text.Corpus.Parsers.CSV (csv_title, csv_abstract, csv_publication_year, csv_publication_month, csv_publication_day, csv'_source, csv'_title, csv'_abstract, csv'_publication_year, csv'_publication_month, csv'_publication_day, csv'_weight)
import Gargantext.Core.Text.Corpus.Parsers.CSV qualified as Csv
import Gargantext.Core.Text.List.Formats.CSV (csvMapTermList)
import Gargantext.Core.Text.Terms.WithList (Patterns, buildPatterns, extractTermsWithList)
import Gargantext.Core.Types.Main (ListType(..))
......@@ -42,13 +43,10 @@ import Gargantext.Core.Viz.Phylo.PhyloMaker (toPhylo, toPhyloWithoutLink)
import Gargantext.Core.Viz.Phylo.PhyloTools (printIOMsg, printIOComment, setConfig, toPeriods, getTimePeriod, getTimeStep)
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Database.Schema.Ngrams (NgramsType(..))
import Gargantext.Prelude
import Gargantext.Prelude hiding (hash, replace)
import Prelude qualified
import System.Directory (listDirectory,doesFileExist)
import System.Environment
import qualified Data.ByteString.Char8 as C8
import qualified Data.Text as T
import qualified Data.Vector as Vector
import qualified Gargantext.Core.Text.Corpus.Parsers.CSV as Csv
data Backup = BackupPhyloWithoutLink | BackupPhylo deriving (Show)
......@@ -110,7 +108,7 @@ csvToDocs parser patterns time path =
Nothing
[]
time
) <$> snd <$> either (\err -> panic $ cs $ "CSV error" <> (show err)) identity <$> Csv.readCSVFile path
) <$> snd <$> either (\err -> panic $ "CSV error" <> (show err)) identity <$> Csv.readCSVFile path
Csv' limit -> Vector.toList
<$> Vector.take limit
<$> Vector.map (\row -> Document (toPhyloDate (csv'_publication_year row) (csv'_publication_month row) (csv'_publication_day row) time)
......@@ -232,7 +230,7 @@ configToSha stage config = unpack
readListV4 :: [Char] -> IO NgramsList
readListV4 path = do
listJson <- (eitherDecode <$> readJson path) :: IO (Either String NgramsList)
listJson <- (eitherDecode <$> readJson path) :: IO (Either Prelude.String NgramsList)
case listJson of
Left err -> do
putStrLn err
......@@ -261,7 +259,7 @@ main = do
printIOMsg "Read the configuration file"
[args] <- getArgs
jsonArgs <- (eitherDecode <$> readJson args) :: IO (Either String PhyloConfig)
jsonArgs <- (eitherDecode <$> readJson args) :: IO (Either Prelude.String PhyloConfig)
case jsonArgs of
Left err -> putStrLn err
......
......@@ -16,24 +16,22 @@ Import a corpus binary.
module Main where
import Data.List qualified as List (cycle, concat, take, unlines)
import Gargantext.API.Dev (withDevEnv)
import Gargantext.API.Node () -- instances only
import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(..), readConfig)
import Prelude (getLine)
import System.Environment (getArgs)
import qualified Data.List as List (cycle, concat, take, unlines)
import Prelude qualified
main :: IO ()
main = do
let ___ = putStrLn
$ List.concat
$ List.take 72
$ List.cycle ["_"]
let ___ = putStrLn ((List.concat
$ List.take 72
$ List.cycle ["_"]) :: Prelude.String)
___
putStrLn "GarganText upgrade to version 0.0.6.9.9.4.4"
putStrLn ("GarganText upgrade to version 0.0.6.9.9.4.4" :: Text)
___
params@[iniPath] <- getArgs
......@@ -53,13 +51,13 @@ main = do
cfg <- readConfig iniPath
let _secret = _gc_secretkey cfg
withDevEnv iniPath $ \env -> do
withDevEnv iniPath $ \_env -> do
-- _ <- runCmdDev env addIndex
-- _ <- runCmdDev env refreshIndex
___
putStrLn "Uprade done with success !"
putStrLn ("Uprade done with success !" :: Text)
___
pure ()
......
......@@ -11,7 +11,7 @@ STORE_DIR="${1:-$DEFAULT_STORE}"
# `expected_cabal_project_freeze_hash` with the
# `sha256sum` result calculated on the `cabal.project` and `cabal.project.freeze`.
# This ensures the files stay deterministic so that CI cache can kick in.
expected_cabal_project_hash="297d2ac44b8a2e65a9d7bbbe1fb6e5e0ff46b144300501c14e5e424e77aa1abf"
expected_cabal_project_hash="3c1002c8ed7be226b2e189fdb7debef5b3c43d0f56e44df73d500954074c4568"
expected_cabal_project_freeze_hash="2d3704d107bd8d08056ce4f0eb1f42202cb7f49a67c62a2445a6c70c7235f861"
cabal --store-dir=$STORE_DIR v2-update 'hackage.haskell.org,2023-06-24T21:28:46Z'
......
......@@ -114,7 +114,7 @@ source-repository-package
source-repository-package
type: git
location: https://gitlab.iscpif.fr/gargantext/haskell-gargantext-prelude
tag: 8f97fef4dfd941d773914ad058d8e02ce2bb1a3e
tag: 40135d166830690b1101180b79e9fd3663284b2b
source-repository-package
type: git
......@@ -167,4 +167,3 @@ package hmatrix
package sparse-linear
ghc-options: -O2 -fsimpl-tick-factor=10000 -fdicts-cheap -fdicts-strict -flate-dmd-anal -fno-state-hack
......@@ -21,5 +21,5 @@ module Gargantext ( module Gargantext.API
import Gargantext.API
import Gargantext.Core
import Gargantext.Database
import Gargantext.Prelude
import Gargantext.Prelude hiding (All)
--import Gargantext.Core.Viz
......@@ -35,18 +35,11 @@ module Gargantext.API
where
import Control.Concurrent
import Control.Exception (catch, finally, SomeException{-, displayException, IOException-})
import Control.Lens hiding (Level)
import Control.Monad.Except
import Control.Monad.Reader (runReaderT)
import Data.Either
import Data.Foldable (foldlM)
import Data.List (lookup)
import Data.Text (pack)
import Data.Text.Encoding (encodeUtf8)
import Data.Text.IO (putStrLn)
import Data.Validity
import GHC.Base (Applicative)
import Gargantext.API.Admin.Auth.Types (AuthContext)
import Gargantext.API.Admin.EnvTypes (Env, Mode(..))
import Gargantext.API.Admin.Settings (newEnv)
......
......@@ -41,39 +41,36 @@ module Gargantext.API.Admin.Auth
)
where
--import Control.Monad.Logger.Aeson
--import qualified Text.Blaze.Html5.Attributes as HA
import Control.Lens (view, (#))
import Data.Aeson
import Data.Swagger (ToSchema(..))
import Data.Text qualified as Text
import Data.Text.Lazy.Encoding qualified as LE
import Data.UUID (UUID, fromText, toText)
import Data.UUID.V4 (nextRandom)
import Gargantext.API.Admin.Auth.Types
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Admin.Types
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.Prelude (HasJoseError(..), joseError, HasServerError, GargServerC, GargServer, _ServerError, GargM, GargError (..))
import Gargantext.Core.Mail (MailModel(..), mail)
import Gargantext.Core.Mail.Types (mailSettings)
import Gargantext.Core.Types.Individu (User(..), Username, GargPassword(..))
import Gargantext.Database.Action.Flow.Types (FlowCmdM)
import Gargantext.Database.Action.User.New (guessUserName)
import Gargantext.Database.Admin.Types.Node (NodeId(..))
import Gargantext.Database.Prelude (Cmd', CmdCommon, DbCmd')
import Gargantext.Database.Query.Table.User
import Gargantext.Database.Query.Tree (isDescendantOf, isIn)
import Gargantext.Database.Query.Tree.Root (getRoot)
import Gargantext.Database.Action.User.New (guessUserName)
import Gargantext.Database.Schema.Node (NodePoly(_node_id))
import Gargantext.Prelude hiding (reverse)
import Gargantext.Prelude hiding (Handler, reverse, to)
import Gargantext.Prelude.Crypto.Auth qualified as Auth
import Gargantext.Prelude.Crypto.Pass.User (gargPass)
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Protolude hiding (Handler, to)
import Servant
import Servant.Auth.Server
import qualified Data.Text as Text
import qualified Data.Text.Lazy.Encoding as LE
import qualified Gargantext.Prelude.Crypto.Auth as Auth
import Gargantext.API.Auth.PolicyCheck
---------------------------------------------------
......
......@@ -16,16 +16,13 @@ module Gargantext.API.Admin.Auth.Types
import Control.Lens hiding (elements, to)
import Data.Aeson.TH (deriveJSON)
import Data.Swagger
import Data.Text (Text)
import GHC.Generics (Generic)
import Servant.Auth.Server
import Test.QuickCheck (elements, oneof)
import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
import Gargantext.Core.Types.Individu (Username, GargPassword(..), arbitraryUsername, arbitraryPassword)
import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
import Gargantext.Database.Admin.Types.Node (NodeId(..), ListId, DocId, UserId)
import Gargantext.Prelude hiding (reverse)
import Servant.Auth.Server
import Test.QuickCheck (elements, oneof)
import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
---------------------------------------------------
......@@ -101,8 +98,8 @@ $(deriveJSON (unPrefix "_authVal_") ''AuthValid)
instance ToSchema AuthValid where
declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_authVal_")
instance Arbitrary AuthValid where
arbitrary = elements [ AuthValid to tr u
| to <- ["token0", "token1"]
arbitrary = elements [ AuthValid to' tr u
| to' <- ["token0", "token1"]
, tr <- [1..3]
, u <- [1..3]
]
......
......@@ -25,18 +25,11 @@ module Gargantext.API.Admin.EnvTypes (
import Control.Lens hiding (Level, (:<))
import Control.Monad.Except
import Control.Monad.Reader
import Data.List ((\\))
import Data.Pool (Pool)
import Data.Sequence (Seq, ViewL(..), viewl)
import Data.Sequence (ViewL(..), viewl)
import Data.Text qualified as T
import Database.PostgreSQL.Simple (Connection)
import GHC.Generics (Generic)
import Network.HTTP.Client (Manager)
import Servant.Client (BaseUrl)
import Servant.Job.Async (HasJobEnv(..), Job)
import qualified Servant.Job.Async as SJ
import qualified Servant.Job.Core
import Data.List ((\\))
import qualified Data.Text as T
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.API.Admin.Types
import Gargantext.API.Job
......@@ -49,10 +42,14 @@ import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(..))
import Gargantext.Prelude.Mail.Types (MailConfig)
import Gargantext.System.Logging
import qualified System.Log.FastLogger as FL
import qualified Gargantext.Utils.Jobs.Monad as Jobs
import Gargantext.Utils.Jobs.Map (LoggerM, J(..), jTask, rjGetLog)
import Gargantext.Utils.Jobs.Monad qualified as Jobs
import Network.HTTP.Client (Manager)
import Servant.Client (BaseUrl)
import Servant.Job.Async (HasJobEnv(..), Job)
import Servant.Job.Async qualified as SJ
import Servant.Job.Core qualified
import System.Log.FastLogger qualified as FL
data Mode = Dev | Mock | Prod
deriving (Show, Read, Generic)
......@@ -83,7 +80,7 @@ instance HasLogger (GargM Env GargError) where
pure $ GargLogger mode logger_set
destroyLogger = \GargLogger{..} -> liftIO $ FL.rmLoggerSet logger_set
logMsg = \(GargLogger mode logger_set) lvl msg -> do
let pfx = "[" <> show lvl <> "] "
let pfx = "[" <> show lvl <> "] " :: Text
when (lvl `elem` (modeToLoggingLevels mode)) $
liftIO $ FL.pushLogStrLn logger_set $ FL.toLogStr pfx <> msg
logTxt lgr lvl msg = logMsg lgr lvl (FL.toLogStr $ T.unpack msg)
......@@ -253,7 +250,7 @@ instance HasLogger (GargM DevEnv GargError) where
pure $ GargDevLogger mode dev_logger_set
destroyLogger = \GargDevLogger{..} -> liftIO $ FL.rmLoggerSet dev_logger_set
logMsg = \(GargDevLogger mode logger_set) lvl msg -> do
let pfx = "[" <> show lvl <> "] "
let pfx = "[" <> show lvl <> "] " :: Text
when (lvl `elem` (modeToLoggingLevels mode)) $
liftIO $ FL.pushLogStrLn logger_set $ FL.toLogStr pfx <> msg
logTxt lgr lvl msg = logMsg lgr lvl (FL.toLogStr $ T.unpack msg)
......
......@@ -13,14 +13,13 @@ module Gargantext.API.Admin.Orchestrator where
import Control.Lens hiding (elements)
import Data.Aeson
import Data.ByteString.Lazy.Char8 qualified as LBS
import Gargantext.API.Admin.Orchestrator.Scrapy.Schedule
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.Prelude hiding (to)
import Servant
import Servant.Job.Async
import Servant.Job.Client
import qualified Data.ByteString.Lazy.Char8 as LBS
import Gargantext.API.Admin.Orchestrator.Scrapy.Schedule
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.Prelude
callJobScrapy :: (ToJSON e, FromJSON e, FromJSON o, MonadClientJob m)
=> JobServerURL e Schedule o
......@@ -64,7 +63,7 @@ pipeline :: FromJSON e => URL -> ClientEnv -> ScraperInput
-> (e -> IO ()) -> IO JobLog
pipeline scrapyurl client_env input log_status = do
e <- runJobMLog client_env log_status $ callScraper scrapyurl input
either (panic . cs . show) pure e -- TODO throwError
either (panic . show) pure e -- TODO throwError
-- TODO integrate to ServerT
-- use:
......@@ -86,4 +85,4 @@ scrapyOrchestrator env = do
(env ^. env_manager) (LogEvent logConsole) $
simpleServeJobsAPI (env ^. env_scrapers) .
simpleJobFunction . pipeline (URL $ env ^. settings . scrapydUrl)
-}
\ No newline at end of file
-}
......@@ -7,13 +7,13 @@ module Gargantext.API.Admin.Orchestrator.Types
import Control.Lens hiding (elements)
import Data.Aeson
import Data.Morpheus.Types
( GQLType
, typeOptions )
import Data.Morpheus.Types ( GQLType, typeOptions )
import Data.Proxy
import Data.Swagger hiding (URL, url, port)
import Data.Text (Text)
import GHC.Generics hiding (to)
import Gargantext.API.GraphQL.Utils qualified as GQLU
import Gargantext.Core.Types (TODO(..))
import Gargantext.Prelude
import Servant
import Servant.Job.Async
import Servant.Job.Types
......@@ -21,10 +21,6 @@ import Servant.Job.Utils (jsonOptions)
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import qualified Gargantext.API.GraphQL.Utils as GQLU
import Gargantext.Core.Types (TODO(..))
import Gargantext.Prelude
------------------------------------------------------------------------
instance Arbitrary a => Arbitrary (JobStatus 'Safe a) where
arbitrary = panic "TODO"
......
......@@ -19,42 +19,36 @@ TODO-SECURITY: Critical
module Gargantext.API.Admin.Settings
where
-- import Control.Debounce (mkDebounce, defaultDebounceSettings, debounceFreq, debounceAction)
import Codec.Serialise (Serialise(), serialise)
import Control.Lens
import Control.Monad.Logger (LogLevel(..))
import Control.Monad.Reader
import Data.Maybe (fromMaybe)
import Data.ByteString.Lazy qualified as L
import Data.Pool (Pool, createPool)
import Database.PostgreSQL.Simple (Connection, connect, close, ConnectInfo)
import Gargantext.Core.NodeStory
import Gargantext.Prelude.Config ({-GargConfig(..),-} {-gc_repofilepath,-} readConfig)
import Network.HTTP.Client.TLS (newTlsManager)
import Servant.Auth.Server (defaultJWTSettings, CookieSettings(..), XsrfCookieSettings(..), defaultCookieSettings, defaultXsrfCookieSettings, readKey, writeKey)
import Servant.Client (parseBaseUrl)
import Servant.Job.Async (newJobEnv, defaultSettings)
import System.Directory
-- import System.FileLock (tryLockFile, unlockFile, SharedExclusive(Exclusive))
import System.IO (FilePath, hClose)
import System.IO.Temp (withTempFile)
import qualified Data.ByteString.Lazy as L
import Gargantext.API.Admin.EnvTypes
import Gargantext.API.Admin.Types
import Gargantext.API.Prelude
-- import Gargantext.API.Ngrams.Types (NgramsRepo, HasRepo(..), RepoEnv(..), r_version, initRepo, renv_var, renv_lock)
import Gargantext.Core.NLP (nlpServerMap)
import Gargantext.Core.NodeStory
import Gargantext.Database.Prelude (databaseParameters, hasConfig)
import Gargantext.Prelude
import Gargantext.Prelude.Config (gc_js_job_timeout, gc_js_id_timeout)
import qualified Gargantext.Prelude.Mail as Mail
import qualified Gargantext.Prelude.NLP as NLP
import qualified Gargantext.Utils.Jobs as Jobs
import qualified Gargantext.Utils.Jobs.Monad as Jobs
import qualified Gargantext.Utils.Jobs.Queue as Jobs
import qualified Gargantext.Utils.Jobs.Settings as Jobs
import Gargantext.Prelude.Config ({-GargConfig(..),-} {-gc_repofilepath,-} readConfig)
import Gargantext.Prelude.Mail qualified as Mail
import Gargantext.Prelude.NLP qualified as NLP
import Gargantext.System.Logging
import Gargantext.Utils.Jobs qualified as Jobs
import Gargantext.Utils.Jobs.Monad qualified as Jobs
import Gargantext.Utils.Jobs.Queue qualified as Jobs
import Gargantext.Utils.Jobs.Settings qualified as Jobs
import Network.HTTP.Client.TLS (newTlsManager)
import Servant.Auth.Server (defaultJWTSettings, CookieSettings(..), XsrfCookieSettings(..), defaultCookieSettings, defaultXsrfCookieSettings, readKey, writeKey)
import Servant.Client (parseBaseUrl)
import Servant.Job.Async (newJobEnv, defaultSettings)
import System.Directory
import System.IO (hClose)
import System.IO.Temp (withTempFile)
devSettings :: FilePath -> IO Settings
devSettings jwkFile = do
......@@ -187,8 +181,8 @@ newEnv logger port file = do
!config_env <- readConfig file
prios <- withLogger () $ \ioLogger -> Jobs.readPrios ioLogger (file <> ".jobs")
let prios' = Jobs.applyPrios prios Jobs.defaultPrios
putStrLn $ "Overrides: " <> show prios
putStrLn $ "New priorities: " <> show prios'
putStrLn ("Overrides: " <> show prios :: Text)
putStrLn ("New priorities: " <> show prios' :: Text)
!self_url_env <- parseBaseUrl $ "http://0.0.0.0:" <> show port
dbParam <- databaseParameters file
!pool <- newPool dbParam
......
......@@ -6,9 +6,7 @@ module Gargantext.API.Admin.Types where
import Control.Lens
import Control.Monad.Logger (LogLevel)
import Data.ByteString (ByteString)
import GHC.Enum
import GHC.Generics (Generic)
import Gargantext.Prelude
import Servant.Auth.Server (JWTSettings, CookieSettings(..))
import Servant.Client (BaseUrl)
......
......@@ -15,7 +15,6 @@ Mainly copied from Servant.Job.Utils (Thanks)
module Gargantext.API.Admin.Utils
where
import Data.Maybe (fromMaybe)
import Gargantext.Prelude
import Prelude (String)
import qualified Data.Text as T
......
......@@ -10,6 +10,8 @@ Portability : POSIX
Count API part of Gargantext.
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE DeriveAnyClass #-}
......@@ -20,10 +22,8 @@ module Gargantext.API.Count
import Data.Aeson hiding (Error)
import Data.Aeson.TH (deriveJSON)
import Data.Either
import Data.List (permutations)
import Data.Swagger
import Data.Text (Text, pack)
import GHC.Generics (Generic)
import Data.Text (pack)
import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
import Gargantext.Prelude
import Servant
......
......@@ -12,10 +12,7 @@ Portability : POSIX
-- Use only for dev/repl
module Gargantext.API.Dev where
import Control.Exception (finally)
import Control.Monad (fail)
import Control.Monad.Except (runExceptT)
import Control.Monad.Reader (runReaderT)
import Gargantext.API.Admin.EnvTypes
import Gargantext.API.Admin.Settings
import Gargantext.API.Ngrams (saveNodeStoryImmediate)
......@@ -29,7 +26,6 @@ import Gargantext.Prelude.Mail qualified as Mail
import Gargantext.Prelude.NLP qualified as NLP
import Gargantext.System.Logging
import Servant
import System.IO (FilePath)
type IniPath = FilePath
-------------------------------------------------------------------
......
......@@ -10,6 +10,8 @@ Portability : POSIX
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
module Gargantext.API.Flow
where
......
{-|
Module : Gargantext.API.GraphQL
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# OPTIONS_GHC -fprint-potential-instances #-}
{-# LANGUAGE DeriveAnyClass #-}
......@@ -8,49 +18,32 @@
module Gargantext.API.GraphQL where
import Data.ByteString.Lazy.Char8
( ByteString
)
import Data.Map.Strict (Map)
import Data.Morpheus
( App
, deriveApp )
import Data.Morpheus.Server
( httpPlayground
)
import Data.Morpheus.Subscriptions
( Event (..)
, Hashable
, httpPubApp
)
import Data.Morpheus.Types
( GQLRequest
, GQLResponse
, GQLType
, RootResolver(..)
, Undefined(..)
)
import Data.ByteString.Lazy.Char8 ( ByteString )
import Data.Morpheus ( App, deriveApp )
import Data.Morpheus.Server ( httpPlayground )
import Data.Morpheus.Subscriptions ( Event (..), httpPubApp )
import Data.Morpheus.Types ( GQLRequest, GQLResponse, GQLType, RootResolver(..), Undefined(..) )
import Data.Proxy
import Data.Text (Text)
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
import Gargantext.API.Admin.Orchestrator.Types (JobLog)
import Gargantext.API.Prelude (HasJobEnv')
import qualified Gargantext.API.GraphQL.Annuaire as GQLA
import qualified Gargantext.API.GraphQL.AsyncTask as GQLAT
import qualified Gargantext.API.GraphQL.Context as GQLCTX
import qualified Gargantext.API.GraphQL.IMT as GQLIMT
import qualified Gargantext.API.GraphQL.NLP as GQLNLP
import qualified Gargantext.API.GraphQL.Node as GQLNode
import qualified Gargantext.API.GraphQL.User as GQLUser
import qualified Gargantext.API.GraphQL.UserInfo as GQLUserInfo
import qualified Gargantext.API.GraphQL.TreeFirstLevel as GQLTree
import qualified Gargantext.API.GraphQL.Team as GQLTeam
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.GraphQL.Annuaire qualified as GQLA
import Gargantext.API.GraphQL.AsyncTask qualified as GQLAT
import Gargantext.API.GraphQL.Context qualified as GQLCTX
import Gargantext.API.GraphQL.IMT qualified as GQLIMT
import Gargantext.API.GraphQL.NLP qualified as GQLNLP
import Gargantext.API.GraphQL.Node qualified as GQLNode
import Gargantext.API.GraphQL.Team qualified as GQLTeam
import Gargantext.API.GraphQL.TreeFirstLevel qualified as GQLTree
import Gargantext.API.GraphQL.User qualified as GQLUser
import Gargantext.API.GraphQL.UserInfo qualified as GQLUserInfo
import Gargantext.API.Prelude (GargM, GargError)
import Gargantext.API.Prelude (HasJobEnv')
import Gargantext.API.Types
import Gargantext.Core.NLP (HasNLPServer)
import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Prelude
import GHC.Generics (Generic)
import Gargantext.Prelude hiding (ByteString)
import Servant
( (:<|>) (..)
, (:>)
......@@ -60,10 +53,8 @@ import Servant
, ReqBody
, ServerT
)
import qualified Servant.Auth as SA
import qualified Servant.Auth.Server as SAS
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Auth.PolicyCheck
import Servant.Auth qualified as SA
import Servant.Auth.Server qualified as SAS
-- | Represents possible GraphQL queries.
......
......@@ -4,12 +4,8 @@
module Gargantext.API.GraphQL.Annuaire where
import Control.Lens
import Data.Morpheus.Types
( GQLType
, lift
)
import Data.Morpheus.Types ( GQLType )
import Data.Proxy
import Data.Text (Text)
import Gargantext.Database.Admin.Types.Hyperdata.Contact
( HyperdataContact
, ContactWho
......@@ -21,7 +17,6 @@ import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Database.Query.Table.Context (getContextWith)
import Gargantext.Database.Schema.Node (node_hyperdata)
import Gargantext.Prelude
import GHC.Generics (Generic)
import Gargantext.API.GraphQL.Types
data AnnuaireContact = AnnuaireContact
......
{-|
Module : Gargantext.API.GraphQL.AsyncTask
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.AsyncTask where
import Control.Concurrent.Async (poll)
import Control.Concurrent.MVar (readMVar)
import Control.Lens
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as Map
import Control.Monad.Reader (ask, liftIO)
import Data.Either (Either(..))
import qualified Data.IntMap.Strict as IntMap
import Data.Maybe (catMaybes)
import Data.Morpheus.Types
( GQLType
, Resolver
, QUERY
, lift
)
import Data.IntMap.Strict qualified as IntMap
import Data.Map.Strict qualified as Map
import Data.Morpheus.Types ( GQLType, Resolver, QUERY )
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..))
import Gargantext.API.Prelude (GargM, GargError, HasJobEnv')
import Gargantext.Database.Prelude (HasConnectionPool, HasConfig)
import Gargantext.Prelude
import GHC.Generics (Generic)
import Servant.Job.Async (HasJobEnv(job_env), jenv_jobs, job_async)
import Servant.Job.Core (env_item, env_map, env_state_mvar)
......
{-|
Module : Gargantext.API.GraphQL.Context
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
......@@ -10,9 +21,8 @@ import Data.Morpheus.Types
, Resolver
, ResolverM
, QUERY
, lift
)
import Data.Text (Text, pack)
import Data.Text (pack)
import Data.Time.Format.ISO8601 (iso8601Show)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Prelude (GargM, GargError)
......@@ -21,11 +31,10 @@ import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument)
import Gargantext.Database.Admin.Types.Node (ContextTitle, NodeId(..), NodeTypeId, UserId, unNodeId)
import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Database.Query.Table.NodeContext (getNodeContext, getContextsForNgramsTerms, ContextForNgramsTerms(..), {- getContextNgrams, -} getContextNgramsMatchingFTS)
import qualified Gargantext.Database.Query.Table.NodeContext as DNC
import Gargantext.Database.Query.Table.NodeContext qualified as DNC
import Gargantext.Database.Schema.NodeContext (NodeContext, NodeContextPoly(..))
import Gargantext.Prelude
import Gargantext.Prelude.Crypto.Hash (Hash)
import GHC.Generics (Generic)
data ContextGQL = ContextGQL
{ c_id :: Int
......
......@@ -9,7 +9,6 @@ module Gargantext.API.GraphQL.IMT
where
import Data.Morpheus.Types (GQLType)
import GHC.Generics (Generic)
import Gargantext.API.GraphQL.Types
import Gargantext.Core.Ext.IMT (School(..), schools)
import Gargantext.Prelude
......
{-|
Module : Gargantext.API.GraphQL.Node
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.Node where
import Data.Aeson
import Data.Either (Either(..))
import Data.Morpheus.Types ( GQLType, lift )
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.HashMap.Strict qualified as HashMap
import Data.Morpheus.Types ( GQLType )
import Data.Text qualified as T
import Gargantext.API.Admin.Auth.Types
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.GraphQL.PolicyCheck (withPolicy)
import Gargantext.API.GraphQL.Types
import Gargantext.Database.Admin.Types.Node (NodeId(..), NodeType)
import Gargantext.Database.Admin.Types.Node qualified as NN
import Gargantext.Database.Prelude (CmdCommon) -- , JSONB)
import Gargantext.Database.Query.Table.Node (getClosestParentIdByType, getNode)
import Gargantext.Database.Schema.Node qualified as N
import Gargantext.Prelude
import Text.Read (readEither)
import qualified Data.HashMap.Strict as HashMap
import qualified Data.Text as T
import qualified Gargantext.Database.Admin.Types.Node as NN
import qualified Gargantext.Database.Schema.Node as N
import qualified PUBMED.Types as PUBMED
import qualified Prelude
import PUBMED.Types qualified as PUBMED
import Prelude qualified
data Corpus = Corpus
{ id :: Int
......
{-|
Module : Gargantext.API.GraphQL.Team
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.Team where
import Data.Morpheus.Types (GQLType, ResolverM, lift)
import Data.Text ( Text )
import GHC.Generics (Generic)
import Data.Morpheus.Types (GQLType, ResolverM)
import Data.Text qualified as T
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.GraphQL.Types (GqlM)
import Gargantext.API.GraphQL.Utils (authUser, AuthStatus (Invalid, Valid))
import Gargantext.API.Prelude (GargM, GargError)
import Gargantext.Core.Types (NodeId(..), unNodeId)
import Gargantext.Core.Types.Individu qualified as Individu
import Gargantext.Database.Action.Share (membersOf, deleteMemberShip)
import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Database.Query.Table.Node (getNode)
......@@ -18,8 +29,6 @@ import Gargantext.Database.Query.Table.User (getUsersWithNodeHyperdata)
import Gargantext.Database.Schema.Node (NodePoly(Node, _node_id), _node_user_id)
import Gargantext.Database.Schema.User (UserLight(..))
import Gargantext.Prelude
import qualified Data.Text as T
import qualified Gargantext.Core.Types.Individu as Individu
data TeamArgs = TeamArgs
{ team_node_id :: Int } deriving (Generic, GQLType)
......
{-|
Module : Gargantext.API.GraphQL.TreeFirstLevel
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.TreeFirstLevel where
import Data.Morpheus.Types (GQLType, lift)
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.Morpheus.Types (GQLType)
import Gargantext.API.Admin.Auth.Types
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.GraphQL.PolicyCheck
......@@ -14,13 +23,13 @@ import Gargantext.Core.Types (Tree, NodeTree, NodeType)
import Gargantext.Core.Types.Main ( Tree(TreeN), _tn_node, _tn_children, NodeTree(NodeTree, _nt_id, _nt_type), _nt_name )
import Gargantext.Database.Admin.Config (fromNodeTypeId)
import Gargantext.Database.Admin.Types.Node (allNodeTypes, NodeId (NodeId))
import Gargantext.Database.Admin.Types.Node qualified as NN
import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Database.Query.Table.Node (getNode)
import Gargantext.Database.Query.Tree qualified as T
import Gargantext.Database.Schema.Node (NodePoly(_node_parent_id))
import Gargantext.Database.Schema.Node qualified as N
import Gargantext.Prelude
import qualified Gargantext.Database.Admin.Types.Node as NN
import qualified Gargantext.Database.Query.Tree as T
import qualified Gargantext.Database.Schema.Node as N
data TreeArgs = TreeArgs
{
......
{-|
Module : Gargantext.API.GraphQL.User
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.User where
import Data.Maybe (listToMaybe)
import Data.Morpheus.Types ( GQLType , lift )
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.Morpheus.Types ( GQLType )
import Gargantext.API.Admin.Auth.Types
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Auth.PolicyCheck
......
{-|
Module : Gargantext.API.GraphQL.UserInfo
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DuplicateRecordFields #-}
module Gargantext.API.GraphQL.UserInfo where
import Control.Lens
import Data.Maybe (fromMaybe)
import Data.Morpheus.Types
( GQLType
, description
, lift
)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Morpheus.Types ( GQLType, description )
import Data.Text qualified as T
import Gargantext.Database.Admin.Types.Hyperdata
( HyperdataUser(..)
, hc_source
......@@ -35,20 +40,19 @@ import Gargantext.Database.Admin.Types.Hyperdata.Contact
, ct_phone
, hc_who
, hc_where)
import Gargantext.API.Admin.Auth.Types hiding (Valid)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.GraphQL.PolicyCheck
import Gargantext.API.GraphQL.Types
import Gargantext.API.GraphQL.Utils (AuthStatus(Invalid, Valid), authUser)
import Gargantext.Core.Types.Individu qualified as Individu
import Gargantext.Database.Prelude (CmdCommon)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Query.Table.User (getUsersWithHyperdata, getUsersWithNodeHyperdata, updateUserEmail)
import Gargantext.Database.Schema.User (UserLight(..))
import Gargantext.Database.Schema.Node (node_id, node_hyperdata, NodePoly (Node, _node_id))
import Gargantext.Database.Schema.User (UserLight(..))
import Gargantext.Prelude
import GHC.Generics (Generic)
import Gargantext.API.GraphQL.Utils (AuthStatus(Invalid, Valid), authUser)
import Gargantext.API.Admin.Types (HasSettings)
import qualified Gargantext.Core.Types.Individu as Individu
import Gargantext.API.GraphQL.Types
import Gargantext.API.Admin.Auth.Types hiding (Valid)
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.GraphQL.PolicyCheck
data UserInfo = UserInfo
{ ui_id :: Int
......
......@@ -10,19 +10,16 @@ Portability : POSIX
module Gargantext.API.GraphQL.Utils where
import Control.Lens.Getter (view)
import Data.Morpheus.Types (GQLTypeOptions, fieldLabelModifier)
import qualified Data.Text as T
import Data.Text qualified as T
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser (AuthenticatedUser, _authUser_id))
import Gargantext.API.Admin.Types (jwtSettings, HasSettings (settings))
import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix)
import Gargantext.Database.Admin.Types.Node (NodeId)
import Gargantext.Database.Prelude (Cmd')
import Gargantext.Prelude
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import Gargantext.API.Admin.Types (jwtSettings, HasSettings (settings))
import Servant.Auth.Server (verifyJWT, JWTSettings)
import Control.Lens.Getter (view)
import Gargantext.Database.Prelude (Cmd')
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser (AuthenticatedUser, _authUser_id))
import Data.ByteString (ByteString)
import Gargantext.Database.Admin.Types.Node (NodeId)
unPrefix :: T.Text -> GQLTypeOptions -> GQLTypeOptions
unPrefix prefix options = options { fieldLabelModifier = nflm }
......
......@@ -13,11 +13,8 @@ module Gargantext.API.HashedResponse where
import Data.Aeson
import Data.Swagger
import Data.Text (Text)
import GHC.Generics (Generic)
import Gargantext.Prelude
import qualified Gargantext.Prelude.Crypto.Hash as Crypto (hash)
import Gargantext.Prelude.Crypto.Hash qualified as Crypto (hash)
data HashedResponse a = HashedResponse { hash :: Text, value :: a }
deriving (Generic)
......
module Gargantext.API.Job where
{-|
Module : Gargantext.API.Job
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
import Control.Lens (over, _Just)
import Data.Maybe
import qualified Data.Text as T
import Gargantext.Prelude
module Gargantext.API.Job where
import Control.Lens (over, _Just)
import Data.Text qualified as T
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.Prelude
newtype RemainingSteps = RemainingSteps { _RemainingSteps :: Int }
deriving (Show, Eq, Num)
jobLogStart :: RemainingSteps -> JobLog
jobLogStart rem =
jobLogStart rem' =
JobLog { _scst_succeeded = Just 0
, _scst_remaining = Just (_RemainingSteps rem)
, _scst_remaining = Just (_RemainingSteps rem')
, _scst_failed = Just 0
, _scst_events = Just [] }
......@@ -60,7 +68,7 @@ jobLogFailTotal (JobLog { _scst_succeeded = mSucc
where
(newRem, newFail) = case mRem of
Nothing -> (Nothing, mFail)
Just rem -> (Just 0, (+ rem) <$> mFail)
Just rem' -> (Just 0, (+ rem') <$> mFail)
jobLogFailTotalWithMessage :: T.Text -> JobLog -> JobLog
jobLogFailTotalWithMessage message jl = addErrorEvent message $ jobLogFailTotal jl
......
{-|
Module : Gargantext.API.Members
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
module Gargantext.API.Members where
import Gargantext.Prelude
import Gargantext.API.Prelude
import Servant
import Data.Text (Text)
import Gargantext.API.Admin.EnvTypes (Env)
import Gargantext.API.Prelude
import Gargantext.Core.Types (UserId)
import Gargantext.Database.Admin.Types.Node (NodeType(NodeTeam))
import Gargantext.Database.Query.Table.Node (getNodesIdWithType)
import Gargantext.Database.Action.Share (membersOf)
import Gargantext.Database.Admin.Types.Node (NodeType(NodeTeam))
import Gargantext.Database.Prelude (CmdCommon)
import Control.Monad.Extra (concatMapM)
import Gargantext.Database.Query.Table.Node (getNodesIdWithType)
import Gargantext.Prelude
import Servant
type MembersAPI = Get '[JSON] [Text]
......
......@@ -19,7 +19,7 @@ module Gargantext.API.Metrics
where
import Control.Lens
import Data.Text (Text)
import Data.HashMap.Strict qualified as HashMap
import Data.Time (UTCTime)
import Data.Vector (Vector)
import Gargantext.API.HashedResponse
......@@ -32,6 +32,7 @@ import Gargantext.Core.Types (CorpusId, ListId, ListType(..))
import Gargantext.Core.Types.Query (Limit)
import Gargantext.Core.Viz.Chart
import Gargantext.Core.Viz.Types
import Gargantext.Database.Action.Metrics qualified as Metrics
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataList(..), hl_chart, hl_pie, hl_scatter, hl_tree)
import Gargantext.Database.Admin.Types.Metrics (ChartMetrics(..), Metric(..), Metrics(..))
import Gargantext.Database.Admin.Types.Node (NodeId)
......@@ -40,10 +41,8 @@ import Gargantext.Database.Query.Table.Node (defaultList, getNodeWith)
import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Schema.Node (node_hyperdata)
import Gargantext.Prelude
import Gargantext.Prelude hiding (hash)
import Servant
import qualified Data.HashMap.Strict as HashMap
import qualified Gargantext.Database.Action.Metrics as Metrics
-------------------------------------------------------------
-- | Scatter metrics API
......
......@@ -16,6 +16,7 @@ add get
-}
{-# OPTIONS_GHC -fno-warn-unused-top-binds #-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
......@@ -92,16 +93,12 @@ import Control.Monad.Reader
import Data.Aeson.Text qualified as DAT
import Data.Foldable
import Data.List qualified as List
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Map.Strict.Patch qualified as PM
import Data.Maybe (fromMaybe)
import Data.Monoid
import Data.Ord (Down(..))
import Data.Patch.Class (Action(act), Transformable(..), ours)
import Data.Set (Set)
import Data.Set qualified as Set
import Data.Text (Text, isInfixOf, toLower, unpack)
import Data.Text (isInfixOf, toLower, unpack)
import Data.Text.Lazy.IO as DTL
import Formatting (hprint, int, (%))
import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
......@@ -123,12 +120,11 @@ import Gargantext.Database.Query.Table.Node (getNode)
import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
import Gargantext.Database.Query.Table.Node.Select
import Gargantext.Database.Schema.Node (node_id, node_parent_id, node_user_id)
import Gargantext.Prelude hiding (log)
import Gargantext.Prelude hiding (log, to, toLower, (%))
import Gargantext.Prelude.Clock (hasTime, getTime)
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Prelude (error)
import Servant hiding (Patch)
import System.IO (stderr)
{-
-- TODO sequences of modifications (Patchs)
......
......@@ -18,14 +18,12 @@ module Gargantext.API.Ngrams.List
import Data.ByteString.Lazy qualified as BSL
import Data.Csv qualified as Csv
import Data.Either (Either(..))
import Data.HashMap.Strict (HashMap)
import Data.HashMap.Strict qualified as HashMap
import Data.Map.Strict (Map, toList)
import Data.Map.Strict (toList)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe)
import Data.Set qualified as Set
import Data.Text (Text, concat, pack, splitOn)
import Data.Text (concat, pack, splitOn)
import Data.Vector (Vector)
import Data.Vector qualified as Vec
import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
......@@ -46,7 +44,7 @@ import Gargantext.Database.Query.Table.Node (getNode)
import Gargantext.Database.Schema.Ngrams
import Gargantext.Database.Schema.Node (_node_parent_id)
import Gargantext.Database.Types (Indexed(..))
import Gargantext.Prelude
import Gargantext.Prelude hiding (concat, toList)
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Gargantext.Utils.Servant qualified as GUS
import Prelude qualified
......
......@@ -16,20 +16,16 @@ module Gargantext.API.Ngrams.NgramsTree
import Data.Aeson.TH (deriveJSON)
import Data.HashMap.Strict (HashMap)
import Data.Maybe (catMaybes)
import Data.Set (Set)
import Data.HashMap.Strict qualified as HashMap
import Data.List qualified as List
import Data.Set qualified as Set
import Data.Swagger
import Data.Text (Text)
import Data.Tree
import GHC.Generics (Generic)
import Gargantext.API.Ngrams.Types
import Gargantext.Core.Types (ListType(..), NodeId)
import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixSwagger)
import Gargantext.Prelude
import Test.QuickCheck
import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List
import qualified Data.Set as Set
type Children = Text
type Root = Text
......
......@@ -14,24 +14,22 @@ Portability : POSIX
module Gargantext.API.Ngrams.Prelude
where
import Data.Maybe (catMaybes)
import Control.Lens (view)
import Data.HashMap.Strict qualified as HM
import Data.List qualified as List
import Data.Map.Strict (fromList)
import Data.Hashable (Hashable)
import Data.Map.Strict qualified as Map
import Data.Text qualified as Text
import Data.Validity
import Gargantext.API.Ngrams (getNgramsTableMap)
import Gargantext.API.Ngrams.Types
import Gargantext.Core.NodeStory (HasNodeStory)
import Gargantext.Core.Text.Context (TermList)
import Gargantext.Core.Text.List.Social.Prelude
import Gargantext.Core.Types (ListType)
import Gargantext.Database.Admin.Types.Node (ListId)
import Gargantext.Database.Schema.Ngrams (NgramsType, ngramsTypes)
import Gargantext.Prelude
import Gargantext.Core.Text.List.Social.Prelude
import Gargantext.API.Ngrams (getNgramsTableMap)
import Gargantext.Core.Text.Context (TermList)
import Gargantext.Core.NodeStory (HasNodeStory)
import Gargantext.Database.Admin.Types.Node (ListId)
import qualified Data.HashMap.Strict as HM
import qualified Data.Map.Strict as Map
import qualified Data.List as List
import qualified Data.Text as Text
------------------------------------------------------------------------
......
......@@ -18,20 +18,18 @@ import Control.Concurrent
import Control.Lens (_Just, (^.), at, view, At, Index, IxValue)
import Control.Monad.Reader
import Data.HashMap.Strict (HashMap)
import Data.Hashable (Hashable)
import Data.HashMap.Strict qualified as HM
import Data.Map.Strict qualified as Map
import Data.Pool (withResource)
import Data.Set (Set)
import Data.Set qualified as Set
import Data.Validity
import Gargantext.API.Ngrams.Types
import Gargantext.Core.NodeStory
import Gargantext.Core.NodeStoryFile qualified as NSF
import Gargantext.Core.Types (ListType(..), NodeId, NodeType(..), ListId)
import Gargantext.Database.Prelude (HasConnectionPool(..))
import Gargantext.Database.Schema.Ngrams (NgramsType)
import Gargantext.Prelude
import qualified Data.HashMap.Strict as HM
import qualified Data.Map.Strict as Map
import qualified Data.Set as Set
import qualified Gargantext.Core.NodeStoryFile as NSF
mergeNgramsElement :: NgramsRepoElement -> NgramsRepoElement -> NgramsRepoElement
......@@ -52,11 +50,11 @@ getRepo listIds = do
repoSize :: Ord k1 => NodeStory (Map.Map k1 (Map.Map k2 a)) p
-> NodeId
-> Map.Map k1 Int
repoSize repo node_id = Map.map Map.size state
repoSize repo node_id = Map.map Map.size state'
where
state = repo ^. unNodeStory
. at node_id . _Just
. a_state
state' = repo ^. unNodeStory
. at node_id . _Just
. a_state
getNodeStoryVar :: HasNodeStory env err m
......
......@@ -19,45 +19,36 @@ module Gargantext.API.Ngrams.Types where
import Codec.Serialise (Serialise())
import Control.Category ((>>>))
import Control.DeepSeq (NFData)
import Control.Lens (makeLenses, makePrisms, Iso', iso, from, (.~), (.=), (?=), (#), to, folded, {-withIndex, ifolded,-} view, use, (^.), (^?), (%~), (.~), (%=), at, _Just, Each(..), itraverse_, both, forOf_, (?~), over)
import Control.Monad.State
import Data.Aeson hiding ((.=))
import Data.Aeson.TH (deriveJSON)
import Data.Either (Either(..))
import Data.Foldable
import Data.Hashable (Hashable)
import Data.Map.Strict (Map)
import Data.Maybe (fromMaybe)
import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap
import Data.Map.Strict qualified as Map
import Data.Map.Strict.Patch qualified as PM
import Data.Monoid
import Data.Patch.Class (Replace, replace, Action(act), Group, Applicable(..), Composable(..), Transformable(..), PairPatch(..), Patched, ConflictResolution, ConflictResolutionReplace, MaybePatch(Mod), unMod, old, new)
import Data.Set (Set)
import Data.String (IsString, fromString)
import Data.Set qualified as Set
import Data.String (IsString(..))
import Data.Swagger hiding (version, patch)
import Data.Text (Text, pack, strip)
import Data.Text (pack, strip)
import Data.Validity
import Database.PostgreSQL.Simple.FromField (FromField, fromField, fromJSONField)
import Database.PostgreSQL.Simple.ToField (ToField, toJSONField, toField)
import GHC.Generics (Generic)
import Gargantext.Core.Text (size)
import Gargantext.Core.Types (ListType(..), ListId, NodeId, TODO)
import Gargantext.Core.Types.Query (Limit, Offset, MaxSize, MinSize)
import Gargantext.Core.Utils.Prefix (unPrefix, unPrefixUntagged, unPrefixSwagger, wellNamedSchema)
import Gargantext.Database.Admin.Types.Node (ContextId)
import Gargantext.Database.Prelude (fromField', HasConnectionPool, HasConfig, CmdM')
import Gargantext.Prelude
import Gargantext.Database.Query.Table.Ngrams qualified as TableNgrams
import Gargantext.Prelude hiding (IsString, hash, from, replace, to)
import Gargantext.Prelude.Crypto.Hash (IsHashable(..))
import Protolude (maybeToEither)
import Servant hiding (Patch)
import Servant.Job.Utils (jsonOptions)
-- import System.FileLock (FileLock)
import Test.QuickCheck (elements, frequency)
import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
import qualified Data.HashMap.Strict.InsOrd as InsOrdHashMap
import qualified Data.Map.Strict as Map
import qualified Data.Map.Strict.Patch as PM
import qualified Data.Set as Set
import qualified Gargantext.Database.Query.Table.Ngrams as TableNgrams
------------------------------------------------------------------------
......@@ -195,8 +186,8 @@ mkNgramsElement :: NgramsTerm
-> Maybe RootParent
-> MSet NgramsTerm
-> NgramsElement
mkNgramsElement ngrams list rp children =
NgramsElement ngrams (size (unNgramsTerm ngrams)) list mempty (_rp_root <$> rp) (_rp_parent <$> rp) children
mkNgramsElement ngrams list' rp children =
NgramsElement ngrams (size (unNgramsTerm ngrams)) list' mempty (_rp_root <$> rp) (_rp_parent <$> rp) children
newNgramsElement :: Maybe ListType -> NgramsTerm -> NgramsElement
newNgramsElement mayList ngrams =
......
......@@ -30,10 +30,7 @@ module Gargantext.API.Node
import Data.Aeson (FromJSON, ToJSON)
import Data.Aeson.TH (deriveJSON)
import Data.Maybe
import Data.Swagger
import Data.Text (Text())
import GHC.Generics (Generic)
import Gargantext.API.Admin.Auth (withAccess, withPolicy)
import Gargantext.API.Admin.Auth.Types (PathId(..), AuthenticatedUser (..))
import Gargantext.API.Admin.EnvTypes
......
......@@ -17,20 +17,17 @@ module Gargantext.API.Node.Corpus.Annuaire
import Control.Lens hiding (elements)
import Data.Aeson
import Data.Swagger
import Data.Text (Text)
import GHC.Generics (Generic)
import Servant
import Servant.Job.Utils (jsonOptions)
import Web.FormUrlEncoded (FromForm)
import qualified Gargantext.API.Node.Corpus.New.Types as NewTypes
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.API.Node.Corpus.New.Types qualified as NewTypes
import Gargantext.Core (Lang(..))
import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
import Gargantext.Database.Action.Flow.Types (FlowCmdM) -- flowAnnuaire
import Gargantext.Database.Admin.Types.Node (AnnuaireId)
import Gargantext.Prelude
import Gargantext.Utils.Jobs (MonadJobStatus(..))
import Servant
import Servant.Job.Utils (jsonOptions)
import Web.FormUrlEncoded (FromForm)
type Api = Summary "New Annuaire endpoint"
......
......@@ -18,12 +18,9 @@ module Gargantext.API.Node.Corpus.Export
import Data.HashMap.Strict qualified as HashMap
import Data.List qualified as List
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe)
import Data.Set (Set)
import Data.Set qualified as Set
import Data.Text (Text, pack)
import Data.Text (pack)
import Gargantext.API.Ngrams.Tools (filterListWithRoot, mapTermListRoot, getRepo)
import Gargantext.API.Ngrams.Types
import Gargantext.API.Node.Corpus.Export.Types
......@@ -41,7 +38,7 @@ import Gargantext.Database.Query.Table.Node.Select (selectNodesWithUsername)
import Gargantext.Database.Query.Table.NodeContext (selectDocNodes)
import Gargantext.Database.Schema.Context (_context_id, _context_hyperdata)
import Gargantext.Database.Schema.Ngrams (NgramsType(..))
import Gargantext.Prelude
import Gargantext.Prelude hiding (hash)
import Gargantext.Prelude.Crypto.Hash (hash)
import Servant (Headers, Header, addHeader)
......
......@@ -28,12 +28,9 @@ import Data.Aeson
import Data.Aeson.TH (deriveJSON)
import Data.ByteString.Base64 qualified as BSB64
import Data.Conduit.Internal (zipSources)
import Data.Either
import Data.Swagger
import Data.Text (Text)
import Data.Text qualified as T
import Data.Text.Encoding qualified as TE
import GHC.Generics (Generic)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Ngrams (commitStatePatch, Versioned(..))
......@@ -65,7 +62,6 @@ import Gargantext.Prelude
import Gargantext.Prelude.Config (gc_max_docs_parsers)
import Gargantext.System.Logging
import Gargantext.Utils.Jobs (JobHandle, MonadJobStatus(..))
import Protolude (mempty)
import Servant
import Servant.Job.Utils (jsonOptions)
import Test.QuickCheck.Arbitrary
......
......@@ -19,22 +19,17 @@ module Gargantext.API.Node.Corpus.New.File
where
import Control.Lens ((.~), (?~))
import Control.Monad (forM)
import Data.Maybe
import Data.Monoid (mempty)
import Data.Swagger
import Data.Text (Text())
import Servant
import Servant.Multipart
import Servant.Swagger.Internal
import Gargantext.API.Node.Corpus.New.Types
import Gargantext.Core.Types (TODO)
import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.Prelude (Cmd)
import Gargantext.Prelude
import Gargantext.Prelude hiding (hash)
import Gargantext.Prelude.Crypto.Hash (hash)
import Servant
import Servant.Multipart
import Servant.Swagger.Internal
-------------------------------------------------------------
type Hash = Text
......
......@@ -3,13 +3,11 @@ module Gargantext.API.Node.Corpus.New.Types where
import Data.Aeson
import Data.Swagger
import Data.Text (pack)
import GHC.Generics (Generic)
import Gargantext.Prelude
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
import Gargantext.Prelude
data FileType = CSV
| CSV_HAL
| Istex
......
{-|
Module : Gargantext.API.Node.Corpus.Searx
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE TemplateHaskell #-}
module Gargantext.API.Node.Corpus.Searx where
......@@ -5,14 +16,12 @@ module Gargantext.API.Node.Corpus.Searx where
import Control.Lens (view)
import Data.Aeson qualified as Aeson
import Data.Aeson.TH (deriveJSON)
import Data.Either (Either(..))
import Data.HashMap.Strict qualified as HashMap
import Data.Text qualified as T
import Data.Text qualified as Text
import Data.Time.Calendar (Day, toGregorian)
import Data.Time.Format (defaultTimeLocale, formatTime, parseTimeM)
import Data.Tuple.Select (sel1, sel2, sel3)
import GHC.Generics (Generic)
import Gargantext.Core (Lang(..))
import Gargantext.Core.NLP (HasNLPServer, nlpServerGet)
import Gargantext.Core.NodeStory (HasNodeStory)
......@@ -36,19 +45,18 @@ import Gargantext.Database.Query.Table.Node (insertDefaultNodeIfNotExists)
import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
import Gargantext.Database.Query.Tree.Error (HasTreeError)
import Gargantext.Database.Query.Tree.Root (getOrMk_RootWithCorpus)
import Gargantext.Prelude
import Gargantext.Prelude hiding (All)
import Gargantext.Prelude.Config
import Gargantext.Utils.Jobs (JobHandle, MonadJobStatus(..))
import Network.HTTP.Client
import Network.HTTP.Client.TLS
import Prelude qualified
import Protolude (catMaybes, encodeUtf8, rightToMaybe, Text, void)
langToSearx :: Lang -> Text
langToSearx All = "en-US"
langToSearx x = (Text.toLower acronym) <> "-" <> acronym
where
acronym = (cs $ show x)
acronym = show x
data SearxResult = SearxResult
{ _sr_url :: Text
......
{-|
Module : Gargantext.API.Node.Corpus.Types
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE TemplateHaskell #-}
module Gargantext.API.Node.Corpus.Types where
......@@ -6,17 +17,13 @@ import Control.Lens hiding (elements, Empty)
import Control.Monad.Fail (fail)
import Data.Aeson
import Data.Aeson.TH (deriveJSON)
import Data.Monoid (mempty)
import Data.Swagger
import GHC.Generics (Generic)
import Test.QuickCheck
import qualified Data.Text as T
import Gargantext.Prelude
import qualified Gargantext.API.Admin.Orchestrator.Types as Types
import Data.Text qualified as T
import Gargantext.API.Admin.Orchestrator.Types qualified as Types
import Gargantext.Core.Utils.Prefix (unPrefix)
import Gargantext.Database.Action.Flow (DataOrigin(..))
import Gargantext.Prelude
import Test.QuickCheck
data Database = Empty
| OpenAlex
......@@ -62,7 +69,7 @@ instance FromJSON Datafield where
instance ToJSON Datafield where
toJSON (External db) = toJSON $ object [ ("External", toJSON db) ]
toJSON s = toJSON $ show s
toJSON s = toJSON (show s :: Text)
instance Arbitrary Datafield where
arbitrary = oneof [pure Gargantext, pure Web, pure Files, External <$> arbitrary]
......
{-|
Module : Gargantext.API.Node.DocumentUpload
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE MonoLocalBinds #-}
{-# LANGUAGE TemplateHaskell #-}
......@@ -8,10 +19,7 @@ module Gargantext.API.Node.DocumentUpload where
import Control.Lens (makeLenses, view)
import Data.Aeson
import Data.Swagger (ToSchema)
import GHC.Generics (Generic)
import Servant
import qualified Data.Text as T
import Data.Text qualified as T
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Prelude
......@@ -28,6 +36,7 @@ import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.Query.Table.Node (getClosestParentIdByType')
import Gargantext.Prelude
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Servant
data DocumentUpload = DocumentUpload
......
......@@ -16,17 +16,12 @@ Portability : POSIX
module Gargantext.API.Node.DocumentsFromWriteNodes
where
-- import Data.Maybe (fromMaybe)
import Conduit
import Control.Lens ((^.))
import Data.Aeson
import Data.Either (Either(..), rights)
import Data.List qualified as List
import Data.Maybe (fromMaybe)
import Data.Swagger
import Data.Text (Text)
import Data.Text qualified as T
import GHC.Generics (Generic)
import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Admin.Types (HasSettings)
......@@ -48,10 +43,7 @@ import Gargantext.Database.Query.Table.Node (getChildrenByType, getClosestParent
import Gargantext.Database.Schema.Node (node_hyperdata, node_name, node_date)
import Gargantext.Prelude
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Protolude (mempty)
import Servant
import Text.Read (readMaybe)
-- import qualified Gargantext.Defaults as Defaults
------------------------------------------------------------------------
type API = Summary " Documents from Write nodes."
......@@ -154,7 +146,7 @@ hyperdataDocumentFromFrameWrite lang paragraphSize (node, contents) =
day' = maybe Defaults.day (\(Date { day }) -> fromIntegral day) date
--}
in
Right (List.map (\(t, ctxt) -> HyperdataDocument { _hd_bdd = Just $ cs $ show Notes
Right (List.map (\(t, ctxt) -> HyperdataDocument { _hd_bdd = Just $ show Notes
, _hd_doi = Nothing
, _hd_url = Nothing
, _hd_uniqId = Nothing
......
{-|
Module : Gargantext.API.Node.File
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-unused-matches #-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE IncoherentInstances #-}
module Gargantext.API.Node.File where
import Control.Lens ((^.))
import Data.ByteString qualified as BS
import Data.ByteString.Lazy qualified as BSL
import Data.MIME.Types qualified as DMT
import Data.Swagger
import Data.Text
import GHC.Generics (Generic)
import Servant
import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as BSL
import qualified Data.MIME.Types as DMT
import qualified Gargantext.Database.GargDB as GargDB
import qualified Network.HTTP.Media as M
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Data.Text qualified as T
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.API.Node.Types
import Gargantext.API.Prelude
......@@ -27,12 +33,14 @@ import Gargantext.Database.Action.Flow.Types
import Gargantext.Database.Action.Node (mkNodeWithParent)
import Gargantext.Database.Admin.Types.Hyperdata.File
import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.GargDB qualified as GargDB
import Gargantext.Database.Query.Table.Node (getNodeWith)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Schema.Node (node_hyperdata)
import Gargantext.Prelude
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Data.Either
import Network.HTTP.Media qualified as M
import Servant
data RESPONSE deriving Typeable
......@@ -77,14 +85,14 @@ fileDownload uId nId = do
let (HyperdataFile { _hff_name = name'
, _hff_path = path }) = node ^. node_hyperdata
Contents c <- GargDB.readGargFile $ unpack path
Contents c <- GargDB.readGargFile $ T.unpack path
let (mMime, _) = DMT.guessType DMT.defaultmtd False $ unpack name'
let (mMime, _) = DMT.guessType DMT.defaultmtd False $ T.unpack name'
mime = case mMime of
Just m -> m
Nothing -> "text/plain"
pure $ addHeader (pack mime) $ BSResponse c
pure $ addHeader (T.pack mime) $ BSResponse c
--pure c
......@@ -127,7 +135,7 @@ addWithFile uId nId nwf@(NewWithFile _d _l fName) jobHandle = do
node <- getNodeWith nId' (Proxy :: Proxy HyperdataFile)
let hl = node ^. node_hyperdata
_ <- updateHyperdata nId' $ hl { _hff_name = fName
, _hff_path = pack fPath }
, _hff_path = T.pack fPath }
-- printDebug "[addWithFile] Created node with id: " nId'
pure ()
......
{-|
Module : Gargantext.API.Node.FrameCalcUpload
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE MonoLocalBinds #-}
{-# LANGUAGE TypeOperators #-}
......@@ -6,24 +17,19 @@ module Gargantext.API.Node.FrameCalcUpload where
import Control.Lens ((^.))
import Data.Aeson
import qualified Data.ByteString.Lazy as BSL
import qualified Data.ByteString.UTF8 as BSU8
import Data.ByteString.Lazy qualified as BSL
import Data.ByteString.UTF8 qualified as BSU8
import Data.Swagger
import qualified Data.Text as T
import GHC.Generics (Generic)
import Network.HTTP.Client (newManager, httpLbs, parseRequest, responseBody)
import Network.HTTP.Client.TLS (tlsManagerSettings)
import Servant
import Web.FormUrlEncoded (FromForm)
import Data.Text qualified as T
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Node.Corpus.New (addToCorpusWithForm)
import Gargantext.API.Node.Corpus.New.Types (FileFormat(..), FileType(..))
import Gargantext.API.Node.Types (NewWithForm(..))
import Gargantext.API.Prelude
import Gargantext.Core.Types.Individu (User(..))
import Gargantext.Core (Lang)
import Gargantext.Core.Text.List.Social (FlowSocialListWith(..))
import Gargantext.Core.Types.Individu (User(..))
import Gargantext.Database.Action.Flow.Types
import Gargantext.Database.Admin.Types.Hyperdata.Frame
import Gargantext.Database.Admin.Types.Node
......@@ -32,7 +38,10 @@ import Gargantext.Database.Query.Table.Node (getClosestParentIdByType, getNodeWi
import Gargantext.Database.Schema.Node (node_hyperdata)
import Gargantext.Prelude
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Gargantext.Core (Lang)
import Network.HTTP.Client (newManager, httpLbs, parseRequest, responseBody)
import Network.HTTP.Client.TLS (tlsManagerSettings)
import Servant
import Web.FormUrlEncoded (FromForm)
data FrameCalcUpload = FrameCalcUpload {
_wf_lang :: !(Maybe Lang)
......
......@@ -11,23 +11,23 @@ Polymorphic Get Node API
-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TypeOperators #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
module Gargantext.API.Node.Get
where
import Data.Aeson
import Data.Swagger
import GHC.Generics (Generic)
import Servant
import Test.QuickCheck.Arbitrary
import Gargantext.API.Prelude
import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.Prelude (JSONB)
import Gargantext.Prelude
import Servant
import Test.QuickCheck.Arbitrary
------------------------------------------------------------------------
type API a = Summary "Polymorphic Get Node Endpoint"
......
......@@ -23,13 +23,6 @@ module Gargantext.API.Node.New
import Control.Lens hiding (elements, Empty)
import Data.Aeson
import Data.Swagger
import Data.Text (Text)
import GHC.Generics (Generic)
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import Web.FormUrlEncoded (FromForm, ToForm)
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Prelude
......@@ -42,6 +35,10 @@ import Gargantext.Database.Query.Table.Node.User
import Gargantext.Database.Schema.Node
import Gargantext.Prelude
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import Web.FormUrlEncoded (FromForm, ToForm)
------------------------------------------------------------------------
data PostNode = PostNode { pn_name :: Text
......
......@@ -16,9 +16,9 @@ module Gargantext.API.Node.Share
where
import Data.Aeson
import Data.List qualified as List
import Data.Swagger
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.Text qualified as Text
import Gargantext.API.Prelude
import Gargantext.Core.NLP (HasNLPServer)
import Gargantext.Core.Types.Individu (User(..), arbitraryUsername)
......@@ -31,12 +31,10 @@ import Gargantext.Database.Prelude (CmdRandom)
import Gargantext.Database.Query.Table.Node.Error (HasNodeError(..))
import Gargantext.Database.Query.Tree (findNodesWithType)
import Gargantext.Prelude
import Gargantext.Utils.Aeson qualified as GUA
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import qualified Data.List as List
import qualified Data.Text as Text
import qualified Gargantext.Utils.Aeson as GUA
------------------------------------------------------------------------
data ShareNodeParams = ShareTeamParams { username :: Text }
......
{-|
Module : Gargantext.API.Node.Types
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE TemplateHaskell #-}
module Gargantext.API.Node.Types where
import Control.Lens hiding (elements, Empty)
import Data.Aeson
import qualified Data.ByteString as BS
import qualified Data.ByteString.Base64 as BSB64
import Data.Either
import Data.ByteString qualified as BS
import Data.ByteString.Base64 qualified as BSB64
import Data.Swagger
import Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import GHC.Generics (Generic)
import Servant.Job.Utils (jsonOptions)
import Web.FormUrlEncoded (FromForm, ToForm)
import Data.Text qualified as T
import Gargantext.API.Node.Corpus.New.Types (FileType, FileFormat)
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.List.Social (FlowSocialListWith)
import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
import Gargantext.Database.GargDB qualified as GargDB
import Gargantext.Prelude
import qualified Gargantext.Database.GargDB as GargDB
import Gargantext.API.Node.Corpus.New.Types (FileType, FileFormat)
import Gargantext.Core.Text.List.Social (FlowSocialListWith)
import Servant.Job.Utils (jsonOptions)
import Web.FormUrlEncoded (FromForm, ToForm)
-------------------------------------------------------
data NewWithForm = NewWithForm
......@@ -64,7 +69,7 @@ instance ToSchema NewWithFile where
instance GargDB.SaveFile NewWithFile where
saveFile' fp (NewWithFile b64d _ _) = do
let eDecoded = BSB64.decode $ TE.encodeUtf8 b64d
let eDecoded = BSB64.decode $ encodeUtf8 b64d
case eDecoded of
Left err -> panic $ T.pack $ "Error decoding: " <> err
Right decoded -> BS.writeFile fp decoded
......
......@@ -18,10 +18,8 @@ module Gargantext.API.Node.Update
import Control.Lens (view)
import Data.Aeson
import Data.Maybe (Maybe(..), fromMaybe)
import Data.Set qualified as Set
import Data.Swagger
import GHC.Generics (Generic)
import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
import Gargantext.API.Admin.Types (HasSettings)
......@@ -45,10 +43,9 @@ import Gargantext.Database.Query.Table.Node (defaultList, getNode)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Schema.Ngrams (NgramsType(NgramsTerms))
import Gargantext.Database.Schema.Node (node_parent_id)
import Gargantext.Prelude (Bool(..), Ord, Eq, (<$>), ($), {-printDebug,-} pure, show, cs, (<>), panic, (<*>))
import Gargantext.Prelude
import Gargantext.Utils.Aeson qualified as GUA
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import Prelude (Enum, Bounded, minBound, maxBound)
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
......@@ -116,7 +113,7 @@ updateNode _uId nid1 (LinkNodeReq nt nid2) jobHandle = do
NodeAnnuaire -> pairing nid2 nid1 Nothing -- defaultList
NodeCorpus -> pairing nid1 nid2 Nothing -- defaultList
_ -> panic $ "[G.API.N.Update.updateNode] NodeType not implemented"
<> cs (show nt <> " nid1: " <> show nid1 <> " nid2: " <> show nid2)
<> show nt <> " nid1: " <> show nid1 <> " nid2: " <> show nid2
markComplete jobHandle
......
......@@ -20,16 +20,11 @@ module Gargantext.API.Prelude
)
where
import Control.Concurrent (threadDelay)
import Control.Exception (Exception)
import Control.Lens (Prism', (#))
import Control.Lens.TH (makePrisms)
import Control.Monad (mapM_)
import Control.Monad.Error.Class (MonadError(..))
import Control.Monad.Except (ExceptT)
import Control.Monad.Reader (ReaderT)
import Crypto.JOSE.Error as Jose
import Data.Aeson.Types
import Data.Text qualified as Text
import Data.Typeable
import Data.Validity
import Gargantext.API.Admin.Orchestrator.Types
......@@ -49,7 +44,6 @@ import Servant
import Servant.Job.Async
import Servant.Job.Core (HasServerError(..), serverError)
import Servant.Job.Types qualified as SJ
import qualified Data.Text as Text
class HasJoseError e where
_JoseError :: Prism' e Jose.Error
......
......@@ -15,34 +15,27 @@ Portability : POSIX
module Gargantext.API.Public
where
import Data.Set (Set)
import Control.Lens ((^?), (^.), _Just)
import Data.Maybe (catMaybes)
import Data.Text (Text)
import Data.List (replicate, null)
import Data.Aeson
import Data.List qualified as List
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Swagger hiding (title, url)
import GHC.Generics (Generic)
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import qualified Data.List as List
import qualified Data.Map.Strict as Map
import qualified Data.Set as Set
import Gargantext.API.Prelude
import Gargantext.API.Node.File
import Gargantext.Database.Query.Table.Node.Error (HasNodeError(..))
import Gargantext.Database.Prelude (Cmd, DBCmd)
import Gargantext.API.Prelude
import Gargantext.Core.Utils.DateUtils (utc2year)
import Gargantext.Database.Admin.Types.Hyperdata
import Gargantext.Database.Admin.Types.Hyperdata.CorpusField
import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.Prelude (Cmd, DBCmd)
import Gargantext.Database.Query.Table.Node.Error (HasNodeError(..))
import Gargantext.Database.Query.Table.NodeNode (selectPublicNodes)
import Gargantext.Core.Utils.DateUtils (utc2year)
import Gargantext.Database.Schema.Node -- (NodePoly(..))
import Gargantext.Prelude
import qualified Gargantext.Utils.Aeson as GUA
import Gargantext.Utils.Aeson qualified as GUA
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
------------------------------------------------------------------------
type API = API_Home
......@@ -113,7 +106,7 @@ toPublicData base (n , mn) = do
abstract <- (hd ^? (_Just . hf_data . cf_desc ))
img <- (Just $ url' mn) -- "images/Gargantextuel-212x300.jpg"
url <- (Just $ url' mn)
date <- Just (cs $ show $ utc2year (n^.node_date))
date <- Just (show $ utc2year (n^.node_date))
database <- (hd ^? (_Just . hf_data . cf_query))
author <- (hd ^? (_Just . hf_data . cf_authors))
pure $ PublicData { .. }
......@@ -124,7 +117,7 @@ toPublicData base (n , mn) = do
url' :: [NodeId] -> Text
url' mn' = base
<> "/public/"
<> (cs $ show $ (maybe 0 unNodeId $ head mn'))
<> (show $ (maybe 0 unNodeId $ head mn'))
<> "/file/download"
......
......@@ -19,25 +19,29 @@ Portability : POSIX
module Gargantext.API.Routes
where
import Control.Concurrent (threadDelay)
import Control.Lens (view)
import Data.Text (Text)
import Data.Validity
import Servant
import Servant.Auth as SA
import Servant.Auth.Swagger ()
import Servant.Swagger.UI
import Gargantext.API.Admin.Auth (ForgotPasswordAPI, ForgotPasswordAsyncAPI, withAccess, withPolicyT)
import Gargantext.API.Admin.Auth.Types (AuthRequest, AuthResponse, AuthenticatedUser(..), PathId(..))
import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
import Gargantext.API.Admin.FrontEnd (FrontEndAPI)
import Gargantext.API.Auth.PolicyCheck
import Gargantext.API.Context
import Gargantext.API.Count (CountAPI, count, Query)
import Gargantext.API.GraphQL qualified as GraphQL
import Gargantext.API.Members (MembersAPI, members)
import Gargantext.API.Ngrams (TableNgramsApi, apiNgramsTableDoc)
import Gargantext.API.Ngrams.List qualified as List
import Gargantext.API.Node
import Gargantext.API.Node.Contact qualified as Contact
import Gargantext.API.Node.Corpus.Annuaire qualified as Annuaire
import Gargantext.API.Node.Corpus.Export qualified as CorpusExport
import Gargantext.API.Node.Corpus.Export.Types qualified as CorpusExport
import Gargantext.API.Node.Corpus.New qualified as New
import Gargantext.API.Node.Document.Export qualified as DocumentExport
import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
import Gargantext.API.Prelude
import Gargantext.API.Public qualified as Public
import Gargantext.Core.Types.Individu (User(..))
import Gargantext.Core.Viz.Graph.API
import Gargantext.Database.Admin.Types.Hyperdata
......@@ -46,17 +50,10 @@ import Gargantext.Database.Prelude (HasConfig(..))
import Gargantext.Prelude
import Gargantext.Prelude.Config (gc_max_docs_scrapers)
import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
import qualified Gargantext.API.GraphQL as GraphQL
import qualified Gargantext.API.Ngrams.List as List
import qualified Gargantext.API.Node.Contact as Contact
import qualified Gargantext.API.Node.Corpus.Annuaire as Annuaire
import qualified Gargantext.API.Node.Corpus.Export as CorpusExport
import qualified Gargantext.API.Node.Corpus.Export.Types as CorpusExport
import qualified Gargantext.API.Node.Corpus.New as New
import qualified Gargantext.API.Node.Document.Export as DocumentExport
import qualified Gargantext.API.Node.Document.Export.Types as DocumentExport
import qualified Gargantext.API.Public as Public
import Gargantext.API.Auth.PolicyCheck
import Servant
import Servant.Auth as SA
import Servant.Auth.Swagger ()
import Servant.Swagger.UI
type GargAPI = MkGargAPI (GargAPIVersion GargAPI')
......@@ -293,7 +290,7 @@ waitAPI n = do
let
m = (10 :: Int) ^ (6 :: Int)
_ <- liftBase $ threadDelay ( m * n)
pure $ "Waited: " <> (cs $ show n)
pure $ "Waited: " <> show n
----------------------------------------
addCorpusWithQuery :: User -> ServerT New.AddWithQuery (GargM Env GargError)
......
......@@ -10,6 +10,8 @@ Portability : POSIX
Count API part of Gargantext.
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE DeriveAnyClass #-}
......@@ -17,11 +19,9 @@ Count API part of Gargantext.
module Gargantext.API.Search
where
-- import Data.List (concat)
import Data.Aeson hiding (defaultTaggedObject)
import Data.Swagger hiding (fieldLabelModifier, Contact)
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.Text qualified as T
import Gargantext.API.Prelude (GargServer)
import Gargantext.Core.Text.Corpus.Query (RawQuery (..), parseQuery)
import Gargantext.Core.Types.Query (Limit, Offset)
......@@ -37,8 +37,6 @@ import Gargantext.Utils.Aeson (defaultTaggedObject)
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary
import qualified Data.Text as T
import Data.Either
-----------------------------------------------------------------------
-- TODO-ACCESS: CanSearch? or is it part of CanGetNode
......
......@@ -11,36 +11,30 @@ Portability : POSIX
{-# LANGUAGE MonoLocalBinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
---------------------------------------------------------------------
module Gargantext.API.Server where
---------------------------------------------------------------------
import Control.Lens ((^.))
import Control.Monad.Except (withExceptT)
import Control.Monad.Reader (runReaderT)
import qualified Data.Aeson as Aeson
import Data.Text (Text)
import Data.Aeson qualified as Aeson
import Data.ByteString.Lazy.Char8 qualified as BL8
import Data.Version (showVersion)
import Servant
import Servant.Swagger.UI (swaggerSchemaUIServer)
import qualified Data.ByteString.Lazy.Char8 as BL8
import qualified Paths_gargantext as PG -- cabal magic build module
import qualified Gargantext.API.Public as Public
import Gargantext.API.Admin.Auth.Types (AuthContext)
import Gargantext.API.Admin.Auth (auth, forgotPassword, forgotPasswordAsync)
import Gargantext.API.Admin.Auth.Types (AuthContext)
import Gargantext.API.Admin.EnvTypes (Env)
import Gargantext.API.Admin.FrontEnd (frontEndServer)
import qualified Gargantext.API.GraphQL as GraphQL
import Gargantext.API.Auth.PolicyCheck ()
import Gargantext.API.GraphQL qualified as GraphQL
import Gargantext.API.Prelude
import Gargantext.API.Public qualified as Public
import Gargantext.API.Routes
import Gargantext.API.Swagger (swaggerDoc)
import Gargantext.API.ThrowAll (serverPrivateGargAPI)
import Gargantext.Database.Query.Table.Node.Error (NodeError(..))
import Gargantext.Database.Prelude (hasConfig)
import Gargantext.Prelude
import Gargantext.Database.Query.Table.Node.Error (NodeError(..))
import Gargantext.Prelude hiding (Handler)
import Gargantext.Prelude.Config (gc_url_backend_api)
import Gargantext.API.Auth.PolicyCheck ()
import Paths_gargantext qualified as PG -- cabal magic build module
import Servant
import Servant.Swagger.UI (swaggerSchemaUIServer)
serverGargAPI :: Text -> ServerT GargAPI (GargM Env GargError)
......
......@@ -34,9 +34,7 @@ module Gargantext.API.Table
import Data.Aeson.TH (deriveJSON)
import Data.Maybe
import Data.Swagger
import Data.Text (Text())
import Data.Text qualified as T
import GHC.Generics (Generic)
import Gargantext.API.HashedResponse
import Gargantext.API.Ngrams.Types (TabType(..))
import Gargantext.API.Prelude (GargServer)
......@@ -52,7 +50,6 @@ import Gargantext.Database.Query.Facet (FacetDoc , runViewDocuments, runCountDoc
import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
import Gargantext.Prelude
import Gargantext.System.Logging
import Prelude
import Servant
import Test.QuickCheck (elements)
import Test.QuickCheck.Arbitrary (Arbitrary, arbitrary)
......@@ -145,7 +142,7 @@ postTableApi cId tq = case tq of
$(logLocM) DEBUG $ "New search with query " <> getRawQuery q
searchInCorpus' cId False q (Just o) (Just l) (Just order)
Trash -> searchInCorpus' cId True q (Just o) (Just l) (Just order)
x -> panic $ "not implemented in tableApi " <> (cs $ show x)
x -> panic $ "not implemented in tableApi " <> (show x)
getTableHashApi :: (CmdM env err m, HasNodeError err, MonadLogger m)
=> NodeId
......@@ -207,7 +204,7 @@ getTable' cId ft o l order query year =
(Just Trash) -> runViewDocuments cId True o l order query year
(Just MoreFav) -> moreLike cId o l order IsFav
(Just MoreTrash) -> moreLike cId o l order IsTrash
x -> panic $ "not implemented in getTable: " <> (cs $ show x)
x -> panic $ "not implemented in getTable: " <> (show x)
getPair :: ContactId -> Maybe TabType
......@@ -217,4 +214,4 @@ getPair cId ft o l order =
case ft of
(Just Docs) -> runViewAuthorsDoc cId False o l order
(Just Trash) -> runViewAuthorsDoc cId True o l order
_ -> panic $ "not implemented: get Pairing" <> (cs $ show ft)
_ -> panic $ "not implemented: get Pairing" <> (show ft)
......@@ -16,15 +16,13 @@ Portability : POSIX
module Gargantext.API.ThrowAll where
import Control.Monad.Except (MonadError(..))
import Control.Lens ((#))
import Servant
import Servant.Auth.Server (AuthResult(..))
import Gargantext.Prelude
import Gargantext.API.Admin.EnvTypes (Env)
import Gargantext.API.Prelude
import Gargantext.API.Routes (GargPrivateAPI, serverPrivateGargAPI')
import Gargantext.Prelude
import Servant
import Servant.Auth.Server (AuthResult(..))
class ThrowAll' e a | a -> e where
-- | 'throwAll' is a convenience function to throw errors across an entire
......
......@@ -15,19 +15,14 @@ module Gargantext.Core
where
import Data.Aeson
import Data.Either(Either(Left))
import Data.Hashable (Hashable)
import Data.LanguageCodes qualified as ISO639
import Data.Maybe (fromMaybe)
import Data.Map qualified as Map
import Data.Morpheus.Types (GQLType)
import Data.Swagger
import Data.Text (Text, pack)
import Data.Tuple.Extra (swap)
import GHC.Generics (Generic)
import Gargantext.Prelude
import Data.Text (pack)
import Gargantext.Prelude hiding (All)
import Servant.API
import Test.QuickCheck
import qualified Data.Map as Map
------------------------------------------------------------------------
-- | Language of a Text
......
......@@ -13,15 +13,12 @@ Portability : POSIX
module Gargantext.Core.Ext.IMT where
import Data.Either (Either(..))
import Data.List qualified as DL
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as M
import Data.Morpheus.Types (GQLType)
import Data.Set qualified as S
import Data.Text (Text, splitOn)
import Data.Text (splitOn)
import Data.Vector qualified as DV
import GHC.Generics (Generic)
import Gargantext.Core.Text.Corpus.Parsers.CSV as CSV
import Gargantext.Core.Text.Metrics.Utils as Utils
import Gargantext.Prelude
......
......@@ -17,19 +17,14 @@ module Gargantext.Core.Ext.IMTUser -- (deserialiseImtUsersFromFile)
where
import Codec.Serialise
import Data.ByteString.Lazy qualified as BL
import Data.Csv
import Data.Either
import Data.Maybe (catMaybes)
import Data.Text (Text)
import Data.Vector (Vector)
import GHC.Generics (Generic)
import Data.Vector qualified as Vector
import Gargantext.Core.Text.Corpus.Parsers.CSV
import Gargantext.Database.Admin.Types.Hyperdata.Contact
import Gargantext.Prelude
import System.FilePath.Posix (takeExtension)
import System.IO (FilePath)
import qualified Data.ByteString.Lazy as BL
import qualified Data.Vector as Vector
------------------------------------------------------------------------
readFile_Annuaire :: FilePath -> IO [HyperdataContact]
......@@ -171,8 +166,8 @@ imtUser2gargContact (IMTUser { id
, _cw_keywords = catMaybes [service]
, _cw_freetags = []
, _cw_description = Nothing }
ou = ContactWhere { _cw_organization = toList entite
, _cw_labTeamDepts = toList service
ou = ContactWhere { _cw_organization = toList' entite
, _cw_labTeamDepts = toList' service
, _cw_role = fonction
, _cw_office = bureau
, _cw_country = Just "France"
......@@ -184,8 +179,8 @@ imtUser2gargContact (IMTUser { id
, _ct_phone = tel
, _ct_url = url }
-- meta = ContactMetaData (Just "IMT annuaire") date_modification'
toList Nothing = []
toList (Just x) = [x]
toList' Nothing = []
toList' (Just x) = [x]
......
......@@ -12,10 +12,9 @@ Portability : POSIX
module Gargantext.Core.Mail where
import Control.Lens (view)
import Control.Monad.Reader (MonadReader)
import Control.Monad.Trans.Control (MonadBaseControl)
import Data.List qualified as List
import Data.Text (Text, unlines, splitOn)
import Data.Text (splitOn)
import Gargantext.Core.Types.Individu
import Gargantext.Database.Prelude (HasConfig(..))
import Gargantext.Database.Schema.User (UserLight(..))
......
......@@ -13,18 +13,15 @@ Références:
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
module Gargantext.Core.Methods.Graph.BAC.Proxemy
where
--import Debug.SimpleReflect
import Gargantext.Prelude
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as Map
import qualified Data.List as List
--import Gargantext.Core.Viz.Graph.IGraph
import Data.List qualified as List
import Data.Map.Strict qualified as Map
import Gargantext.Core.Viz.Graph.FGL
-- import qualified Graph.BAC.ProxemyOptim as BAC
import Gargantext.Prelude
type Length = Int
type FalseReflexive = Bool
......@@ -42,10 +39,10 @@ similarity_conf_x_y :: Graph_Undirected -> (Node,Node) -> Length -> FalseReflexi
similarity_conf_x_y g (x,y) l r rm_e = similarity
where
similarity :: Double
similarity | denominator == 0 = 0
| otherwise = prox_x_y / denominator
similarity | denominator' == 0 = 0
| otherwise = prox_x_y / denominator'
where
denominator = prox_x_y + lim_SC
denominator' = prox_x_y + lim_SC
prox_x_y :: Double
prox_x_y = maybe 0 identity $ Map.lookup y xline
......@@ -62,14 +59,14 @@ similarity_conf_x_y g (x,y) l r rm_e = similarity
lim_SC :: Double
lim_SC
| denominator == 0 = 0
| denominator' == 0 = 0
| otherwise = if pair_is_edge
then (degree g y + 1-1) / denominator
else (degree g y + 1 ) / denominator
then (degree g y + 1-1) / denominator'
else (degree g y + 1 ) / denominator'
where
denominator = if pair_is_edge
then (2 * (ecount g) + (vcount g) - 2)
else (2 * (ecount g) + (vcount g) )
denominator' = if pair_is_edge
then (2 * (ecount g) + (vcount g) - 2)
else (2 * (ecount g) + (vcount g) )
rm_edge_neighbors :: (Node, Node) -> Graph_Undirected -> Node -> [Node]
......
......@@ -52,20 +52,17 @@ def fast_maximal_cliques(g):
module Gargantext.Core.Methods.Graph.MaxClique
where
import Data.Maybe (catMaybes)
import Gargantext.Prelude
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as Map
import Data.List (sortOn, nub, concat)
import Data.Set (Set)
import Data.Set (fromList, toList, isSubsetOf)
import Data.Graph.Inductive hiding (Graph, neighbors, subgraph, (&))
import Gargantext.Core.Viz.Graph.FGL (Graph_Undirected, degree, neighbors, mkGraphUfromEdges)
import Gargantext.Core.Viz.Graph.Tools (cooc2graph',cooc2graph'', Threshold)
import Data.List (nub)
import Data.Map.Strict qualified as Map
import Data.Set (fromList, isSubsetOf)
import Gargantext.Core.Methods.Similarities (Similarity)
import Gargantext.Core.Viz.Graph.FGL (Graph_Undirected, degree, neighbors, mkGraphUfromEdges)
import Gargantext.Core.Viz.Graph.Index (createIndices, toIndex)
import Gargantext.Core.Viz.Graph.Tools (cooc2graph',cooc2graph'', Threshold)
import Gargantext.Core.Viz.Phylo
-- import Debug.Trace (trace)
import Gargantext.Prelude
type Graph = Graph_Undirected
type Neighbor = Node
......@@ -75,9 +72,9 @@ type Neighbor = Node
getMaxCliques :: Ord a => MaxCliqueFilter -> Similarity -> Threshold -> Map (a, a) Int -> [[a]]
getMaxCliques f d t m = map fromIndices $ getMaxCliques' t m'
where
m' = toIndex to m
(to,from) = createIndices m
fromIndices = catMaybes . map (\n -> Map.lookup n from)
m' = toIndex to' m
(to', from') = createIndices m
fromIndices = catMaybes . map (\n -> Map.lookup n from')
getMaxCliques' :: Threshold -> Map (Int, Int) Int -> [[Int]]
getMaxCliques' t' n = maxCliques graph
......
......@@ -22,6 +22,8 @@ Implementation use Accelerate library which enables GPU and CPU computation:
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE ScopedTypeVariables #-}
......
......@@ -12,18 +12,15 @@ Motivation and definition of the @Conditional@ distance.
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE Strict #-}
module Gargantext.Core.Methods.Similarities.Conditional
where
import Control.DeepSeq (NFData)
import Control.Parallel.Strategies (parList, rdeepseq, using)
import Data.Hashable (Hashable)
import Data.List (unzip)
import Data.Maybe (catMaybes)
import Gargantext.Prelude
import Data.HashMap.Strict qualified as Map
import Data.Set qualified as Set
import Gargantext.Core.Viz.Graph.Utils (getMax)
import qualified Data.HashMap.Strict as Map
import qualified Data.Set as Set
import Gargantext.Prelude
type HashMap = Map.HashMap
......
......@@ -92,44 +92,37 @@ module Gargantext.Core.NodeStory
, fixNodeStoryVersions )
where
-- import Debug.Trace (traceShow)
import Codec.Serialise.Class
import Control.Concurrent (MVar(), newMVar, modifyMVar_)
import Control.Debounce (mkDebounce, defaultDebounceSettings, debounceFreq, debounceAction)
import Control.Exception (catch, throw, SomeException(..))
import Control.Lens (makeLenses, Getter, (^.), (.~), (%~), _Just, at, traverse, view)
import Control.Exception (throw)
import Control.Lens (makeLenses, Getter, (^.), (.~), (%~), _Just, at, view)
import Control.Monad.Except
import Control.Monad.Reader
import Data.Aeson hiding ((.=), decode)
import Data.ByteString.Char8 (hPutStrLn)
import Data.HashMap.Strict (HashMap)
import Data.Map.Strict (Map)
import Data.Maybe (catMaybes)
import Data.HashMap.Strict qualified as HashMap
import Data.Map.Strict qualified as Map
import Data.Map.Strict.Patch qualified as PM
import Data.Monoid
import Data.Pool (Pool, withResource)
import Data.Profunctor.Product.TH (makeAdaptorAndInstance)
import Data.Semigroup
import Data.Set qualified as Set
import Data.Text qualified as Text
import Database.PostgreSQL.Simple qualified as PGS
import Database.PostgreSQL.Simple.FromField (FromField(fromField), fromJSONField)
import Database.PostgreSQL.Simple.SqlQQ (sql)
import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
import GHC.Generics (Generic)
import Gargantext.API.Ngrams.Types
import Gargantext.Core.Types (ListId, NodeId(..), NodeType)
import Gargantext.Core.Utils.Prefix (unPrefix)
import Gargantext.Database.Admin.Config (nodeTypeId)
import Gargantext.Database.Prelude (DbCmd', HasConnectionPool(..))
import Gargantext.Database.Query.Table.Ngrams qualified as TableNgrams
import Gargantext.Database.Query.Table.Node.Error (HasNodeError())
import Gargantext.Database.Schema.Ngrams (NgramsType)
import Gargantext.Prelude
import Opaleye (DefaultFromField(..), SqlJsonb, fromPGSFromField)
import System.IO (stderr)
import qualified Data.HashMap.Strict as HashMap
import qualified Data.Map.Strict as Map
import qualified Data.Map.Strict.Patch as PM
import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified Database.PostgreSQL.Simple as PGS
import qualified Gargantext.Database.Query.Table.Ngrams as TableNgrams
------------------------------------------------------------------------
data NodeStoryEnv = NodeStoryEnv
......@@ -673,8 +666,8 @@ nodeStoryVar :: Pool PGS.Connection
-> [NodeId]
-> IO (MVar NodeListStory)
nodeStoryVar pool Nothing nIds = do
state <- withResource pool $ \c -> nodeStoryIncs c Nothing nIds
newMVar state
state' <- withResource pool $ \c -> nodeStoryIncs c Nothing nIds
newMVar state'
nodeStoryVar pool (Just mv) nIds = do
_ <- withResource pool
$ \c -> modifyMVar_ mv
......@@ -702,8 +695,8 @@ mkNodeStorySaver saver = mkDebounce settings
--withMVar mvns (\ns -> printDebug "[mkNodeStorySaver] debounce nodestory" ns)
, debounceFreq = 1*minute
}
minute = 60*second
second = 10^(6 :: Int)
minute = 60*sec
sec = 10^(6 :: Int)
clearHistory :: NodeListStory -> NodeListStory
clearHistory (NodeStory ns) = NodeStory $ ns & (traverse . a_history) .~ emptyHistory
......
......@@ -15,24 +15,22 @@ NOTE This is legacy code. It keeps node stories in a directory
module Gargantext.Core.NodeStoryFile where
import Control.Lens (view)
import Control.Monad (foldM)
import Codec.Serialise (serialise, deserialise)
import Codec.Serialise.Class
import Control.Concurrent (MVar(), modifyMVar_, newMVar, readMVar, withMVar)
import Control.Debounce (mkDebounce, defaultDebounceSettings, debounceFreq, debounceAction)
import Control.Lens (view)
import Data.ByteString.Lazy qualified as DBL
import Data.List qualified as List
import Data.Map.Strict qualified as Map
import Gargantext.Core.NodeStory hiding (readNodeStoryEnv)
import Gargantext.Core.Types (ListId, NodeId(..))
import Gargantext.Database.Prelude (hasConfig)
import Gargantext.Database.Query.Table.Ngrams qualified as TableNgrams
import Gargantext.Prelude
import Gargantext.Prelude.Config (gc_repofilepath)
import System.Directory (renameFile, createDirectoryIfMissing, doesFileExist, removeFile)
import System.IO (FilePath, hClose)
import System.IO (hClose)
import System.IO.Temp (withTempFile)
import qualified Data.ByteString.Lazy as DBL
import qualified Data.List as List
import qualified Data.Map.Strict as Map
import qualified Gargantext.Database.Query.Table.Ngrams as TableNgrams
getRepo :: HasNodeStory env err m
......@@ -85,8 +83,8 @@ mkNodeStorySaver nsd mvns = mkDebounce settings
, debounceFreq = 1 * minute
-- , debounceEdge = trailingEdge -- Trigger on the trailing edge
}
minute = 60 * second
second = 10^(6 :: Int)
minute = 60 * sec
sec = 10^(6 :: Int)
nodeStoryVar :: NodeStoryDir
-> Maybe (MVar NodeListStory)
......@@ -180,7 +178,7 @@ splitByNode (NodeStory m) =
saverAction' :: Serialise a => NodeStoryDir -> NodeId -> a -> IO ()
saverAction' repoDir nId a = do
withTempFile repoDir ((cs $ show nId) <> "-tmp-repo.cbor") $ \fp h -> do
withTempFile repoDir ((show nId) <> "-tmp-repo.cbor") $ \fp h -> do
-- printDebug "[repoSaverAction]" fp
DBL.hPut h $ serialise a
hClose h
......@@ -189,7 +187,7 @@ saverAction' repoDir nId a = do
nodeStoryPath :: NodeStoryDir -> NodeId -> FilePath
nodeStoryPath repoDir nId = repoDir <> "/" <> filename
where
filename = "repo" <> "-" <> (cs $ show nId) <> ".cbor"
filename = "repo" <> "-" <> (show nId) <> ".cbor"
------------------------------------------------------------------------
......
......@@ -14,13 +14,12 @@ module Gargantext.Core.Statistics
where
import Data.Map.Strict (Map)
import Data.Array.IArray (Array, listArray, elems)
import Data.List qualified as List
import Data.Map.Strict qualified as Map
import Data.Vector.Storable qualified as Vec
import Gargantext.Prelude
import Numeric.Statistics.PCA (pcaReduceN)
import Data.Array.IArray (Array, listArray, elems)
import qualified Data.Vector.Storable as Vec
import qualified Data.List as List
import qualified Data.Map.Strict as Map
data Dimension = Dimension Int
......
......@@ -15,10 +15,11 @@ Text gathers terms in unit of contexts.
module Gargantext.Core.Text
where
import Data.Text (Text, split)
import Data.Text (split)
import Data.Text qualified as DT
import Gargantext.Prelude hiding (filter)
import NLP.FullStop (segment)
import qualified Data.Text as DT
import Prelude qualified
-----------------------------------------------------------------
class HasText h
......@@ -39,22 +40,22 @@ type Titre = Phrase
-----------------------------------------------------------------
instance Show Texte where
instance Prelude.Show Texte where
show (Texte t) = show t
instance Show Paragraphe where
instance Prelude.Show Paragraphe where
show (Paragraphe p) = show p
instance Show Phrase where
instance Prelude.Show Phrase where
show (Phrase p) = show p
instance Show MultiTerme where
instance Prelude.Show MultiTerme where
show (MultiTerme mt) = show mt
instance Show Mot where
instance Prelude.Show Mot where
show (Mot t) = show t
instance Show Lettre where
instance Prelude.Show Lettre where
show (Lettre l) = show l
-----------------------------------------------------------------
......
......@@ -23,12 +23,10 @@ How to split contexts is describes in this module.
module Gargantext.Core.Text.Context
where
import Data.Text (Text, pack, unpack)
import Data.String (IsString)
import Text.HTML.TagSoup (parseTags, isTagText, Tag(..))
import Data.Text (pack, unpack)
import Gargantext.Core.Text
import Gargantext.Prelude hiding (length)
import Text.HTML.TagSoup (parseTags, isTagText, Tag(..))
------------------------------------------------------------------------
type Term = Text
......
......@@ -16,11 +16,9 @@ Format Converter.
module Gargantext.Core.Text.Convert (risPress2csvWrite)
where
import Data.Either (Either(..))
import Gargantext.Core.Text.Corpus.Parsers (parseFile, FileFormat(..), FileType(..))
import Gargantext.Core.Text.Corpus.Parsers.CSV (writeDocs2Csv)
import Gargantext.Prelude
import System.FilePath (FilePath()) -- , takeExtension)
risPress2csvWrite :: FilePath -> IO ()
......
......@@ -20,23 +20,20 @@ module Gargantext.Core.Text.Corpus.API
import Conduit
import Control.Monad.Except
import Data.Bifunctor
import Data.Either (Either(..))
import Data.Maybe
import Data.Text qualified as T
import Gargantext.API.Admin.Orchestrator.Types (ExternalAPIs(..), externalAPIs)
import Gargantext.Core (Lang(..), toISO639)
import Gargantext.Core.Text.Corpus.API.Arxiv qualified as Arxiv
import Gargantext.Core.Text.Corpus.API.Hal qualified as HAL
import Gargantext.Core.Text.Corpus.API.Isidore qualified as ISIDORE
import Gargantext.Core.Text.Corpus.API.Istex qualified as ISTEX
import Gargantext.Core.Text.Corpus.API.OpenAlex qualified as OpenAlex
import Gargantext.Core.Text.Corpus.API.Pubmed qualified as PUBMED
import Gargantext.Core.Text.Corpus.Query qualified as Corpus
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude
import Gargantext.Prelude hiding (get)
import PUBMED.Types qualified as PUBMED
import Servant.Client (ClientError)
import qualified Data.Text as T
import qualified Gargantext.Core.Text.Corpus.API.Arxiv as Arxiv
import qualified Gargantext.Core.Text.Corpus.API.Hal as HAL
import qualified Gargantext.Core.Text.Corpus.API.Isidore as ISIDORE
import qualified Gargantext.Core.Text.Corpus.API.Istex as ISTEX
import qualified Gargantext.Core.Text.Corpus.API.OpenAlex as OpenAlex
import qualified Gargantext.Core.Text.Corpus.API.Pubmed as PUBMED
import qualified Gargantext.Core.Text.Corpus.Query as Corpus
import qualified PUBMED.Types as PUBMED
data GetCorpusError
= -- | We couldn't parse the user input query into something meaningful.
......
......@@ -18,19 +18,16 @@ module Gargantext.Core.Text.Corpus.API.Arxiv
, convertQuery
) where
import Arxiv qualified as Arxiv
import Conduit
import Data.Maybe
import Data.Text (Text, unpack)
import qualified Data.Text as Text
import Gargantext.Prelude
import Data.Text (unpack)
import Data.Text qualified as Text
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Core.Text.Corpus.Query as Corpus
import Gargantext.Core.Types (Term(..))
import qualified Arxiv as Arxiv
import qualified Network.Api.Arxiv as Ax
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude hiding (get)
import Network.Api.Arxiv qualified as Ax
-- | Converts a Gargantext's generic boolean query into an Arxiv Query.
......
......@@ -17,12 +17,11 @@ import Data.Either
import Data.LanguageCodes qualified as ISO639
import Data.Map.Strict qualified as Map
import Data.Maybe
import Data.Text (Text, pack, intercalate)
-- import Gargantext.Core (Lang(..), toISO639Lang)
import Data.Text (pack, intercalate)
import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Defaults qualified as Defaults
import Gargantext.Prelude
import Gargantext.Prelude hiding (intercalate)
import HAL qualified as HAL
import HAL.Client qualified as HAL
import HAL.Doc.Corpus qualified as HAL
......@@ -58,14 +57,14 @@ toDoc' la (HAL.Corpus { .. }) = do
, _hd_page = Nothing
, _hd_title = Just $ intercalate " " _corpus_title
, _hd_authors = Just $ foldl (\x y -> x <> ", " <> y) "" _corpus_authors_names
, _hd_institutes = Just $ foldl (\x y -> x <> ", " <> y) "" $ _corpus_authors_affiliations <> map (cs . show) _corpus_struct_id
, _hd_institutes = Just $ foldl (\x y -> x <> ", " <> y) "" $ _corpus_authors_affiliations <> map show _corpus_struct_id
, _hd_source = Just $ maybe "Nothing" identity _corpus_source
, _hd_abstract = Just abstract
, _hd_publication_date = fmap (pack . show) utctime
, _hd_publication_date = fmap show utctime
, _hd_publication_year = pub_year
, _hd_publication_month = pub_month
, _hd_publication_day = pub_day
, _hd_publication_hour = Nothing
, _hd_publication_minute = Nothing
, _hd_publication_second = Nothing
, _hd_language_iso2 = Just $ (pack . show) la }
, _hd_language_iso2 = Just $ show la }
......@@ -13,19 +13,17 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.API.Isidore where
import System.FilePath (FilePath())
import Data.Text (Text)
import qualified Data.Text as Text
import Data.Text qualified as Text
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.Corpus.Parsers (cleanText)
import Gargantext.Core.Text.Corpus.Parsers.CSV (writeDocs2Csv)
import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import qualified Gargantext.Defaults as Defaults
import Gargantext.Prelude
import Gargantext.Defaults qualified as Defaults
import Gargantext.Prelude hiding (get)
import Isidore qualified as Isidore
import Isidore.Client
import Servant.Client
import qualified Gargantext.Core.Text.Corpus.Parsers.Date as Date
import qualified Isidore as Isidore
import Gargantext.Core.Text.Corpus.Parsers.CSV (writeDocs2Csv)
import Gargantext.Core.Text.Corpus.Parsers (cleanText)
-- | TODO work with the ServantErr
get :: Lang -> Maybe Isidore.Limit
......@@ -34,7 +32,7 @@ get :: Lang -> Maybe Isidore.Limit
get la l q a = do
let
printErr (DecodeFailure e _) = panic e
printErr e = panic (cs $ show e)
printErr e = panic (show e)
toIsidoreDocs :: Reply -> [IsidoreDoc]
toIsidoreDocs (ReplyOnly r) = [r]
......
......@@ -13,10 +13,7 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.API.Istex
where
import Data.Either (Either(..))
import Data.List qualified as List
import Data.Maybe
import Data.Text (Text)
import Data.Text qualified as Text
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.Corpus.Parsers.JSON.Istex (toDoc)
......
......@@ -21,26 +21,18 @@ module Gargantext.Core.Text.Corpus.API.Pubmed
where
import Conduit
import Control.Monad.Reader (runReaderT)
import Data.Either (Either)
import Data.Maybe
import Data.Semigroup
import Data.Monoid
import Data.Text (Text)
import qualified Data.Text as Text
import qualified Data.Text.Encoding as TE
import Network.HTTP.Types.URI (EscapeItem(..), renderQueryPartialEscape)
import Servant.Client (ClientError)
import Gargantext.Prelude
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TE
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.Corpus.Query as Corpus
import Gargantext.Core.Types (Term(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import qualified PUBMED as PubMed
import qualified PUBMED.Parser as PubMedDoc
import Gargantext.Prelude hiding (get)
import Network.HTTP.Types.URI (EscapeItem(..), renderQueryPartialEscape)
import PUBMED qualified as PubMed
import PUBMED.Parser qualified as PubMedDoc
import PUBMED.Types (Config(..))
import Servant.Client (ClientError)
-- | A pubmed query.
......
......@@ -14,18 +14,15 @@ Get Book into GarganText
module Gargantext.Core.Text.Corpus.Parsers.Book
where
import Data.Maybe
import Data.Text (Text)
import GHC.IO (FilePath)
import Data.ByteString.Lazy qualified as DBL
import Data.List qualified as List
import Data.Text qualified as DT
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.Corpus.Parsers.CSV (hyperdataDocument2csv)
import Gargantext.Core.Text.Corpus.Parsers.FrameWrite (text2titleParagraphs)
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude
import System.Directory -- (getDirectoryContents)
import qualified Data.ByteString.Lazy as DBL
import qualified Data.List as List
import qualified Data.Text as DT
------------------------------------------------------------------------
-- Main Export Function
......
......@@ -14,37 +14,36 @@ import Gargantext.Core.Text.Corpus.Parsers.Date as DGP
DGP.parseDateRaw DGP.FR "12 avril 2010" == "2010-04-12T00:00:00.000+00:00"
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE TypeFamilies #-}
module Gargantext.Core.Text.Corpus.Parsers.Date
{-(parse, parseRaw, dateSplit, Year, Month, Day)-}
where
--import qualified Control.Exception as CE
import Data.Aeson (toJSON, Value)
import Data.Either (Either(..))
import Data.Aeson qualified as Json
import Data.HashMap.Strict as HM hiding (map)
import Data.Maybe (fromMaybe)
import Data.Text (Text, unpack, splitOn, replace)
import Data.HashSet qualified as HashSet
import Data.List qualified as List
import Data.Text (unpack, splitOn, replace)
import Data.Time (defaultTimeLocale, iso8601DateFormat, parseTimeM, toGregorian)
import Data.Time.Calendar qualified as DTC
import Data.Time.Clock ( secondsToDiffTime)
import Data.Time.Clock (UTCTime(..), getCurrentTime)
import Data.Time.LocalTime (utc)
import Data.Time.LocalTime.TimeZone.Series (zonedTimeToZoneSeriesTime)
import Duckling.Api (analyze)
import Duckling.Core (makeLocale, Dimension(Time))
import Duckling.Core qualified as DC
import Duckling.Resolve (fromUTC, Context(Context, referenceTime, locale), DucklingTime(DucklingTime), Options(..))
import Duckling.Types (ResolvedToken(..), ResolvedVal(..))
import Duckling.Types (Seal(..))
import Gargantext.Core (Lang(FR,EN))
import Gargantext.Core.Types (DebugMode(..), withDebugMode)
import Gargantext.Prelude
import Gargantext.Prelude hiding (replace)
import System.Environment (getEnv)
import qualified Data.Aeson as Json
import qualified Data.HashSet as HashSet
import qualified Data.Time.Calendar as DTC
import qualified Duckling.Core as DC
import qualified Data.List as List
------------------------------------------------------------------------
-- | Parse date to Ints
-- TODO add hours, minutes and seconds
......@@ -131,7 +130,7 @@ readDate txt = do
parserLang :: Lang -> DC.Lang
parserLang FR = DC.FR
parserLang EN = DC.EN
parserLang lang = panic $ "[G.C.T.C.P.Date] Lang not implemented" <> (cs $ show lang)
parserLang lang = panic $ "[G.C.T.C.P.Date] Lang not implemented" <> (show lang)
-- | Final Date parser API
-- IO can be avoided here:
......@@ -163,7 +162,7 @@ parseRaw lang text = do -- case result
Just result -> Right result
Nothing -> do
-- printDebug ("[G.C.T.C.P.D.parseRaw] ERROR " <> (cs . show) lang) text
Left $ "[G.C.T.C.P.D.parseRaw ERROR] " <> (cs . show) lang <> " :: " <> text
Left $ "[G.C.T.C.P.D.parseRaw ERROR] " <> show lang <> " :: " <> text
getTimeValue :: [ResolvedToken] -> Maybe Value
getTimeValue rt = case head rt of
......
......@@ -14,10 +14,7 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.Parsers.Date.Attoparsec
where
import Control.Applicative ((<*))
import Data.Attoparsec.ByteString (Parser, take)
import Data.ByteString (ByteString)
import Data.Tuple.Extra (first)
import Gargantext.Prelude hiding (takeWhile, take)
-------------------------------------------------------------
......
......@@ -13,20 +13,18 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.Parsers.Date.Parsec
where
import Data.Either (Either)
import Data.Fixed (Fixed (MkFixed))
import Data.String (String)
import Data.Text (Text, unpack)
import Data.Text qualified as T
import Data.Time (ZonedTime(..), LocalTime(..), TimeZone(..), TimeOfDay(..))
import Data.Time.Calendar (Day, fromGregorian)
import Gargantext.Prelude
import Prelude (toInteger, (++))
import Text.Parsec.Error (ParseError)
import Text.Parsec.Prim (Stream, ParsecT)
import Text.Parsec.String (Parser)
import Text.ParserCombinators.Parsec (many1, noneOf, anyChar, char, oneOf)
import Text.ParserCombinators.Parsec qualified (parse)
import Text.XML.HXT.DOM.Util (decimalStringToInt)
import qualified Text.ParserCombinators.Parsec (parse)
-- | Permit to transform a String to an Int in a monadic context
wrapDST :: Monad m => String -> m Int
......@@ -88,4 +86,4 @@ parseZonedTime= do
---- | Opposite of toRFC3339
fromRFC3339 :: Text -> Either ParseError ZonedTime
fromRFC3339 t = Text.ParserCombinators.Parsec.parse parseZonedTime "ERROR: Couldn't parse zoned time." input
where input = unpack t
where input = T.unpack t
......@@ -12,18 +12,16 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.Parsers.FrameWrite
where
import Control.Applicative ((*>))
import Control.Monad (void)
import Data.Either
import Data.List qualified as List
import Data.Maybe
import Data.Text hiding (foldl)
import Data.Text qualified as DT
import Gargantext.Core.Text (sentences)
import Gargantext.Prelude
import Prelude ((++), read)
import Gargantext.Prelude hiding (ByteString, (<|>), many, try)
import Prelude (read)
import Text.Parsec hiding (Line)
import Text.Parsec.String
import qualified Data.Text as DT
import qualified Data.List as List
-- https://gitlab.iscpif.fr/gargantext/purescript-gargantext/issues/331
......
{-|
Module : Gargantext.Core.Text.Corpus.Parsers.Gitlab
Description :
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
module Gargantext.Core.Text.Corpus.Parsers.Gitlab (
Issue(..), gitlabIssue2hyperdataDocument, readFile_Issues, readFile_IssuesAsDocs
) where
import Data.Aeson
import Data.ByteString.Lazy qualified as DBL
import Data.Text qualified as DT
import Data.Time
import qualified Data.Text as DT
import qualified Data.ByteString.Lazy as DBL
import System.FilePath (FilePath)
import Gargantext.Prelude
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude
data Issue = Issue { _issue_id :: !Int
, _issue_title :: !DT.Text
......
......@@ -27,15 +27,13 @@ module Gargantext.Core.Text.Corpus.Parsers.GrandDebat
where
import Data.Aeson (ToJSON, FromJSON)
import Data.Text (Text)
import GHC.Generics (Generic)
import Data.ByteString.Lazy qualified as DBL
import Data.JsonStream.Parser qualified as P
import Data.Text qualified as Text
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..), ToHyperdataDocument, toHyperdataDocument)
import Gargantext.Prelude
import Gargantext.Database.GargDB
import qualified Data.ByteString.Lazy as DBL
import qualified Data.JsonStream.Parser as P
import qualified Data.Text as Text
import Gargantext.Prelude
data GrandDebatReference = GrandDebatReference
{ id :: !(Maybe Text)
......
......@@ -14,6 +14,7 @@ TODO:
- use more ontologies to help building corpora
-}
{-# OPTIONS_GHC -fno-warn-deprecations #-}
{-# LANGUAGE ScopedTypeVariables #-}
......@@ -22,24 +23,24 @@ module Gargantext.Core.Text.Corpus.Parsers.Isidore where
import Control.Lens hiding (contains)
import Data.ByteString.Lazy (ByteString)
import Data.RDF hiding (triple, Query)
import Data.Text hiding (groupBy, map)
import Data.Text qualified as T
import Database.HSparql.Connection
import Database.HSparql.QueryGenerator
import Gargantext.Core (Lang)
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude
import Gargantext.Prelude hiding (ByteString)
import Network.Wreq (getWith, Response, defaults, header, param, responseStatus, responseBody)
import Prelude (String)
import Prelude qualified
route :: EndPoint
route = "https://isidore.science/sparql/"
selectQueryRaw' :: String -> String -> IO (Response ByteString)
selectQueryRaw' :: Prelude.String -> Prelude.String -> IO (Response ByteString)
selectQueryRaw' uri q = getWith opts uri
where
opts = defaults & header "Accept" .~ ["application/sparql-results+xml"]
& header "User-Agent" .~ ["gargantext-hsparql-client"]
& param "query" .~ [Data.Text.pack q]
& param "query" .~ [T.pack q]
isidoreGet :: Lang -> Int -> Text -> IO (Maybe [HyperdataDocument])
isidoreGet la li q = do
......@@ -53,7 +54,7 @@ isidoreGet' l q = do
let s = createSelectQuery $ isidoreSelect l q
putStrLn s
r <- selectQueryRaw' route s
putStrLn $ show $ r ^. responseStatus
putStrLn (show $ r ^. responseStatus :: Text)
pure $ structureContent $ r ^. responseBody
-- res <- selectQuery route $ simpleSelect q
-- pure res
......@@ -70,7 +71,7 @@ isidoreSelect lim q = do
--ore <- prefix "ore" (iriRef "http://www.openarchives.org/ore/terms/")
--bif <- prefix "bif" (iriRef "bif:")
link <- var
link' <- var
title <- var
date <- var
abstract <- var
......@@ -80,17 +81,17 @@ isidoreSelect lim q = do
publisher <- var
--agg <- var
triple_ link (rdf .:. "type") (isidore .:. "Document")
triple_ link (dcterms .:. "title") title
triple_ link (dcterms .:. "date") date
triple_ link (dcterms .:. "creator") authors
triple_ link' (rdf .:. "type") (isidore .:. "Document")
triple_ link' (dcterms .:. "title") title
triple_ link' (dcterms .:. "date") date
triple_ link' (dcterms .:. "creator") authors
--triple_ link (dcterms .:. "language") langDoc
triple_ link (dc .:. "description") abstract
triple_ link' (dc .:. "description") abstract
--triple_ link (ore .:. "isAggregatedBy") agg
--triple_ agg (dcterms .:. "title") title
optional_ $ triple_ link (dcterms .:. "source") source
optional_ $ triple_ link (dcterms .:. "publisher") publisher
optional_ $ triple_ link' (dcterms .:. "source") source
optional_ $ triple_ link' (dcterms .:. "publisher") publisher
-- TODO FIX BUG with (.||.) operator
--filterExpr_ $ (.||.) (contains title q) (contains abstract q)
......@@ -106,7 +107,7 @@ isidoreSelect lim q = do
orderNextDesc date
limit_ lim
distinct_
selectVars [link, date, langDoc, authors, source, publisher, title, abstract]
selectVars [link', date, langDoc, authors, source, publisher, title, abstract]
-- | TODO : check if all cases are taken into account
unbound :: Lang -> BindingValue -> Maybe Text
......@@ -114,14 +115,14 @@ unbound _ Unbound = Nothing
unbound _ (Bound (UNode x)) = Just x
unbound _ (Bound (LNode (TypedL x _))) = Just x
unbound _ (Bound (LNode (PlainL x))) = Just x
unbound l (Bound (LNode (PlainLL x l'))) = if l' == (toLower $ cs $ show l) then Just x else Nothing
unbound l (Bound (LNode (PlainLL x l'))) = if l' == (T.toLower $ show l) then Just x else Nothing
unbound _ _ = Nothing
bind2doc :: Lang -> [BindingValue] -> HyperdataDocument
bind2doc l [ link, date, langDoc, authors, _source, publisher, title, abstract ] =
bind2doc l [ link', date, langDoc, authors, _source, publisher, title, abstract ] =
HyperdataDocument { _hd_bdd = Just "Isidore"
, _hd_doi = Nothing
, _hd_url = unbound l link
, _hd_url = unbound l link'
, _hd_uniqId = Nothing
, _hd_uniqIdBdd = Nothing
, _hd_page = Nothing
......
......@@ -17,17 +17,16 @@ Json parser to export towoard CSV GargV3 format.
module Gargantext.Core.Text.Corpus.Parsers.Json2Csv (json2csv, readPatents)
where
import Prelude (read)
import Data.Aeson
import Data.Aeson.TH (deriveJSON)
import Data.ByteString.Lazy (readFile)
import Data.Text (Text, unpack)
import Gargantext.Core.Utils.Prefix (unPrefix)
import qualified Gargantext.Defaults as Defaults
import Gargantext.Prelude
import System.IO (FilePath)
import Gargantext.Core.Text.Corpus.Parsers.CSV (CsvDoc(..), writeFile, headerCsvGargV3)
import Data.Text (unpack)
import Data.Vector (fromList)
import Gargantext.Core.Text.Corpus.Parsers.CSV (CsvDoc(..), writeFile, headerCsvGargV3)
import Gargantext.Core.Utils.Prefix (unPrefix)
import Gargantext.Defaults qualified as Defaults
import Gargantext.Prelude hiding (readFile, writeFile)
import Prelude (read)
data Patent = Patent { _patent_title :: Text
, _patent_abstract :: Text
......
......@@ -18,13 +18,13 @@ citation programs to exchange data.
module Gargantext.Core.Text.Corpus.Parsers.RIS (parser, onField, fieldWith, lines) where
import Data.List (lookup)
import Control.Applicative
import Data.Attoparsec.ByteString (Parser, try, takeTill, take, many1)
import Data.Attoparsec.ByteString (Parser, takeTill, take, try, many1)
import Data.Attoparsec.ByteString.Char8 (isEndOfLine)
import Data.ByteString (ByteString, intercalate)
import Gargantext.Prelude hiding (takeWhile, take)
import qualified Data.List as DL
import Data.ByteString (intercalate)
import Data.List (lookup)
import Data.List qualified as DL
import Gargantext.Prelude hiding (intercalate, lines, takeWhile, take, try)
-------------------------------------------------------------
parser :: Parser [[(ByteString, ByteString)]]
......
......@@ -14,14 +14,14 @@ Presse RIS format parser for Europresse Database.
module Gargantext.Core.Text.Corpus.Parsers.RIS.Presse (presseEnrich) where
import Data.List (lookup)
import Data.Tuple.Extra (first, both)
import Data.Attoparsec.ByteString (parseOnly)
import Data.ByteString (ByteString, length)
import Gargantext.Prelude hiding (takeWhile, take, length)
import Gargantext.Core.Text.Corpus.Parsers.RIS (onField)
import Data.ByteString (length)
import Data.List (lookup)
import Data.Tuple.Extra (both)
import Gargantext.Core (Lang(..))
import qualified Gargantext.Core.Text.Corpus.Parsers.Date.Attoparsec as Date
import Gargantext.Core.Text.Corpus.Parsers.Date.Attoparsec qualified as Date
import Gargantext.Core.Text.Corpus.Parsers.RIS (onField)
import Gargantext.Prelude hiding (takeWhile, take, length)
......@@ -35,8 +35,8 @@ parseDate :: ByteString -> [(ByteString, ByteString)]
parseDate str = either (const []) identity $ parseOnly (Date.parserWith "/") str
parseLang :: ByteString -> [(ByteString, ByteString)]
parseLang "Français" = [(langField, cs $ show FR)]
parseLang "English" = [(langField, cs $ show EN)]
parseLang "Français" = [(langField, cs $ (show FR :: Text))]
parseLang "English" = [(langField, cs $ (show EN :: Text))]
parseLang x = [(langField, x)]
langField :: ByteString
......
{-|
Module : Gargantext.Core.Text.Corpus.Parsers.Telegram
Description :
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
module Gargantext.Core.Text.Corpus.Parsers.Telegram
where
import Data.Aeson
import Data.Text (Text)
--import Data.Time
import GHC.Generics (Generic)
--import Gargantext.Core (Lang(..))
--import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Data.ByteString.Lazy qualified as DBL
import Gargantext.Prelude
import System.FilePath (FilePath)
import qualified Data.ByteString.Lazy as DBL
readFile_Telegram :: FilePath -> IO [TelegramMsg]
readFile_Telegram fp = do
......
......@@ -19,17 +19,16 @@ Portability : POSIX
module Gargantext.Core.Text.Corpus.Parsers.Wikidata where
import Control.Lens (makeLenses, (^.) )
import Data.Maybe (catMaybes)
import Data.Text (Text, concat)
import Data.List qualified as List
import Data.Text (concat)
import Database.HSparql.Connection
import Gargantext.Core (Lang(..))
import Gargantext.Core.Text.Corpus.Parsers.Date (dateSplit)
import Gargantext.Core.Text.Corpus.Parsers.Isidore (unbound)
import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
import Gargantext.Prelude
import Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
import Prelude (String)
import qualified Data.List as List
import Gargantext.Core.Text.Corpus.Parsers.Date (dateSplit)
import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
import Gargantext.Prelude hiding (concat)
import Prelude qualified
......@@ -78,12 +77,12 @@ wikiPageToDocument m wr = do
let hour = Nothing
minute = Nothing
second = Nothing
iso2 = Just $ cs $ show EN
sec = Nothing
iso2 = Just $ show EN
pure $ HyperdataDocument bdd doi url uniqId uniqIdBdd
page title authors institutes source
abstract ((cs . show) <$> date) year month day hour minute second iso2
abstract (show <$> date) year month day hour minute sec iso2
wikidataSelect :: Int -> IO [WikiResult]
......@@ -104,7 +103,7 @@ toWikiResult _ = panic "[G.C.T.C.Parsers.Wikidata.toWikiResult]
wikidataRoute :: EndPoint
wikidataRoute = "https://query.wikidata.org/sparql"
wikidataQuery :: Int -> String
wikidataQuery :: Int -> Prelude.String
wikidataQuery n = List.unlines
[" PREFIX wd: <http://www.wikidata.org/entity/>"
," PREFIX wdt: <http://www.wikidata.org/prop/direct/>"
......@@ -128,5 +127,5 @@ wikidataQuery n = List.unlines
," OPTIONAL {?cid (wdt:P582) ?dateEnd .}"
," OPTIONAL {?cid (wdt:P571) ?dateFlorish .}"
," }"
," LIMIT " <> (cs $ show n)
," LIMIT " <> show n
]
......@@ -18,10 +18,9 @@ module Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
import Control.Lens hiding (element, elements, children)
import Data.ByteString.Lazy (ByteString)
import Data.Text (Text, unpack)
import Data.Text.Encoding.Error (lenientDecode)
import Data.Text (unpack)
import Data.Text.Lazy.Encoding (decodeUtf8With)
import Gargantext.Prelude
import Gargantext.Prelude hiding (ByteString, get, to, decodeUtf8With)
import Network.HTTP.Client (Response)
import Network.Wreq (responseBody, get)
import Text.Taggy.Lens
......
......@@ -22,7 +22,7 @@ import Data.Conduit
import Data.Either
import Data.Text as T
import Data.XML.Types (Event, Name)
import Gargantext.Prelude
import Gargantext.Prelude hiding (force)
import Text.Pandoc
import Text.XML.Stream.Parse
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment