Commit 73ccc76a authored by Alexandre Delanoë's avatar Alexandre Delanoë

merge

parents fa4d0d51 0faeb112
## Version 0.0.5.8.5
* [FRONT] CSS + Design, Graph Toolbar and many things
* [BACK] Security FIX GQL route
* [BACK] Arxiv API connexion
## Version 0.0.5.8.4
* [BACK] GraphQL routes
* [FRONT] CSS, Forest Sidebar
* [HAL] parser back and front
## Version 0.0.5.8.3 ## Version 0.0.5.8.3
* [CRAWLERS] HAL for organizations, example done for IMT * [CRAWLERS] HAL for organizations, example done for IMT
......
...@@ -5,7 +5,7 @@ cabal-version: 1.12 ...@@ -5,7 +5,7 @@ cabal-version: 1.12
-- see: https://github.com/sol/hpack -- see: https://github.com/sol/hpack
name: gargantext name: gargantext
version: 0.0.5.8.3 version: 0.0.5.8.5
synopsis: Search, map, share synopsis: Search, map, share
description: Please see README.md description: Please see README.md
category: Data category: Data
...@@ -160,6 +160,7 @@ library ...@@ -160,6 +160,7 @@ library
Gargantext.Core.Methods.Matrix.Accelerate.Utils Gargantext.Core.Methods.Matrix.Accelerate.Utils
Gargantext.Core.Statistics Gargantext.Core.Statistics
Gargantext.Core.Text.Convert Gargantext.Core.Text.Convert
Gargantext.Core.Text.Corpus.API.Arxiv
Gargantext.Core.Text.Corpus.API.Hal Gargantext.Core.Text.Corpus.API.Hal
Gargantext.Core.Text.Corpus.API.Isidore Gargantext.Core.Text.Corpus.API.Isidore
Gargantext.Core.Text.Corpus.API.Istex Gargantext.Core.Text.Corpus.API.Istex
...@@ -183,7 +184,6 @@ library ...@@ -183,7 +184,6 @@ library
Gargantext.Core.Text.List.Group.WithScores Gargantext.Core.Text.List.Group.WithScores
Gargantext.Core.Text.List.Group.WithStem Gargantext.Core.Text.List.Group.WithStem
Gargantext.Core.Text.List.Learn Gargantext.Core.Text.List.Learn
Gargantext.Core.Text.List.Management
Gargantext.Core.Text.List.Merge Gargantext.Core.Text.List.Merge
Gargantext.Core.Text.List.Social Gargantext.Core.Text.List.Social
Gargantext.Core.Text.List.Social.Find Gargantext.Core.Text.List.Social.Find
...@@ -342,6 +342,7 @@ library ...@@ -342,6 +342,7 @@ library
, aeson-lens , aeson-lens
, aeson-pretty , aeson-pretty
, array , array
, arxiv
, async , async
, attoparsec , attoparsec
, auto-update , auto-update
...@@ -360,6 +361,7 @@ library ...@@ -360,6 +361,7 @@ library
, conduit-extra , conduit-extra
, containers , containers
, contravariant , contravariant
, crawlerArxiv
, crawlerHAL , crawlerHAL
, crawlerISTEX , crawlerISTEX
, crawlerIsidore , crawlerIsidore
......
...@@ -6,7 +6,7 @@ name: gargantext ...@@ -6,7 +6,7 @@ name: gargantext
# | | | +----- Layers * : New versions with API additions # | | | +----- Layers * : New versions with API additions
# | | | | +--- Layers * : New versions without API breaking changes # | | | | +--- Layers * : New versions without API breaking changes
# | | | | | # | | | | |
version: '0.0.5.8.3' version: '0.0.5.8.5'
synopsis: Search, map, share synopsis: Search, map, share
description: Please see README.md description: Please see README.md
category: Data category: Data
...@@ -132,6 +132,7 @@ library: ...@@ -132,6 +132,7 @@ library:
- aeson-lens - aeson-lens
- aeson-pretty - aeson-pretty
- array - array
- arxiv
- async - async
- attoparsec - attoparsec
- auto-update - auto-update
...@@ -150,6 +151,7 @@ library: ...@@ -150,6 +151,7 @@ library:
- conduit-extra - conduit-extra
- containers - containers
- contravariant - contravariant
- crawlerArxiv
- crawlerHAL - crawlerHAL
- crawlerISTEX - crawlerISTEX
- crawlerIsidore - crawlerIsidore
......
...@@ -36,6 +36,7 @@ instance Arbitrary a => Arbitrary (JobOutput a) where ...@@ -36,6 +36,7 @@ instance Arbitrary a => Arbitrary (JobOutput a) where
-- TODO IsidoreAuth -- TODO IsidoreAuth
data ExternalAPIs = All data ExternalAPIs = All
| PubMed | PubMed
| Arxiv
| HAL | HAL
| IsTex | IsTex
| Isidore | Isidore
......
{-# OPTIONS_GHC -freduction-depth=0 #-} {-# OPTIONS_GHC -freduction-depth=0 #-}
{-# OPTIONS_GHC -O0 #-} {-# OPTIONS_GHC -O0 #-}
{-# LANGUAGE TypeOperators #-}
module Gargantext.API.Client where module Gargantext.API.Client where
import Data.Int import Data.Int
import Data.Maybe import Data.Maybe
import Data.Map (Map) import Data.Map (Map)
import Data.Morpheus.Types.IO (GQLRequest, GQLResponse)
import Data.Proxy import Data.Proxy
import Data.Text (Text) import Data.Text (Text)
import Data.Time.Clock import Data.Time.Clock
...@@ -15,6 +17,7 @@ import Gargantext.API.Admin.Auth.Types hiding (Token) ...@@ -15,6 +17,7 @@ import Gargantext.API.Admin.Auth.Types hiding (Token)
import Gargantext.API.Admin.Orchestrator.Types import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.API.Count import Gargantext.API.Count
import Gargantext.API.EKG import Gargantext.API.EKG
import qualified Gargantext.API.GraphQL as GraphQL
import Gargantext.API.HashedResponse import Gargantext.API.HashedResponse
import Gargantext.API.Ngrams as Ngrams import Gargantext.API.Ngrams as Ngrams
import Gargantext.API.Ngrams.NgramsTree import Gargantext.API.Ngrams.NgramsTree
...@@ -420,6 +423,13 @@ getMetricsSample :: ClientM Sample ...@@ -420,6 +423,13 @@ getMetricsSample :: ClientM Sample
-- | open @<backend:port\/ekg\/index.html@ to see a list of metrics -- | open @<backend:port\/ekg\/index.html@ to see a list of metrics
getMetricSample :: [Text] -> ClientM Value getMetricSample :: [Text] -> ClientM Value
-- * graphql api
postGraphQL :: Token -> GQLRequest -> ClientM GQLResponse
postGraphQL = client (fstEndpoint (flatten GraphQL.gqapi))
where fstEndpoint :: Proxy (a :<|> b) -> Proxy a
fstEndpoint _ = Proxy
-- * unpacking of client functions to derive all the individual clients -- * unpacking of client functions to derive all the individual clients
clientApi :: Client ClientM (Flat GargAPI) clientApi :: Client ClientM (Flat GargAPI)
......
...@@ -29,6 +29,7 @@ import Data.Morpheus.Types ...@@ -29,6 +29,7 @@ import Data.Morpheus.Types
, RootResolver(..) , RootResolver(..)
, Undefined(..) , Undefined(..)
) )
import Data.Proxy
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser) import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
import Gargantext.API.Admin.Orchestrator.Types (JobLog) import Gargantext.API.Admin.Orchestrator.Types (JobLog)
import Gargantext.API.Prelude (HasJobEnv') import Gargantext.API.Prelude (HasJobEnv')
...@@ -58,6 +59,7 @@ import Servant ...@@ -58,6 +59,7 @@ import Servant
) )
import qualified Servant.Auth as SA import qualified Servant.Auth as SA
import qualified Servant.Auth.Server as SAS import qualified Servant.Auth.Server as SAS
import Gargantext.API.Admin.Types (HasSettings)
-- | Represents possible GraphQL queries. -- | Represents possible GraphQL queries.
data Query m data Query m
...@@ -94,7 +96,7 @@ data Contet m ...@@ -94,7 +96,7 @@ data Contet m
-- | The main GraphQL resolver: how queries, mutations and -- | The main GraphQL resolver: how queries, mutations and
-- subscriptions are handled. -- subscriptions are handled.
rootResolver rootResolver
:: (HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env) :: (HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> RootResolver (GargM env GargError) e Query Mutation Undefined => RootResolver (GargM env GargError) e Query Mutation Undefined
rootResolver = rootResolver =
RootResolver RootResolver
...@@ -110,7 +112,7 @@ rootResolver = ...@@ -110,7 +112,7 @@ rootResolver =
-- | Main GraphQL "app". -- | Main GraphQL "app".
app app
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env) :: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> App (EVENT (GargM env GargError)) (GargM env GargError) => App (EVENT (GargM env GargError)) (GargM env GargError)
app = deriveApp rootResolver app = deriveApp rootResolver
...@@ -136,6 +138,9 @@ type Playground = Get '[HTML] ByteString ...@@ -136,6 +138,9 @@ type Playground = Get '[HTML] ByteString
type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser
:> "gql" :> (GQAPI :<|> Playground) :> "gql" :> (GQAPI :<|> Playground)
gqapi :: Proxy API
gqapi = Proxy
-- serveEndpoint :: -- serveEndpoint ::
-- ( SubApp ServerApp e -- ( SubApp ServerApp e
-- , PubApp e -- , PubApp e
...@@ -151,8 +156,7 @@ type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser ...@@ -151,8 +156,7 @@ type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser
-- | Implementation of our API. -- | Implementation of our API.
--api :: Server API --api :: Server API
api api
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env) :: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> ServerT API (GargM env GargError) => ServerT API (GargM env GargError)
api (SAS.Authenticated _auser) = httpPubApp [] app :<|> pure httpPlayground api (SAS.Authenticated _auser) = httpPubApp [] app :<|> pure httpPlayground
--api _ = panic "401 in graphql" --SAS.throwAll (_ServerError # err401) api _ = panic "401 in graphql" -- SAS.throwAll (_ServerError # err401)
api _ = httpPubApp [] app :<|> pure httpPlayground
...@@ -42,9 +42,12 @@ import Gargantext.Database.Prelude (HasConnectionPool, HasConfig) ...@@ -42,9 +42,12 @@ import Gargantext.Database.Prelude (HasConnectionPool, HasConfig)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata) import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Query.Table.User (getUsersWithHyperdata, getUsersWithNodeHyperdata, updateUserEmail) import Gargantext.Database.Query.Table.User (getUsersWithHyperdata, getUsersWithNodeHyperdata, updateUserEmail)
import Gargantext.Database.Schema.User (UserLight(..)) import Gargantext.Database.Schema.User (UserLight(..))
import Gargantext.Database.Schema.Node (node_id, node_hyperdata) import Gargantext.Database.Schema.Node (node_id, node_hyperdata, NodePoly (Node, _node_id))
import Gargantext.Prelude import Gargantext.Prelude
import GHC.Generics (Generic) import GHC.Generics (Generic)
import Gargantext.API.GraphQL.Utils (AuthStatus(Invalid, Valid), authUser)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.Database.Admin.Types.Node (unNodeId)
data UserInfo = UserInfo data UserInfo = UserInfo
{ ui_id :: Int { ui_id :: Int
...@@ -75,6 +78,7 @@ data UserInfoArgs ...@@ -75,6 +78,7 @@ data UserInfoArgs
data UserInfoMArgs data UserInfoMArgs
= UserInfoMArgs = UserInfoMArgs
{ ui_id :: Int { ui_id :: Int
, token :: Text
, ui_username :: Maybe Text , ui_username :: Maybe Text
, ui_email :: Maybe Text , ui_email :: Maybe Text
, ui_title :: Maybe Text , ui_title :: Maybe Text
...@@ -92,6 +96,7 @@ data UserInfoMArgs ...@@ -92,6 +96,7 @@ data UserInfoMArgs
} deriving (Generic, GQLType) } deriving (Generic, GQLType)
type GqlM e env = Resolver QUERY e (GargM env GargError) type GqlM e env = Resolver QUERY e (GargM env GargError)
type GqlM' e env err = ResolverM e (GargM env err) Int
-- | Function to resolve user from a query. -- | Function to resolve user from a query.
resolveUserInfos resolveUserInfos
...@@ -101,47 +106,53 @@ resolveUserInfos UserInfoArgs { user_id } = dbUsers user_id ...@@ -101,47 +106,53 @@ resolveUserInfos UserInfoArgs { user_id } = dbUsers user_id
-- | Mutation for user info -- | Mutation for user info
updateUserInfo updateUserInfo
:: (HasConnectionPool env, HasConfig env, HasMail env) :: (HasConnectionPool env, HasConfig env, HasMail env, HasSettings env)
=> UserInfoMArgs -> ResolverM e (GargM env GargError) Int -- => UserInfoMArgs -> ResolverM e (GargM env err) Int
=> UserInfoMArgs -> GqlM' e env err
updateUserInfo (UserInfoMArgs { ui_id, .. }) = do updateUserInfo (UserInfoMArgs { ui_id, .. }) = do
-- lift $ printDebug "[updateUserInfo] ui_id" ui_id -- lift $ printDebug "[updateUserInfo] ui_id" ui_id
users <- lift (getUsersWithNodeHyperdata ui_id) users <- lift (getUsersWithNodeHyperdata ui_id)
case users of case users of
[] -> panic $ "[updateUserInfo] User with id " <> (T.pack $ show ui_id) <> " doesn't exist." [] -> panic $ "[updateUserInfo] User with id " <> (T.pack $ show ui_id) <> " doesn't exist."
((UserLight { .. }, node_u):_) -> do ((UserLight { .. }, node_u):_) -> do
let u_hyperdata = node_u ^. node_hyperdata testAuthUser <- lift $ authUser (nId node_u) token
-- lift $ printDebug "[updateUserInfo] u" u case testAuthUser of
let u_hyperdata' = uh ui_titleL ui_title $ Invalid -> panic "[updateUserInfo] failed to validate user"
uh ui_sourceL ui_source $ Valid -> do
uh ui_cwFirstNameL ui_cwFirstName $ let u_hyperdata = node_u ^. node_hyperdata
uh ui_cwLastNameL ui_cwLastName $ -- lift $ printDebug "[updateUserInfo] u" u
uh ui_cwCityL ui_cwCity $ let u_hyperdata' = uh ui_titleL ui_title $
uh ui_cwCountryL ui_cwCountry $ uh ui_sourceL ui_source $
uh' ui_cwLabTeamDeptsL ui_cwLabTeamDepts $ uh ui_cwFirstNameL ui_cwFirstName $
uh' ui_cwOrganizationL ui_cwOrganization $ uh ui_cwLastNameL ui_cwLastName $
uh ui_cwOfficeL ui_cwOffice $ uh ui_cwCityL ui_cwCity $
uh ui_cwRoleL ui_cwRole $ uh ui_cwCountryL ui_cwCountry $
uh ui_cwTouchMailL ui_cwTouchMail $ uh' ui_cwLabTeamDeptsL ui_cwLabTeamDepts $
uh ui_cwTouchPhoneL ui_cwTouchPhone $ uh' ui_cwOrganizationL ui_cwOrganization $
u_hyperdata uh ui_cwOfficeL ui_cwOffice $
-- NOTE: We have 1 username and 2 emails: userLight_email and ui_cwTouchMail uh ui_cwRoleL ui_cwRole $
-- The userLight_email is more important: it is used for login and sending mail. uh ui_cwTouchMailL ui_cwTouchMail $
-- Therefore we update ui_cwTouchMail and userLight_email. uh ui_cwTouchPhoneL ui_cwTouchPhone $
-- ui_cwTouchMail is to be removed in the future. u_hyperdata
let u' = UserLight { userLight_id -- NOTE: We have 1 username and 2 emails: userLight_email and ui_cwTouchMail
, userLight_username -- The userLight_email is more important: it is used for login and sending mail.
, userLight_email = fromMaybe userLight_email $ view ui_cwTouchMailL u_hyperdata -- Therefore we update ui_cwTouchMail and userLight_email.
, userLight_password } -- ui_cwTouchMail is to be removed in the future.
-- lift $ printDebug "[updateUserInfo] with firstName" u_hyperdata' let u' = UserLight { userLight_id
_ <- lift $ updateHyperdata (node_u ^. node_id) u_hyperdata' , userLight_username
_ <- lift $ updateUserEmail u' , userLight_email = fromMaybe userLight_email $ view ui_cwTouchMailL u_hyperdata
--let _newUser = toUser (u, u_hyperdata') , userLight_password }
pure 1 -- lift $ printDebug "[updateUserInfo] with firstName" u_hyperdata'
_ <- lift $ updateHyperdata (node_u ^. node_id) u_hyperdata'
_ <- lift $ updateUserEmail u'
--let _newUser = toUser (u, u_hyperdata')
pure 1
where where
uh _ Nothing u_hyperdata = u_hyperdata uh _ Nothing u_hyperdata = u_hyperdata
uh lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ Just val uh lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ Just val
uh' _ Nothing u_hyperdata = u_hyperdata uh' _ Nothing u_hyperdata = u_hyperdata
uh' lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ val uh' lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ val
nId Node {_node_id} = unNodeId _node_id
-- | Inner function to fetch the user from DB. -- | Inner function to fetch the user from DB.
dbUsers dbUsers
......
{-|
Module : Gargantext.API.GraphQL.Utils
Description : Utils for GraphQL API
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
module Gargantext.API.GraphQL.Utils where module Gargantext.API.GraphQL.Utils where
import Data.Morpheus.Types (GQLTypeOptions, fieldLabelModifier) import Data.Morpheus.Types (GQLTypeOptions, fieldLabelModifier)
import qualified Data.Text as T import qualified Data.Text as T
import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix) import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix)
import Gargantext.Prelude import Gargantext.Prelude
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import Gargantext.API.Admin.Types (jwtSettings, HasSettings (settings))
import Servant.Auth.Server (verifyJWT, JWTSettings)
import Control.Lens.Getter (view)
import Gargantext.Database.Prelude (Cmd')
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser (AuthenticatedUser, _authUser_id))
import Data.ByteString (ByteString)
import Gargantext.Database.Admin.Types.Node (unNodeId)
unPrefix :: T.Text -> GQLTypeOptions -> GQLTypeOptions unPrefix :: T.Text -> GQLTypeOptions -> GQLTypeOptions
unPrefix prefix options = options { fieldLabelModifier = nflm } unPrefix prefix options = options { fieldLabelModifier = nflm }
where where
nflm label = unCapitalize $ dropPrefix (T.unpack prefix) $ ( fieldLabelModifier options ) label nflm label = unCapitalize $ dropPrefix (T.unpack prefix) $ ( fieldLabelModifier options ) label
data AuthStatus = Valid | Invalid
authUser :: (HasSettings env) => Int -> Text -> Cmd' env err AuthStatus
authUser ui_id token = do
let token' = encodeUtf8 token
jwtS <- view $ settings . jwtSettings
u <- liftBase $ getUserFromToken jwtS token'
case u of
Nothing -> pure Invalid
Just au ->
if nId au == ui_id
then pure Valid
else pure Invalid
where
nId AuthenticatedUser {_authUser_id} = unNodeId _authUser_id
getUserFromToken :: JWTSettings -> ByteString -> IO (Maybe AuthenticatedUser)
getUserFromToken = verifyJWT
...@@ -22,6 +22,7 @@ import Gargantext.Database.Action.Flow (DataOrigin(..)) ...@@ -22,6 +22,7 @@ import Gargantext.Database.Action.Flow (DataOrigin(..))
data Database = Empty data Database = Empty
| PubMed | PubMed
| Arxiv
| HAL | HAL
| IsTex | IsTex
| Isidore | Isidore
...@@ -33,6 +34,7 @@ instance ToSchema Database ...@@ -33,6 +34,7 @@ instance ToSchema Database
database2origin :: Database -> DataOrigin database2origin :: Database -> DataOrigin
database2origin Empty = InternalOrigin T.IsTex database2origin Empty = InternalOrigin T.IsTex
database2origin PubMed = ExternalOrigin T.PubMed database2origin PubMed = ExternalOrigin T.PubMed
database2origin Arxiv = ExternalOrigin T.Arxiv
database2origin HAL = ExternalOrigin T.HAL database2origin HAL = ExternalOrigin T.HAL
database2origin IsTex = ExternalOrigin T.IsTex database2origin IsTex = ExternalOrigin T.IsTex
database2origin Isidore = ExternalOrigin T.Isidore database2origin Isidore = ExternalOrigin T.Isidore
......
...@@ -24,9 +24,9 @@ import Control.Concurrent (threadDelay) ...@@ -24,9 +24,9 @@ import Control.Concurrent (threadDelay)
import Control.Exception (Exception) import Control.Exception (Exception)
import Control.Lens (Prism', (#)) import Control.Lens (Prism', (#))
import Control.Lens.TH (makePrisms) import Control.Lens.TH (makePrisms)
import Control.Monad.Error.Class (MonadError(..))
import Control.Monad.Except (ExceptT) import Control.Monad.Except (ExceptT)
import Control.Monad.Reader (ReaderT) import Control.Monad.Reader (ReaderT)
import Control.Monad.Error.Class (MonadError(..))
import Crypto.JOSE.Error as Jose import Crypto.JOSE.Error as Jose
import Data.Aeson.Types import Data.Aeson.Types
import Data.Typeable import Data.Typeable
...@@ -72,11 +72,11 @@ type ErrC err = ...@@ -72,11 +72,11 @@ type ErrC err =
) )
type GargServerC env err m = type GargServerC env err m =
( CmdRandom env err m ( CmdRandom env err m
, HasNodeStory env err m , HasNodeStory env err m
, EnvC env , EnvC env
, ErrC err , ErrC err
, ToJSON err , ToJSON err
) )
type GargServerT env err m api = GargServerC env err m => ServerT api m type GargServerT env err m api = GargServerC env err m => ServerT api m
...@@ -102,7 +102,6 @@ type GargNoServer' env err m = ...@@ -102,7 +102,6 @@ type GargNoServer' env err m =
) )
------------------------------------------------------------------- -------------------------------------------------------------------
data GargError data GargError
= GargNodeError NodeError = GargNodeError NodeError
| GargTreeError TreeError | GargTreeError TreeError
...@@ -133,7 +132,6 @@ instance HasServerError GargError where ...@@ -133,7 +132,6 @@ instance HasServerError GargError where
instance HasJoseError GargError where instance HasJoseError GargError where
_JoseError = _GargJoseError _JoseError = _GargJoseError
------------------------------------------------------------------------ ------------------------------------------------------------------------
-- | Utils -- | Utils
-- | Simulate logs -- | Simulate logs
......
...@@ -25,6 +25,7 @@ import Gargantext.API.Admin.Orchestrator.Types (ExternalAPIs(..), externalAPIs) ...@@ -25,6 +25,7 @@ import Gargantext.API.Admin.Orchestrator.Types (ExternalAPIs(..), externalAPIs)
import Gargantext.Core (Lang(..)) import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..)) import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude import Gargantext.Prelude
import qualified Gargantext.Core.Text.Corpus.API.Arxiv as Arxiv
import qualified Gargantext.Core.Text.Corpus.API.Hal as HAL import qualified Gargantext.Core.Text.Corpus.API.Hal as HAL
import qualified Gargantext.Core.Text.Corpus.API.Isidore as ISIDORE import qualified Gargantext.Core.Text.Corpus.API.Isidore as ISIDORE
import qualified Gargantext.Core.Text.Corpus.API.Istex as ISTEX import qualified Gargantext.Core.Text.Corpus.API.Istex as ISTEX
...@@ -41,6 +42,7 @@ get :: ExternalAPIs ...@@ -41,6 +42,7 @@ get :: ExternalAPIs
get PubMed _la q limit = PUBMED.get q limit get PubMed _la q limit = PUBMED.get q limit
--docs <- PUBMED.get q default_limit -- EN only by default --docs <- PUBMED.get q default_limit -- EN only by default
--pure (Just $ fromIntegral $ length docs, yieldMany docs) --pure (Just $ fromIntegral $ length docs, yieldMany docs)
get Arxiv la q limit = Arxiv.get la q (fromIntegral <$> limit)
get HAL la q limit = HAL.getC la q limit get HAL la q limit = HAL.getC la q limit
get IsTex la q limit = do get IsTex la q limit = do
docs <- ISTEX.get la q limit docs <- ISTEX.get la q limit
......
{-|
Module : Gargantext.Core.Text.Corpus.API.Arxiv
Description : Pubmed API connection
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-unused-top-binds #-}
module Gargantext.Core.Text.Corpus.API.Arxiv
where
import Conduit
import Data.Either (Either(..))
import Data.Maybe
import Data.Text (Text)
import qualified Data.Text as Text
import Servant.Client (ClientError)
import Gargantext.Prelude
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import qualified Arxiv as Arxiv
import qualified Network.Api.Arxiv as Ax
type Query = Text
type Limit = Arxiv.Limit
-- | TODO put default pubmed query in gargantext.ini
-- by default: 10K docs
get :: Lang -> Query -> Maybe Limit -> IO (Either ClientError (Maybe Integer, ConduitT () HyperdataDocument IO ()))
get la q l = do
(cnt, resC) <- Arxiv.apiSimpleC l [Text.unpack q]
pure $ Right (Just $ fromIntegral cnt, resC .| mapC (toDoc la))
toDoc :: Lang -> Arxiv.Result -> HyperdataDocument
toDoc l (Arxiv.Result { abstract
, authors = aus
--, categories
, doi
, id
, journal
--, primaryCategory
, publication_date
, title
--, total
, url
, year }
) = HyperdataDocument { _hd_bdd = Just "Arxiv"
, _hd_doi = Just $ Text.pack doi
, _hd_url = Just $ Text.pack url
, _hd_uniqId = Just $ Text.pack id
, _hd_uniqIdBdd = Nothing
, _hd_page = Nothing
, _hd_title = Just $ Text.pack title
, _hd_authors = authors aus
, _hd_institutes = institutes aus
, _hd_source = Just $ Text.pack journal
, _hd_abstract = Just $ Text.pack abstract
, _hd_publication_date = Just $ Text.pack publication_date
, _hd_publication_year = fromIntegral <$> year
, _hd_publication_month = Nothing -- TODO parse publication_date
, _hd_publication_day = Nothing
, _hd_publication_hour = Nothing
, _hd_publication_minute = Nothing
, _hd_publication_second = Nothing
, _hd_language_iso2 = Just $ (Text.pack . show) l }
where
authors :: [Ax.Author] -> Maybe Text
authors [] = Nothing
authors aus' = Just $ (Text.intercalate ", ")
$ map Text.pack
$ map Ax.auName aus'
institutes :: [Ax.Author] -> Maybe Text
institutes [] = Nothing
institutes aus' = Just $ (Text.intercalate ", ")
$ (map (Text.replace ", " " - "))
$ map Text.pack
$ map Ax.auFil aus'
...@@ -40,6 +40,7 @@ import qualified Data.HashMap.Strict as HashMap ...@@ -40,6 +40,7 @@ import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List import qualified Data.List as List
import qualified Data.Map as Map import qualified Data.Map as Map
import qualified Data.Set as Set import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified Data.Vector.Storable as Vec import qualified Data.Vector.Storable as Vec
import qualified Graph.BAC.ProxemyOptim as BAC import qualified Graph.BAC.ProxemyOptim as BAC
import qualified IGraph as Igraph import qualified IGraph as Igraph
...@@ -105,14 +106,17 @@ cooc2graphWith' doPartitions distance threshold myCooc = do ...@@ -105,14 +106,17 @@ cooc2graphWith' doPartitions distance threshold myCooc = do
(distanceMap, diag, ti) = doDistanceMap distance threshold myCooc (distanceMap, diag, ti) = doDistanceMap distance threshold myCooc
--{- -- Debug --{- -- Debug
saveAsFileDebug "/tmp/distanceMap" distanceMap -- saveAsFileDebug "/tmp/distanceMap" distanceMap
saveAsFileDebug "/tmp/distanceMap.keys" (List.length $ Map.keys distanceMap) -- saveAsFileDebug "/tmp/distanceMap.keys" (List.length $ Map.keys distanceMap)
-- printDebug "similarities" similarities -- printDebug "similarities" similarities
--} --}
partitions <- if (Map.size distanceMap > 0) partitions <- if (Map.size distanceMap > 0)
then doPartitions distanceMap then doPartitions distanceMap
else panic "Text.Flow: DistanceMap is empty" else panic $ Text.unlines [ "[Gargantext.C.V.Graph.Tools] Similarity Matrix is empty"
, "Maybe you should add more Map Terms in your list"
, "Tutorial: link todo"
]
let let
nodesApprox :: Int nodesApprox :: Int
......
resolver: resolver:
url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/18.yaml url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/28.yaml
flags: {} flags: {}
extra-package-dbs: [] extra-package-dbs: []
skip-ghc-check: true skip-ghc-check: true
...@@ -73,9 +73,13 @@ extra-deps: ...@@ -73,9 +73,13 @@ extra-deps:
- git: https://gitlab.iscpif.fr/gargantext/crawlers/istex.git - git: https://gitlab.iscpif.fr/gargantext/crawlers/istex.git
commit: daeae80365250c4bd539f0a65e271f9aa37f731f commit: daeae80365250c4bd539f0a65e271f9aa37f731f
- git: https://gitlab.iscpif.fr/gargantext/crawlers/hal.git - git: https://gitlab.iscpif.fr/gargantext/crawlers/hal.git
commit: 3bf77f28d3dc71d2e8349cbf422a34cf4c23cd11 commit: 9a43470241690a19c1c381c42a62c5dd4e28dff2
- git: https://gitlab.iscpif.fr/gargantext/crawlers/isidore.git - git: https://gitlab.iscpif.fr/gargantext/crawlers/isidore.git
commit: 3db385e767d2100d8abe900833c6e7de3ac55e1b commit: 3db385e767d2100d8abe900833c6e7de3ac55e1b
- git: https://gitlab.iscpif.fr/gargantext/crawlers/arxiv-api.git
#- git: https://gitlab.iscpif.fr/cgenie/arxiv-api.git
commit: f3e517cc40d92e282c5245b23d253d2ca3f802e5
- arxiv-0.0.3@sha256:02de1114091d11f1f3ab401d104d125ad4301260806feb7f63b3dcefc7db88cf,1588
# NP libs # NP libs
#- git: https://github.com/np/servant-job.git # waiting for PR #- git: https://github.com/np/servant-job.git # waiting for PR
......
...@@ -15,7 +15,9 @@ vim CHANGELOG.md < /dev/tty ...@@ -15,7 +15,9 @@ vim CHANGELOG.md < /dev/tty
# Haskell # Haskell
################################################################# #################################################################
YAML="package.yaml" YAML="package.yaml"
CABL="gargantext.cabal"
sed -i "s/version:.*/version: \'$VERSION\'/" $YAML sed -i "s/version:.*/version: \'$VERSION\'/" $YAML
sed -i "s/version:.*/version: $VERSION/" $CABL
git add -u git add -u
git commit -m "[VERSION] +1 to ${VERSION}" git commit -m "[VERSION] +1 to ${VERSION}"
git tag $VERSION git tag $VERSION
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment