Commit 73ccc76a authored by Alexandre Delanoë's avatar Alexandre Delanoë

merge

parents fa4d0d51 0faeb112
## Version 0.0.5.8.5
* [FRONT] CSS + Design, Graph Toolbar and many things
* [BACK] Security FIX GQL route
* [BACK] Arxiv API connexion
## Version 0.0.5.8.4
* [BACK] GraphQL routes
* [FRONT] CSS, Forest Sidebar
* [HAL] parser back and front
## Version 0.0.5.8.3
* [CRAWLERS] HAL for organizations, example done for IMT
......
......@@ -5,7 +5,7 @@ cabal-version: 1.12
-- see: https://github.com/sol/hpack
name: gargantext
version: 0.0.5.8.3
version: 0.0.5.8.5
synopsis: Search, map, share
description: Please see README.md
category: Data
......@@ -160,6 +160,7 @@ library
Gargantext.Core.Methods.Matrix.Accelerate.Utils
Gargantext.Core.Statistics
Gargantext.Core.Text.Convert
Gargantext.Core.Text.Corpus.API.Arxiv
Gargantext.Core.Text.Corpus.API.Hal
Gargantext.Core.Text.Corpus.API.Isidore
Gargantext.Core.Text.Corpus.API.Istex
......@@ -183,7 +184,6 @@ library
Gargantext.Core.Text.List.Group.WithScores
Gargantext.Core.Text.List.Group.WithStem
Gargantext.Core.Text.List.Learn
Gargantext.Core.Text.List.Management
Gargantext.Core.Text.List.Merge
Gargantext.Core.Text.List.Social
Gargantext.Core.Text.List.Social.Find
......@@ -342,6 +342,7 @@ library
, aeson-lens
, aeson-pretty
, array
, arxiv
, async
, attoparsec
, auto-update
......@@ -360,6 +361,7 @@ library
, conduit-extra
, containers
, contravariant
, crawlerArxiv
, crawlerHAL
, crawlerISTEX
, crawlerIsidore
......
......@@ -6,7 +6,7 @@ name: gargantext
# | | | +----- Layers * : New versions with API additions
# | | | | +--- Layers * : New versions without API breaking changes
# | | | | |
version: '0.0.5.8.3'
version: '0.0.5.8.5'
synopsis: Search, map, share
description: Please see README.md
category: Data
......@@ -132,6 +132,7 @@ library:
- aeson-lens
- aeson-pretty
- array
- arxiv
- async
- attoparsec
- auto-update
......@@ -150,6 +151,7 @@ library:
- conduit-extra
- containers
- contravariant
- crawlerArxiv
- crawlerHAL
- crawlerISTEX
- crawlerIsidore
......
......@@ -36,6 +36,7 @@ instance Arbitrary a => Arbitrary (JobOutput a) where
-- TODO IsidoreAuth
data ExternalAPIs = All
| PubMed
| Arxiv
| HAL
| IsTex
| Isidore
......
{-# OPTIONS_GHC -freduction-depth=0 #-}
{-# OPTIONS_GHC -O0 #-}
{-# LANGUAGE TypeOperators #-}
module Gargantext.API.Client where
import Data.Int
import Data.Maybe
import Data.Map (Map)
import Data.Morpheus.Types.IO (GQLRequest, GQLResponse)
import Data.Proxy
import Data.Text (Text)
import Data.Time.Clock
......@@ -15,6 +17,7 @@ import Gargantext.API.Admin.Auth.Types hiding (Token)
import Gargantext.API.Admin.Orchestrator.Types
import Gargantext.API.Count
import Gargantext.API.EKG
import qualified Gargantext.API.GraphQL as GraphQL
import Gargantext.API.HashedResponse
import Gargantext.API.Ngrams as Ngrams
import Gargantext.API.Ngrams.NgramsTree
......@@ -420,6 +423,13 @@ getMetricsSample :: ClientM Sample
-- | open @<backend:port\/ekg\/index.html@ to see a list of metrics
getMetricSample :: [Text] -> ClientM Value
-- * graphql api
postGraphQL :: Token -> GQLRequest -> ClientM GQLResponse
postGraphQL = client (fstEndpoint (flatten GraphQL.gqapi))
where fstEndpoint :: Proxy (a :<|> b) -> Proxy a
fstEndpoint _ = Proxy
-- * unpacking of client functions to derive all the individual clients
clientApi :: Client ClientM (Flat GargAPI)
......
......@@ -29,6 +29,7 @@ import Data.Morpheus.Types
, RootResolver(..)
, Undefined(..)
)
import Data.Proxy
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser)
import Gargantext.API.Admin.Orchestrator.Types (JobLog)
import Gargantext.API.Prelude (HasJobEnv')
......@@ -58,6 +59,7 @@ import Servant
)
import qualified Servant.Auth as SA
import qualified Servant.Auth.Server as SAS
import Gargantext.API.Admin.Types (HasSettings)
-- | Represents possible GraphQL queries.
data Query m
......@@ -94,7 +96,7 @@ data Contet m
-- | The main GraphQL resolver: how queries, mutations and
-- subscriptions are handled.
rootResolver
:: (HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env)
:: (HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> RootResolver (GargM env GargError) e Query Mutation Undefined
rootResolver =
RootResolver
......@@ -110,7 +112,7 @@ rootResolver =
-- | Main GraphQL "app".
app
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env)
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> App (EVENT (GargM env GargError)) (GargM env GargError)
app = deriveApp rootResolver
......@@ -136,6 +138,9 @@ type Playground = Get '[HTML] ByteString
type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser
:> "gql" :> (GQAPI :<|> Playground)
gqapi :: Proxy API
gqapi = Proxy
-- serveEndpoint ::
-- ( SubApp ServerApp e
-- , PubApp e
......@@ -151,8 +156,7 @@ type API = SA.Auth '[SA.JWT, SA.Cookie] AuthenticatedUser
-- | Implementation of our API.
--api :: Server API
api
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env)
:: (Typeable env, HasConnectionPool env, HasConfig env, HasMail env, HasJobEnv' env, HasSettings env)
=> ServerT API (GargM env GargError)
api (SAS.Authenticated _auser) = httpPubApp [] app :<|> pure httpPlayground
--api _ = panic "401 in graphql" --SAS.throwAll (_ServerError # err401)
api _ = httpPubApp [] app :<|> pure httpPlayground
api _ = panic "401 in graphql" -- SAS.throwAll (_ServerError # err401)
......@@ -42,9 +42,12 @@ import Gargantext.Database.Prelude (HasConnectionPool, HasConfig)
import Gargantext.Database.Query.Table.Node.UpdateOpaleye (updateHyperdata)
import Gargantext.Database.Query.Table.User (getUsersWithHyperdata, getUsersWithNodeHyperdata, updateUserEmail)
import Gargantext.Database.Schema.User (UserLight(..))
import Gargantext.Database.Schema.Node (node_id, node_hyperdata)
import Gargantext.Database.Schema.Node (node_id, node_hyperdata, NodePoly (Node, _node_id))
import Gargantext.Prelude
import GHC.Generics (Generic)
import Gargantext.API.GraphQL.Utils (AuthStatus(Invalid, Valid), authUser)
import Gargantext.API.Admin.Types (HasSettings)
import Gargantext.Database.Admin.Types.Node (unNodeId)
data UserInfo = UserInfo
{ ui_id :: Int
......@@ -75,6 +78,7 @@ data UserInfoArgs
data UserInfoMArgs
= UserInfoMArgs
{ ui_id :: Int
, token :: Text
, ui_username :: Maybe Text
, ui_email :: Maybe Text
, ui_title :: Maybe Text
......@@ -92,6 +96,7 @@ data UserInfoMArgs
} deriving (Generic, GQLType)
type GqlM e env = Resolver QUERY e (GargM env GargError)
type GqlM' e env err = ResolverM e (GargM env err) Int
-- | Function to resolve user from a query.
resolveUserInfos
......@@ -101,47 +106,53 @@ resolveUserInfos UserInfoArgs { user_id } = dbUsers user_id
-- | Mutation for user info
updateUserInfo
:: (HasConnectionPool env, HasConfig env, HasMail env)
=> UserInfoMArgs -> ResolverM e (GargM env GargError) Int
:: (HasConnectionPool env, HasConfig env, HasMail env, HasSettings env)
-- => UserInfoMArgs -> ResolverM e (GargM env err) Int
=> UserInfoMArgs -> GqlM' e env err
updateUserInfo (UserInfoMArgs { ui_id, .. }) = do
-- lift $ printDebug "[updateUserInfo] ui_id" ui_id
users <- lift (getUsersWithNodeHyperdata ui_id)
case users of
[] -> panic $ "[updateUserInfo] User with id " <> (T.pack $ show ui_id) <> " doesn't exist."
((UserLight { .. }, node_u):_) -> do
let u_hyperdata = node_u ^. node_hyperdata
-- lift $ printDebug "[updateUserInfo] u" u
let u_hyperdata' = uh ui_titleL ui_title $
uh ui_sourceL ui_source $
uh ui_cwFirstNameL ui_cwFirstName $
uh ui_cwLastNameL ui_cwLastName $
uh ui_cwCityL ui_cwCity $
uh ui_cwCountryL ui_cwCountry $
uh' ui_cwLabTeamDeptsL ui_cwLabTeamDepts $
uh' ui_cwOrganizationL ui_cwOrganization $
uh ui_cwOfficeL ui_cwOffice $
uh ui_cwRoleL ui_cwRole $
uh ui_cwTouchMailL ui_cwTouchMail $
uh ui_cwTouchPhoneL ui_cwTouchPhone $
u_hyperdata
-- NOTE: We have 1 username and 2 emails: userLight_email and ui_cwTouchMail
-- The userLight_email is more important: it is used for login and sending mail.
-- Therefore we update ui_cwTouchMail and userLight_email.
-- ui_cwTouchMail is to be removed in the future.
let u' = UserLight { userLight_id
, userLight_username
, userLight_email = fromMaybe userLight_email $ view ui_cwTouchMailL u_hyperdata
, userLight_password }
-- lift $ printDebug "[updateUserInfo] with firstName" u_hyperdata'
_ <- lift $ updateHyperdata (node_u ^. node_id) u_hyperdata'
_ <- lift $ updateUserEmail u'
--let _newUser = toUser (u, u_hyperdata')
pure 1
testAuthUser <- lift $ authUser (nId node_u) token
case testAuthUser of
Invalid -> panic "[updateUserInfo] failed to validate user"
Valid -> do
let u_hyperdata = node_u ^. node_hyperdata
-- lift $ printDebug "[updateUserInfo] u" u
let u_hyperdata' = uh ui_titleL ui_title $
uh ui_sourceL ui_source $
uh ui_cwFirstNameL ui_cwFirstName $
uh ui_cwLastNameL ui_cwLastName $
uh ui_cwCityL ui_cwCity $
uh ui_cwCountryL ui_cwCountry $
uh' ui_cwLabTeamDeptsL ui_cwLabTeamDepts $
uh' ui_cwOrganizationL ui_cwOrganization $
uh ui_cwOfficeL ui_cwOffice $
uh ui_cwRoleL ui_cwRole $
uh ui_cwTouchMailL ui_cwTouchMail $
uh ui_cwTouchPhoneL ui_cwTouchPhone $
u_hyperdata
-- NOTE: We have 1 username and 2 emails: userLight_email and ui_cwTouchMail
-- The userLight_email is more important: it is used for login and sending mail.
-- Therefore we update ui_cwTouchMail and userLight_email.
-- ui_cwTouchMail is to be removed in the future.
let u' = UserLight { userLight_id
, userLight_username
, userLight_email = fromMaybe userLight_email $ view ui_cwTouchMailL u_hyperdata
, userLight_password }
-- lift $ printDebug "[updateUserInfo] with firstName" u_hyperdata'
_ <- lift $ updateHyperdata (node_u ^. node_id) u_hyperdata'
_ <- lift $ updateUserEmail u'
--let _newUser = toUser (u, u_hyperdata')
pure 1
where
uh _ Nothing u_hyperdata = u_hyperdata
uh lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ Just val
uh' _ Nothing u_hyperdata = u_hyperdata
uh' lens' (Just val) u_hyperdata = u_hyperdata & lens' .~ val
nId Node {_node_id} = unNodeId _node_id
-- | Inner function to fetch the user from DB.
dbUsers
......
{-|
Module : Gargantext.API.GraphQL.Utils
Description : Utils for GraphQL API
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
module Gargantext.API.GraphQL.Utils where
import Data.Morpheus.Types (GQLTypeOptions, fieldLabelModifier)
import qualified Data.Text as T
import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix)
import Gargantext.Prelude
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import Gargantext.API.Admin.Types (jwtSettings, HasSettings (settings))
import Servant.Auth.Server (verifyJWT, JWTSettings)
import Control.Lens.Getter (view)
import Gargantext.Database.Prelude (Cmd')
import Gargantext.API.Admin.Auth.Types (AuthenticatedUser (AuthenticatedUser, _authUser_id))
import Data.ByteString (ByteString)
import Gargantext.Database.Admin.Types.Node (unNodeId)
unPrefix :: T.Text -> GQLTypeOptions -> GQLTypeOptions
unPrefix prefix options = options { fieldLabelModifier = nflm }
where
nflm label = unCapitalize $ dropPrefix (T.unpack prefix) $ ( fieldLabelModifier options ) label
data AuthStatus = Valid | Invalid
authUser :: (HasSettings env) => Int -> Text -> Cmd' env err AuthStatus
authUser ui_id token = do
let token' = encodeUtf8 token
jwtS <- view $ settings . jwtSettings
u <- liftBase $ getUserFromToken jwtS token'
case u of
Nothing -> pure Invalid
Just au ->
if nId au == ui_id
then pure Valid
else pure Invalid
where
nId AuthenticatedUser {_authUser_id} = unNodeId _authUser_id
getUserFromToken :: JWTSettings -> ByteString -> IO (Maybe AuthenticatedUser)
getUserFromToken = verifyJWT
......@@ -22,6 +22,7 @@ import Gargantext.Database.Action.Flow (DataOrigin(..))
data Database = Empty
| PubMed
| Arxiv
| HAL
| IsTex
| Isidore
......@@ -33,6 +34,7 @@ instance ToSchema Database
database2origin :: Database -> DataOrigin
database2origin Empty = InternalOrigin T.IsTex
database2origin PubMed = ExternalOrigin T.PubMed
database2origin Arxiv = ExternalOrigin T.Arxiv
database2origin HAL = ExternalOrigin T.HAL
database2origin IsTex = ExternalOrigin T.IsTex
database2origin Isidore = ExternalOrigin T.Isidore
......
......@@ -24,9 +24,9 @@ import Control.Concurrent (threadDelay)
import Control.Exception (Exception)
import Control.Lens (Prism', (#))
import Control.Lens.TH (makePrisms)
import Control.Monad.Error.Class (MonadError(..))
import Control.Monad.Except (ExceptT)
import Control.Monad.Reader (ReaderT)
import Control.Monad.Error.Class (MonadError(..))
import Crypto.JOSE.Error as Jose
import Data.Aeson.Types
import Data.Typeable
......@@ -72,11 +72,11 @@ type ErrC err =
)
type GargServerC env err m =
( CmdRandom env err m
( CmdRandom env err m
, HasNodeStory env err m
, EnvC env
, ErrC err
, ToJSON err
, EnvC env
, ErrC err
, ToJSON err
)
type GargServerT env err m api = GargServerC env err m => ServerT api m
......@@ -102,7 +102,6 @@ type GargNoServer' env err m =
)
-------------------------------------------------------------------
data GargError
= GargNodeError NodeError
| GargTreeError TreeError
......@@ -133,7 +132,6 @@ instance HasServerError GargError where
instance HasJoseError GargError where
_JoseError = _GargJoseError
------------------------------------------------------------------------
-- | Utils
-- | Simulate logs
......
......@@ -25,6 +25,7 @@ import Gargantext.API.Admin.Orchestrator.Types (ExternalAPIs(..), externalAPIs)
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import Gargantext.Prelude
import qualified Gargantext.Core.Text.Corpus.API.Arxiv as Arxiv
import qualified Gargantext.Core.Text.Corpus.API.Hal as HAL
import qualified Gargantext.Core.Text.Corpus.API.Isidore as ISIDORE
import qualified Gargantext.Core.Text.Corpus.API.Istex as ISTEX
......@@ -41,6 +42,7 @@ get :: ExternalAPIs
get PubMed _la q limit = PUBMED.get q limit
--docs <- PUBMED.get q default_limit -- EN only by default
--pure (Just $ fromIntegral $ length docs, yieldMany docs)
get Arxiv la q limit = Arxiv.get la q (fromIntegral <$> limit)
get HAL la q limit = HAL.getC la q limit
get IsTex la q limit = do
docs <- ISTEX.get la q limit
......
{-|
Module : Gargantext.Core.Text.Corpus.API.Arxiv
Description : Pubmed API connection
Copyright : (c) CNRS, 2017
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-unused-top-binds #-}
module Gargantext.Core.Text.Corpus.API.Arxiv
where
import Conduit
import Data.Either (Either(..))
import Data.Maybe
import Data.Text (Text)
import qualified Data.Text as Text
import Servant.Client (ClientError)
import Gargantext.Prelude
import Gargantext.Core (Lang(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
import qualified Arxiv as Arxiv
import qualified Network.Api.Arxiv as Ax
type Query = Text
type Limit = Arxiv.Limit
-- | TODO put default pubmed query in gargantext.ini
-- by default: 10K docs
get :: Lang -> Query -> Maybe Limit -> IO (Either ClientError (Maybe Integer, ConduitT () HyperdataDocument IO ()))
get la q l = do
(cnt, resC) <- Arxiv.apiSimpleC l [Text.unpack q]
pure $ Right (Just $ fromIntegral cnt, resC .| mapC (toDoc la))
toDoc :: Lang -> Arxiv.Result -> HyperdataDocument
toDoc l (Arxiv.Result { abstract
, authors = aus
--, categories
, doi
, id
, journal
--, primaryCategory
, publication_date
, title
--, total
, url
, year }
) = HyperdataDocument { _hd_bdd = Just "Arxiv"
, _hd_doi = Just $ Text.pack doi
, _hd_url = Just $ Text.pack url
, _hd_uniqId = Just $ Text.pack id
, _hd_uniqIdBdd = Nothing
, _hd_page = Nothing
, _hd_title = Just $ Text.pack title
, _hd_authors = authors aus
, _hd_institutes = institutes aus
, _hd_source = Just $ Text.pack journal
, _hd_abstract = Just $ Text.pack abstract
, _hd_publication_date = Just $ Text.pack publication_date
, _hd_publication_year = fromIntegral <$> year
, _hd_publication_month = Nothing -- TODO parse publication_date
, _hd_publication_day = Nothing
, _hd_publication_hour = Nothing
, _hd_publication_minute = Nothing
, _hd_publication_second = Nothing
, _hd_language_iso2 = Just $ (Text.pack . show) l }
where
authors :: [Ax.Author] -> Maybe Text
authors [] = Nothing
authors aus' = Just $ (Text.intercalate ", ")
$ map Text.pack
$ map Ax.auName aus'
institutes :: [Ax.Author] -> Maybe Text
institutes [] = Nothing
institutes aus' = Just $ (Text.intercalate ", ")
$ (map (Text.replace ", " " - "))
$ map Text.pack
$ map Ax.auFil aus'
......@@ -40,6 +40,7 @@ import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List
import qualified Data.Map as Map
import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified Data.Vector.Storable as Vec
import qualified Graph.BAC.ProxemyOptim as BAC
import qualified IGraph as Igraph
......@@ -105,14 +106,17 @@ cooc2graphWith' doPartitions distance threshold myCooc = do
(distanceMap, diag, ti) = doDistanceMap distance threshold myCooc
--{- -- Debug
saveAsFileDebug "/tmp/distanceMap" distanceMap
saveAsFileDebug "/tmp/distanceMap.keys" (List.length $ Map.keys distanceMap)
-- saveAsFileDebug "/tmp/distanceMap" distanceMap
-- saveAsFileDebug "/tmp/distanceMap.keys" (List.length $ Map.keys distanceMap)
-- printDebug "similarities" similarities
--}
partitions <- if (Map.size distanceMap > 0)
then doPartitions distanceMap
else panic "Text.Flow: DistanceMap is empty"
else panic $ Text.unlines [ "[Gargantext.C.V.Graph.Tools] Similarity Matrix is empty"
, "Maybe you should add more Map Terms in your list"
, "Tutorial: link todo"
]
let
nodesApprox :: Int
......
resolver:
url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/18.yaml
url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/18/28.yaml
flags: {}
extra-package-dbs: []
skip-ghc-check: true
......@@ -73,9 +73,13 @@ extra-deps:
- git: https://gitlab.iscpif.fr/gargantext/crawlers/istex.git
commit: daeae80365250c4bd539f0a65e271f9aa37f731f
- git: https://gitlab.iscpif.fr/gargantext/crawlers/hal.git
commit: 3bf77f28d3dc71d2e8349cbf422a34cf4c23cd11
commit: 9a43470241690a19c1c381c42a62c5dd4e28dff2
- git: https://gitlab.iscpif.fr/gargantext/crawlers/isidore.git
commit: 3db385e767d2100d8abe900833c6e7de3ac55e1b
- git: https://gitlab.iscpif.fr/gargantext/crawlers/arxiv-api.git
#- git: https://gitlab.iscpif.fr/cgenie/arxiv-api.git
commit: f3e517cc40d92e282c5245b23d253d2ca3f802e5
- arxiv-0.0.3@sha256:02de1114091d11f1f3ab401d104d125ad4301260806feb7f63b3dcefc7db88cf,1588
# NP libs
#- git: https://github.com/np/servant-job.git # waiting for PR
......
......@@ -15,7 +15,9 @@ vim CHANGELOG.md < /dev/tty
# Haskell
#################################################################
YAML="package.yaml"
CABL="gargantext.cabal"
sed -i "s/version:.*/version: \'$VERSION\'/" $YAML
sed -i "s/version:.*/version: $VERSION/" $CABL
git add -u
git commit -m "[VERSION] +1 to ${VERSION}"
git tag $VERSION
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment