Skip to content

haskell-gargantext
haskell-gargantext
Commit 45c0817e authored by Alp Mestanogullari's avatar Alp Mestanogullari
Browse files
Options

introduce EKG-based monitoring infrastructure

parent f26f41e8
Pipeline #2027 failed with stage
    in 10 minutes and 11 seconds
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing with 15139 additions and 8 deletions
    cabal.project
    View file @ 45c0817e
    ...@@ -13,6 +13,11 @@ source-repository-package ...@@ -13,6 +13,11 @@ source-repository-package
    location: https://gitlab.iscpif.fr/gargantext/patches-class.git location: https://gitlab.iscpif.fr/gargantext/patches-class.git
    tag: 271ba32d6c940029dc653354dd7974a819f48e77 tag: 271ba32d6c940029dc653354dd7974a819f48e77
    source-repository-package
    type: git
    location: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude.git
    tag: 35b09629a658fc16cc9ff63e7591e58511cd98a7
    -- External Data API connectors -- External Data API connectors
    source-repository-package source-repository-package
    type: git type: git
    ...@@ -76,7 +81,7 @@ source-repository-package ...@@ -76,7 +81,7 @@ source-repository-package
    source-repository-package source-repository-package
    type: git type: git
    location: https://github.com/delanoe/haskell-opaleye.git location: https://github.com/delanoe/haskell-opaleye.git
    tag: 63ee65d974e9d20eaaf17a2e83652175988cbb79 tag: d3ab7acd5ede737478763630035aa880f7e34444
    source-repository-package source-repository-package
    type: git type: git
    ...@@ -100,4 +105,4 @@ source-repository-package ...@@ -100,4 +105,4 @@ source-repository-package
    location: https://gitlab.iscpif.fr/anoe/accelerate-utility.git location: https://gitlab.iscpif.fr/anoe/accelerate-utility.git
    tag: 83ada76e78ac10d9559af8ed6bd4064ec81308e4 tag: 83ada76e78ac10d9559af8ed6bd4064ec81308e4
    constraints: unordered-containers==0.2.13.* constraints: unordered-containers==0.2.14.*
    \ No newline at end of file \ No newline at end of file
    ekg-assets/bootstrap-1.4.0.css 0 → 100755
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/bootstrap-1.4.0.min.css 0 → 100755
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/index.html 0 → 100644
    View file @ 45c0817e
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset="utf-8">
    <link rel="stylesheet" href="bootstrap-1.4.0.min.css">
    <link rel="stylesheet" href="monitor.css" type="text/css">
    <script type="text/javascript" src="jquery-1.6.4.min.js"></script>
    <script type="text/javascript" src="jquery.flot.min.js"></script>
    <title>ekg</title>
    </head>
    <body>
    <div class="topbar">
    <div class="topbar-inner">
    <div class="container-fluid">
    <span class="brand">ekg</span>
    <p class="pull-right">Polling interval:
    <select id="updateInterval" class="small">
    <option value="100">100 ms</option>
    <option value="200">200 ms</option>
    <option value="500">500 ms</option>
    <option value="1000" selected="selected">1 s</option>
    <option value="2000">2 s</option>
    <option value="5000">5 s</option>
    <option value="10000">10 s</option>
    </select> |
    <button id="pause-ui" class="btn">Pause UI</button>
    </p>
    </div>
    </div>
    </div>
    <div class="container">
    <div class="row">
    <div class="alert-message error fade in hide" data-alert="alert">
    <p>Lost connection to server.</p>
    </div>
    </div>
    <div class="row">
    <div id="plots" class="span11">
    <div id="current-bytes-used-plot" class="plot-container">
    <h3>Current residency</h3>
    <div class="plot"></div>
    </div>
    <div id="allocation-rate-plot" class="plot-container">
    <h3>Allocation rate</h3>
    <div class="plot"></div>
    </div>
    <div id="productivity-plot" class="plot-container">
    <h3>Productivity</h3>
    <div class="plot"></div>
    </div>
    </div>
    <div class="span5">
    <h3>GC and memory statistics</h3>
    <table class="condensed-table">
    <thead>
    <tr>
    <th>Statistic</th>
    <th>Value</th>
    </tr>
    </thead>
    <tbody>
    <tr>
    <td>Maximum residency</td>
    <td id="max-bytes-used" class="span3 value">0</td>
    </tr>
    <tr>
    <td>Current residency</td>
    <td id="current-bytes-used" class="value">0</td>
    </tr>
    <tr>
    <td>Maximum slop</td>
    <td id="max-bytes-slop" class="value">0</td>
    </tr>
    <tr>
    <td>Current slop</td>
    <td id="current-bytes-slop" class="value">0</td>
    </tr>
    <tr>
    <td>Productivity (wall clock time)</td>
    <td id="productivity-wall" class="value">0</td>
    </tr>
    <tr>
    <td>Productivity (cpu time)</td>
    <td id="productivity-cpu" class="value">0</td>
    </tr>
    <tr>
    <td>Allocation rate</td>
    <td id="allocation-rate" class="value">0</td>
    </tr>
    </tbody>
    </table>
    <h3>Metrics</h3>
    <table id="metric-table" class="condensed-table">
    <thead>
    <tr>
    <th class="span4">Name</th>
    <th class="span1">Value</th>
    </tr>
    </thead>
    <tbody>
    </tbody>
    </table>
    </div>
    </div>
    </div>
    <script type="text/javascript" src="monitor.js"></script>
    </body>
    </html>
    ekg-assets/jquery-1.6.4.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery-1.6.4.min.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery.flot.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery.flot.min.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/monitor.css 0 → 100644
    View file @ 45c0817e
    /**
    * Blueprint/flot compatibility
    *
    * Resets some styles back to the browser default.
    */
    .plot table {
    width: auto;
    border-spacing: 2px;
    }
    .plot th,
    .plot td,
    .plot caption {
    padding: 0;
    }
    /**
    * Body margin
    */
    body {
    padding-top: 60px;
    }
    /**
    * Plots
    */
    .plot {
    width: 600px;
    height: 300px;
    margin-bottom: 1.5em;
    }
    .close-button {
    float: right;
    cursor: pointer;
    }
    /**
    * Table
    */
    .value {
    text-align: right;
    }
    .string {
    text-align: left;
    }
    .graph-button {
    cursor: pointer;
    vertical-align: middle;
    }
    ekg-assets/monitor.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    package.yaml
    View file @ 45c0817e
    ...@@ -25,6 +25,15 @@ default-extensions: ...@@ -25,6 +25,15 @@ default-extensions:
    - OverloadedStrings - OverloadedStrings
    - RankNTypes - RankNTypes
    - RecordWildCards - RecordWildCards
    data-files:
    - ekg-assets/index.html
    - ekg-assets/monitor.js
    - ekg-assets/monitor.css
    - ekg-assets/jquery.flot.min.js
    - ekg-assets/jquery-1.6.4.min.js
    - ekg-assets/bootstrap-1.4.0.min.css
    - ekg-assets/chart_line_add.png
    - ekg-assets/cross.png
    library: library:
    source-dirs: src source-dirs: src
    ghc-options: ghc-options:
    ...@@ -137,6 +146,8 @@ library: ...@@ -137,6 +146,8 @@ library:
    - deepseq - deepseq
    - directory - directory
    - duckling - duckling
    - ekg-core
    - ekg-json
    - exceptions - exceptions
    - fast-logger - fast-logger
    - fclabels - fclabels
    ...@@ -206,6 +217,7 @@ library: ...@@ -206,6 +217,7 @@ library:
    - servant-blaze - servant-blaze
    - servant-cassava - servant-cassava
    - servant-client - servant-client
    - servant-ekg
    - servant-job - servant-job
    - servant-mock - servant-mock
    - servant-multipart - servant-multipart
    ...@@ -257,6 +269,7 @@ executables: ...@@ -257,6 +269,7 @@ executables:
    - -rtsopts - -rtsopts
    - -threaded - -threaded
    - -with-rtsopts=-N - -with-rtsopts=-N
    - -with-rtsopts=-T
    - -fprof-auto - -fprof-auto
    dependencies: dependencies:
    - base - base
    ......
    src/Gargantext/API.hs
    View file @ 45c0817e
    ...@@ -27,7 +27,7 @@ Pouillard (who mainly made it). ...@@ -27,7 +27,7 @@ Pouillard (who mainly made it).
    -} -}
    {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ScopedTypeVariables #-}
    {-# LANGUAGE TypeOperators #-}
    module Gargantext.API module Gargantext.API
    where where
    ...@@ -43,6 +43,7 @@ import GHC.Generics (Generic) ...@@ -43,6 +43,7 @@ import GHC.Generics (Generic)
    import Gargantext.API.Admin.Auth.Types (AuthContext) import Gargantext.API.Admin.Auth.Types (AuthContext)
    import Gargantext.API.Admin.Settings (newEnv) import Gargantext.API.Admin.Settings (newEnv)
    import Gargantext.API.Admin.Types (FireWall(..), PortNumber, cookieSettings, jwtSettings, settings) import Gargantext.API.Admin.Types (FireWall(..), PortNumber, cookieSettings, jwtSettings, settings)
    import Gargantext.API.EKG
    import Gargantext.API.Ngrams (saveNodeStory) import Gargantext.API.Ngrams (saveNodeStory)
    import Gargantext.API.Prelude import Gargantext.API.Prelude
    import Gargantext.API.Routes import Gargantext.API.Routes
    ...@@ -54,11 +55,11 @@ import Network.Wai ...@@ -54,11 +55,11 @@ import Network.Wai
    import Network.Wai.Handler.Warp hiding (defaultSettings) import Network.Wai.Handler.Warp hiding (defaultSettings)
    import Network.Wai.Middleware.Cors import Network.Wai.Middleware.Cors
    import Network.Wai.Middleware.RequestLogger import Network.Wai.Middleware.RequestLogger
    import Paths_gargantext (getDataDir)
    import Servant import Servant
    import System.IO (FilePath) import System.FilePath
    data Mode = Dev | Mock | Prod data Mode = Dev | Mock | Prod
    deriving (Show, Read, Generic) deriving (Show, Read, Generic)
    -- | startGargantext takes as parameters port number and Ini file. -- | startGargantext takes as parameters port number and Ini file.
    ...@@ -191,8 +192,15 @@ serverGargAdminAPI = roots ...@@ -191,8 +192,15 @@ serverGargAdminAPI = roots
    --gargMock :: Server GargAPI --gargMock :: Server GargAPI
    --gargMock = mock apiGarg Proxy --gargMock = mock apiGarg Proxy
    --------------------------------------------------------------------- ---------------------------------------------------------------------
    makeApp :: EnvC env => env -> IO Application makeApp :: EnvC env => env -> IO Application
    makeApp env = serveWithContext api cfg <$> server env makeApp env = do
    serv <- server env
    (ekgStore, ekgMid) <- newEkgStore api
    ekgDir <- (</> "ekg-assets") <$> getDataDir
    return $ ekgMid $ serveWithContext apiWithEkg cfg
    (ekgServer ekgDir ekgStore :<|> serv)
    where where
    cfg :: Servant.Context AuthContext cfg :: Servant.Context AuthContext
    cfg = env ^. settings . jwtSettings cfg = env ^. settings . jwtSettings
    ...@@ -206,6 +214,9 @@ makeApp env = serveWithContext api cfg <$> server env ...@@ -206,6 +214,9 @@ makeApp env = serveWithContext api cfg <$> server env
    api :: Proxy API api :: Proxy API
    api = Proxy api = Proxy
    apiWithEkg :: Proxy (EkgAPI :<|> API)
    apiWithEkg = Proxy
    apiGarg :: Proxy GargAPI apiGarg :: Proxy GargAPI
    apiGarg = Proxy apiGarg = Proxy
    --------------------------------------------------------------------- ---------------------------------------------------------------------
    ......
    src/Gargantext/API/EKG.hs 0 → 100644
    View file @ 45c0817e
    {-# OPTIONS_GHC -fno-warn-orphans #-}
    {-# LANGUAGE TypeOperators #-}
    {-# LANGUAGE ScopedTypeVariables #-}
    module Gargantext.API.EKG where
    import Data.HashMap.Strict as HM
    import Data.Text as T
    import Data.Text.IO as T
    import Data.Time.Clock.POSIX (getPOSIXTime)
    import Network.Wai
    import Protolude
    import Servant
    import Servant.Auth
    import Servant.Ekg
    import System.Metrics
    import qualified System.Metrics.Json as J
    -- Mimics https://github.com/tibbe/ekg/blob/master/System/Remote/Snap.hs#L98
    type EkgAPI =
    "ekg" :>
    ( "api" :>
    ( Get '[JSON] J.Sample :<|>
    CaptureAll "segments" Text :> Get '[JSON] J.Value
    ) :<|>
    Raw
    )
    ekgServer :: FilePath -> Store -> Server EkgAPI
    ekgServer assetsDir store = (getAll :<|> getOne) :<|> serveDirectoryFileServer assetsDir
    where getAll = J.Sample <$> liftIO (sampleAll store)
    getOne segments = do
    let metric = T.intercalate "." segments
    metrics <- liftIO (sampleAll store)
    maybe (liftIO (T.putStrLn "not found boohoo") >> throwError err404) (return . J.Value) (HM.lookup metric metrics)
    newEkgStore :: HasEndpoint api => Proxy api -> IO (Store, Middleware)
    newEkgStore api = do
    s <- newStore
    registerGcMetrics s
    registerCounter "ekg.server_timestamp_ms" getTimeMs s -- used by UI
    mid <- monitorEndpoints api s
    return (s, mid)
    where getTimeMs = (round . (* 1000)) `fmap` getPOSIXTime
    instance HasEndpoint api => HasEndpoint (Auth xs a :> api) where
    getEndpoint _ = getEndpoint (Proxy :: Proxy api)
    enumerateEndpoints _ = enumerateEndpoints (Proxy :: Proxy api)
    src/Gargantext/Core/Text/Corpus/Parsers/FrameWrite.hs
    View file @ 45c0817e
    ...@@ -178,7 +178,7 @@ datePrefixP = do ...@@ -178,7 +178,7 @@ datePrefixP = do
    dateP :: Parser Date dateP :: Parser Date
    dateP = try datePrefixP dateP = try datePrefixP
    *> dateISOP *> dateISOP
    -- *> many (noneOf "\n") -- *> many (noneOf "\n")
    dateISOP :: Parser Date dateISOP :: Parser Date
    dateISOP = do dateISOP = do
    ......
    stack.yaml
    View file @ 45c0817e
    ...@@ -112,3 +112,4 @@ extra-deps: ...@@ -112,3 +112,4 @@ extra-deps:
    # need Vector.uncons # need Vector.uncons
    - vector-0.12.3.0@sha256:0ae2c1ba86f0077910be242ec6802cc3d7725fe7b2bea6987201aa3737b239b5,7953 - vector-0.12.3.0@sha256:0ae2c1ba86f0077910be242ec6802cc3d7725fe7b2bea6987201aa3737b239b5,7953
    - servant-ekg-0.3.1@sha256:19bd9dc3943983da8e79d6f607614c68faea4054fb889d508c8a2b67b6bdd448,2203
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment