Skip to content

haskell-gargantext
haskell-gargantext
Commit 45c0817e authored by Alp Mestanogullari's avatar Alp Mestanogullari
Browse files
Options

introduce EKG-based monitoring infrastructure

parent f26f41e8
Pipeline #2027 failed with stage
    in 10 minutes and 11 seconds
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing with 15139 additions and 8 deletions
    cabal.project
    View file @ 45c0817e
    ......@@ -13,6 +13,11 @@ source-repository-package
    location: https://gitlab.iscpif.fr/gargantext/patches-class.git
    tag: 271ba32d6c940029dc653354dd7974a819f48e77
    source-repository-package
    type: git
    location: https://gitlab.iscpif.fr/cgenie/haskell-gargantext-prelude.git
    tag: 35b09629a658fc16cc9ff63e7591e58511cd98a7
    -- External Data API connectors
    source-repository-package
    type: git
    ......@@ -76,7 +81,7 @@ source-repository-package
    source-repository-package
    type: git
    location: https://github.com/delanoe/haskell-opaleye.git
    tag: 63ee65d974e9d20eaaf17a2e83652175988cbb79
    tag: d3ab7acd5ede737478763630035aa880f7e34444
    source-repository-package
    type: git
    ......@@ -100,4 +105,4 @@ source-repository-package
    location: https://gitlab.iscpif.fr/anoe/accelerate-utility.git
    tag: 83ada76e78ac10d9559af8ed6bd4064ec81308e4
    constraints: unordered-containers==0.2.13.*
    \ No newline at end of file
    constraints: unordered-containers==0.2.14.*
    \ No newline at end of file
    ekg-assets/bootstrap-1.4.0.css 0 → 100755
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/bootstrap-1.4.0.min.css 0 → 100755
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/index.html 0 → 100644
    View file @ 45c0817e
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset="utf-8">
    <link rel="stylesheet" href="bootstrap-1.4.0.min.css">
    <link rel="stylesheet" href="monitor.css" type="text/css">
    <script type="text/javascript" src="jquery-1.6.4.min.js"></script>
    <script type="text/javascript" src="jquery.flot.min.js"></script>
    <title>ekg</title>
    </head>
    <body>
    <div class="topbar">
    <div class="topbar-inner">
    <div class="container-fluid">
    <span class="brand">ekg</span>
    <p class="pull-right">Polling interval:
    <select id="updateInterval" class="small">
    <option value="100">100 ms</option>
    <option value="200">200 ms</option>
    <option value="500">500 ms</option>
    <option value="1000" selected="selected">1 s</option>
    <option value="2000">2 s</option>
    <option value="5000">5 s</option>
    <option value="10000">10 s</option>
    </select> |
    <button id="pause-ui" class="btn">Pause UI</button>
    </p>
    </div>
    </div>
    </div>
    <div class="container">
    <div class="row">
    <div class="alert-message error fade in hide" data-alert="alert">
    <p>Lost connection to server.</p>
    </div>
    </div>
    <div class="row">
    <div id="plots" class="span11">
    <div id="current-bytes-used-plot" class="plot-container">
    <h3>Current residency</h3>
    <div class="plot"></div>
    </div>
    <div id="allocation-rate-plot" class="plot-container">
    <h3>Allocation rate</h3>
    <div class="plot"></div>
    </div>
    <div id="productivity-plot" class="plot-container">
    <h3>Productivity</h3>
    <div class="plot"></div>
    </div>
    </div>
    <div class="span5">
    <h3>GC and memory statistics</h3>
    <table class="condensed-table">
    <thead>
    <tr>
    <th>Statistic</th>
    <th>Value</th>
    </tr>
    </thead>
    <tbody>
    <tr>
    <td>Maximum residency</td>
    <td id="max-bytes-used" class="span3 value">0</td>
    </tr>
    <tr>
    <td>Current residency</td>
    <td id="current-bytes-used" class="value">0</td>
    </tr>
    <tr>
    <td>Maximum slop</td>
    <td id="max-bytes-slop" class="value">0</td>
    </tr>
    <tr>
    <td>Current slop</td>
    <td id="current-bytes-slop" class="value">0</td>
    </tr>
    <tr>
    <td>Productivity (wall clock time)</td>
    <td id="productivity-wall" class="value">0</td>
    </tr>
    <tr>
    <td>Productivity (cpu time)</td>
    <td id="productivity-cpu" class="value">0</td>
    </tr>
    <tr>
    <td>Allocation rate</td>
    <td id="allocation-rate" class="value">0</td>
    </tr>
    </tbody>
    </table>
    <h3>Metrics</h3>
    <table id="metric-table" class="condensed-table">
    <thead>
    <tr>
    <th class="span4">Name</th>
    <th class="span1">Value</th>
    </tr>
    </thead>
    <tbody>
    </tbody>
    </table>
    </div>
    </div>
    </div>
    <script type="text/javascript" src="monitor.js"></script>
    </body>
    </html>
    ekg-assets/jquery-1.6.4.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery-1.6.4.min.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery.flot.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/jquery.flot.min.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    ekg-assets/monitor.css 0 → 100644
    View file @ 45c0817e
    /**
    * Blueprint/flot compatibility
    *
    * Resets some styles back to the browser default.
    */
    .plot table {
    width: auto;
    border-spacing: 2px;
    }
    .plot th,
    .plot td,
    .plot caption {
    padding: 0;
    }
    /**
    * Body margin
    */
    body {
    padding-top: 60px;
    }
    /**
    * Plots
    */
    .plot {
    width: 600px;
    height: 300px;
    margin-bottom: 1.5em;
    }
    .close-button {
    float: right;
    cursor: pointer;
    }
    /**
    * Table
    */
    .value {
    text-align: right;
    }
    .string {
    text-align: left;
    }
    .graph-button {
    cursor: pointer;
    vertical-align: middle;
    }
    ekg-assets/monitor.js 0 → 100644
    View file @ 45c0817e
    This diff is collapsed.
    package.yaml
    View file @ 45c0817e
    ......@@ -25,6 +25,15 @@ default-extensions:
    - OverloadedStrings
    - RankNTypes
    - RecordWildCards
    data-files:
    - ekg-assets/index.html
    - ekg-assets/monitor.js
    - ekg-assets/monitor.css
    - ekg-assets/jquery.flot.min.js
    - ekg-assets/jquery-1.6.4.min.js
    - ekg-assets/bootstrap-1.4.0.min.css
    - ekg-assets/chart_line_add.png
    - ekg-assets/cross.png
    library:
    source-dirs: src
    ghc-options:
    ......@@ -137,6 +146,8 @@ library:
    - deepseq
    - directory
    - duckling
    - ekg-core
    - ekg-json
    - exceptions
    - fast-logger
    - fclabels
    ......@@ -206,6 +217,7 @@ library:
    - servant-blaze
    - servant-cassava
    - servant-client
    - servant-ekg
    - servant-job
    - servant-mock
    - servant-multipart
    ......@@ -257,6 +269,7 @@ executables:
    - -rtsopts
    - -threaded
    - -with-rtsopts=-N
    - -with-rtsopts=-T
    - -fprof-auto
    dependencies:
    - base
    ......
    src/Gargantext/API.hs
    View file @ 45c0817e
    ......@@ -27,7 +27,7 @@ Pouillard (who mainly made it).
    -}
    {-# LANGUAGE ScopedTypeVariables #-}
    {-# LANGUAGE TypeOperators #-}
    module Gargantext.API
    where
    ......@@ -43,6 +43,7 @@ import GHC.Generics (Generic)
    import Gargantext.API.Admin.Auth.Types (AuthContext)
    import Gargantext.API.Admin.Settings (newEnv)
    import Gargantext.API.Admin.Types (FireWall(..), PortNumber, cookieSettings, jwtSettings, settings)
    import Gargantext.API.EKG
    import Gargantext.API.Ngrams (saveNodeStory)
    import Gargantext.API.Prelude
    import Gargantext.API.Routes
    ......@@ -54,11 +55,11 @@ import Network.Wai
    import Network.Wai.Handler.Warp hiding (defaultSettings)
    import Network.Wai.Middleware.Cors
    import Network.Wai.Middleware.RequestLogger
    import Paths_gargantext (getDataDir)
    import Servant
    import System.IO (FilePath)
    import System.FilePath
    data Mode = Dev | Mock | Prod
    data Mode = Dev | Mock | Prod
    deriving (Show, Read, Generic)
    -- | startGargantext takes as parameters port number and Ini file.
    ......@@ -191,8 +192,15 @@ serverGargAdminAPI = roots
    --gargMock :: Server GargAPI
    --gargMock = mock apiGarg Proxy
    ---------------------------------------------------------------------
    makeApp :: EnvC env => env -> IO Application
    makeApp env = serveWithContext api cfg <$> server env
    makeApp env = do
    serv <- server env
    (ekgStore, ekgMid) <- newEkgStore api
    ekgDir <- (</> "ekg-assets") <$> getDataDir
    return $ ekgMid $ serveWithContext apiWithEkg cfg
    (ekgServer ekgDir ekgStore :<|> serv)
    where
    cfg :: Servant.Context AuthContext
    cfg = env ^. settings . jwtSettings
    ......@@ -206,6 +214,9 @@ makeApp env = serveWithContext api cfg <$> server env
    api :: Proxy API
    api = Proxy
    apiWithEkg :: Proxy (EkgAPI :<|> API)
    apiWithEkg = Proxy
    apiGarg :: Proxy GargAPI
    apiGarg = Proxy
    ---------------------------------------------------------------------
    ......
    src/Gargantext/API/EKG.hs 0 → 100644
    View file @ 45c0817e
    {-# OPTIONS_GHC -fno-warn-orphans #-}
    {-# LANGUAGE TypeOperators #-}
    {-# LANGUAGE ScopedTypeVariables #-}
    module Gargantext.API.EKG where
    import Data.HashMap.Strict as HM
    import Data.Text as T
    import Data.Text.IO as T
    import Data.Time.Clock.POSIX (getPOSIXTime)
    import Network.Wai
    import Protolude
    import Servant
    import Servant.Auth
    import Servant.Ekg
    import System.Metrics
    import qualified System.Metrics.Json as J
    -- Mimics https://github.com/tibbe/ekg/blob/master/System/Remote/Snap.hs#L98
    type EkgAPI =
    "ekg" :>
    ( "api" :>
    ( Get '[JSON] J.Sample :<|>
    CaptureAll "segments" Text :> Get '[JSON] J.Value
    ) :<|>
    Raw
    )
    ekgServer :: FilePath -> Store -> Server EkgAPI
    ekgServer assetsDir store = (getAll :<|> getOne) :<|> serveDirectoryFileServer assetsDir
    where getAll = J.Sample <$> liftIO (sampleAll store)
    getOne segments = do
    let metric = T.intercalate "." segments
    metrics <- liftIO (sampleAll store)
    maybe (liftIO (T.putStrLn "not found boohoo") >> throwError err404) (return . J.Value) (HM.lookup metric metrics)
    newEkgStore :: HasEndpoint api => Proxy api -> IO (Store, Middleware)
    newEkgStore api = do
    s <- newStore
    registerGcMetrics s
    registerCounter "ekg.server_timestamp_ms" getTimeMs s -- used by UI
    mid <- monitorEndpoints api s
    return (s, mid)
    where getTimeMs = (round . (* 1000)) `fmap` getPOSIXTime
    instance HasEndpoint api => HasEndpoint (Auth xs a :> api) where
    getEndpoint _ = getEndpoint (Proxy :: Proxy api)
    enumerateEndpoints _ = enumerateEndpoints (Proxy :: Proxy api)
    src/Gargantext/Core/Text/Corpus/Parsers/FrameWrite.hs
    View file @ 45c0817e
    ......@@ -178,7 +178,7 @@ datePrefixP = do
    dateP :: Parser Date
    dateP = try datePrefixP
    *> dateISOP
    -- *> many (noneOf "\n")
    -- *> many (noneOf "\n")
    dateISOP :: Parser Date
    dateISOP = do
    ......
    stack.yaml
    View file @ 45c0817e
    ......@@ -112,3 +112,4 @@ extra-deps:
    # need Vector.uncons
    - vector-0.12.3.0@sha256:0ae2c1ba86f0077910be242ec6802cc3d7725fe7b2bea6987201aa3737b239b5,7953
    - servant-ekg-0.3.1@sha256:19bd9dc3943983da8e79d6f607614c68faea4054fb889d508c8a2b67b6bdd448,2203
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment