[API] scroll implementation, first draft

README.md

@@ -33,3 +33,8 @@ main = do
       print $ take 5 val
 ```
  
+
+
+https://api.istex.fr/document/?output=author,title,abstract,publicationDate,refBibs&scroll=1m&q=coffee
+
+FatalError {fatalErrorMessage = "ConnectionError (HttpExceptionRequest Request {\n  host                 = \"api.istex.fr\"\n  port                 = 443\n  secure               = True\n  requestHeaders       = [(\"Accept\",\"application/json;charset=utf-8,application/json\")]\n  path                 = \"document\"\n  queryString          = \"?output=author,title,abstract,publicationDate,refBibs&scroll=1m&q=coffee\"\n  method               = \"GET\"\n  proxy                = Nothing\n  rawBody              = False\n  redirectCount        = 10\n  responseTimeout      = ResponseTimeoutDefault\n  requestVersion       = HTTP/1.1\n}\n ConnectionTimeout)"}

app/Main.hs

-{-# LANGUAGE OverloadedStrings #-}
-
 module Main where
 
 import ISTEX

crawlerISTEX.cabal

 cabal-version: 1.12
 
--- This file has been generated from package.yaml by hpack version 0.31.1.
+-- This file has been generated from package.yaml by hpack version 0.34.4.
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 1b6f6ece1304eb1586d6ae93969d55e01cea6c4a9bae285b84794dd960541b1b
+-- hash: 940816b6d9cd2940801674b8216a13ce8c7bb42b5951821a262fbc6086def8cc
 
 name:           crawlerISTEX
 version:        0.1.0.0
@@ -33,9 +33,17 @@ library
       Paths_crawlerISTEX
   hs-source-dirs:
       src
+  default-extensions:
+      DataKinds
+      DeriveGeneric
+      NamedFieldPuns
+      OverloadedStrings
+      RecordWildCards
+      TypeOperators
   build-depends:
       aeson
     , base >=4.7 && <5
+    , ghc
     , http-client
     , http-client-tls
     , lens
@@ -50,11 +58,19 @@ executable crawlerISTEX-exe
       Paths_crawlerISTEX
   hs-source-dirs:
       app
+  default-extensions:
+      DataKinds
+      DeriveGeneric
+      NamedFieldPuns
+      OverloadedStrings
+      RecordWildCards
+      TypeOperators
   ghc-options: -threaded -rtsopts -with-rtsopts=-N
   build-depends:
       aeson
     , base >=4.7 && <5
     , crawlerISTEX
+    , ghc
     , http-client
     , http-client-tls
     , lens
@@ -70,11 +86,19 @@ test-suite crawlerISTEX-test
       Paths_crawlerISTEX
   hs-source-dirs:
       test
+  default-extensions:
+      DataKinds
+      DeriveGeneric
+      NamedFieldPuns
+      OverloadedStrings
+      RecordWildCards
+      TypeOperators
   ghc-options: -threaded -rtsopts -with-rtsopts=-N
   build-depends:
       aeson
     , base >=4.7 && <5
     , crawlerISTEX
+    , ghc
     , http-client
     , http-client-tls
     , lens

package.yaml

@@ -22,6 +22,7 @@ description:         Please see the README on GitHub at <https://github.com/Muda
 dependencies:
 - aeson
 - base >= 4.7 && < 5
+- ghc  # Panic
 - text
 - lens
 - servant
@@ -29,6 +30,14 @@ dependencies:
 - http-client
 - http-client-tls
 
+default-extensions:
+  - DataKinds
+  - DeriveGeneric
+  - NamedFieldPuns
+  - OverloadedStrings
+  - RecordWildCards
+  - TypeOperators
+
 library:
   source-dirs: src
 

src/ISTEX.hs

-{-# LANGUAGE OverloadedStrings #-}
-
 module ISTEX where
 
 import ISTEX.Client
@@ -22,6 +20,14 @@ getMetadataWith q n = do
     n
     (Just q)
 
+getMetadataScrollWith :: Text -> Maybe Text -> IO (Either ClientError Documents)
+getMetadataScrollWith q n = do
+  runIstexAPIClient $
+    searchScroll
+    (Just "author,title,abstract,publicationDate,refBibs")
+    n
+    (Just q)
+
 type Query = Text
 
 runTest :: Query -> Maybe Int -> IO ()

src/ISTEX/Client.hs

-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE TypeOperators #-}
-{-# LANGUAGE DataKinds #-}
-{-# LANGUAGE DeriveGeneric #-}
 {-# LANGUAGE TemplateHaskell #-}
 
 module ISTEX.Client where
 
-import GHC.Generics
-import Data.Aeson
 import Control.Applicative ((<|>))
+import Data.Aeson
+import GHC.Generics
+import Panic (panic)
+import Text.Read (readEither)
 
 import Servant.API
 import Servant.Client
@@ -34,12 +32,23 @@ data Source = Source
   } deriving (Show, Generic)
 L.makeLenses ''Source
 
+parsePubDate :: Maybe T.Text -> Maybe Int
+parsePubDate Nothing = Nothing
+parsePubDate (Just pubDate) = 
+  case ePubDate of
+    Left err -> panic $ T.unpack $ "[parseJSON] Cannot parse publicationDate (Int): " <> pubDate <> "(" <> (T.pack err) <> ")"
+    Right pubDate' -> Just pubDate'
+  where
+    ePubDate = readEither (T.unpack pubDate) :: Either String Int
+
 instance FromJSON Source where
-  parseJSON (Object o) =
-    Source <$> (o .:? "title")
-           <*> (o .: "author")
-           <*> do pPubDate <- (o .:? "publicationDate")
-                  return $ (read . T.unpack) <$> pPubDate
+  parseJSON (Object o) = do
+    _source_title <- o .:? "title"
+    _source_authors <- o .: "author"
+    mPubDate <- o .:? "publicationDate"
+
+    pure $ Source { _source_publicationDate = parsePubDate mPubDate
+                  , .. }
 
 data Document = Document
   { _document_id              :: T.Text
@@ -52,14 +61,16 @@ data Document = Document
 L.makeLenses ''Document
 
 instance FromJSON Document where
-  parseJSON (Object o) =
-    Document <$> (o .: "id")
-             <*> (o .:? "title")
-             <*> (o .: "author" <|> pure [])
-             <*> (o .:? "abstract")
-             <*> ((o .:? "publicationDate") >>= \date -> return $ fmap (read . T.unpack) date)
-             <*> (o .: "refBibs" <|> pure [])
-
+  parseJSON (Object o) = do
+    _document_id <- o .: "id"
+    _document_title <- o .:? "title"
+    _document_authors <- o .: "author" <|> pure []
+    _document_abstract <- o .:? "abstract"
+    _document_sources <- o .: "refBibs" <|> pure []
+    mPubDate <- o .:? "publicationDate"
+    
+    pure $ Document { _document_publicationDate = parsePubDate mPubDate
+                    , .. }
 data Documents = Documents
   { _documents_total :: Int
   , _documents_hits  :: [Document]
@@ -82,3 +93,14 @@ istexProxy = Proxy
 
 search :: Maybe T.Text -> Maybe Int -> Maybe T.Text -> ClientM Documents
 search = client istexProxy
+
+type SearchScroll = QueryParam "output" T.Text
+                 :> QueryParam "scroll" T.Text
+                 :> QueryParam "q" T.Text
+                 :> Get '[JSON] Documents
+
+istexProxyScroll :: Proxy SearchScroll
+istexProxyScroll = Proxy
+
+searchScroll :: Maybe T.Text -> Maybe T.Text -> Maybe T.Text -> ClientM Documents
+searchScroll = client istexProxyScroll

stack.yaml

@@ -38,6 +38,7 @@ packages:
 # using the same syntax as the packages field.
 # (e.g., acme-missiles-0.3)
 # extra-deps: []
+extra-deps: []
 
 # Override default flag values for local packages and extra-deps
 # flags: {}