conduit fetch added

epo-api-client.cabal

@@ -69,6 +69,7 @@ library
     -- Other library packages from which modules are imported.
     build-depends:    base ^>=4.14.3.0
                     , aeson ^>= 1.5.6.0
+                    , conduit ^>= 1.3.5
                     , containers ^>= 0.6.7
                     , http-client >= 0.7.13.1 && < 0.8
                     , http-client-tls >= 0.3.6.2 && < 0.4

src/EPO/API/Client/Implementation.hs

@@ -10,16 +10,24 @@ Portability : POSIX
 
 module EPO.API.Client.Implementation
   ( searchEPOAPI
+  , searchEPOAPIC
   )
 where
 
+import Conduit
 import EPO.API.Client
 import EPO.API.Client.Types
 import Network.URI qualified as URI
 import Protolude
-import Servant.Client (runClientM)
+import Servant.Client (ClientError, runClientM)
 
 
+batchSize :: Int
+batchSize = 20
+
+
+-- | This is a basic function for searching, don't use it for full
+-- search, it fetches whole page at once.
 searchEPOAPI :: URI.URI
              -> AuthKey
              -> Start
@@ -32,3 +40,29 @@ searchEPOAPI uri auth start end search' = do
   case eRes of
     Left err-> throwIO err
     Right res -> pure res
+
+
+
+searchEPOAPIC :: URI.URI
+              -> AuthKey
+              -> Maybe Start
+              -> Maybe End
+              -> Search
+              -> IO (Either ClientError (Integer, ConduitT () HyperdataDocument IO ()))
+searchEPOAPIC uri auth mStart mEnd search' = do
+  let start = fromMaybe 1 mStart
+  env <- defaultClientEnv uri
+  eRes1 <- runClientM (search (Just auth) (Just 1) (Just 2) (Just search')) env
+  case eRes1 of
+    Left err -> pure $ Left err
+    Right (Paginated { total }) -> do
+      let end = fromMaybe total mEnd
+      let fetch s = do
+            let endS = minimum [s + batchSize - 1, end]
+            eRes <- runClientM (search (Just auth) (Just s) (Just endS) (Just search')) env
+            case eRes of
+              Left err -> throwIO err
+              Right (Paginated { items }) -> pure items
+      pure $ Right (fromIntegral total
+                   , yieldMany [start, start + batchSize .. end]
+                     .| concatMapMC fetch)