Merge branch 'master' into refactoring

app/Main.hs

@@ -3,20 +3,80 @@
 
 module Main where
 
+import Conduit
 import Data.LanguageCodes (ISO639_1(..))
 import Data.Text qualified as T
 import HAL
 import HAL (getMetadataWith)
 import HAL.Client
 import HAL.Doc
+import HAL.Doc.Corpus (Corpus(..))
 import NeatInterpolation (text)
 import Network.HTTP.Client (newManager)
 import Network.HTTP.Client.TLS (tlsManagerSettings)
+import Options.Applicative
 import Protolude
 import Servant.Client
 import Tree
 
 
+data CountParams = CountParams
+  { cp_query :: T.Text }
+
+data FetchParams = FetchParams
+  { fp_query :: T.Text }
+
+data Command =
+    Count CountParams
+  | Fetch FetchParams
+
+countParams :: Parser Command
+countParams = Count <$>
+  (CountParams
+  <$> strArgument (metavar "query"))
+
+fetchParams :: Parser Command
+fetchParams = Fetch <$>
+  (FetchParams
+  <$> strArgument (metavar "query"))
+
+params :: Parser Command
+params = subparser
+  (command "count" (info countParams (progDesc "Count number of docs for a given query"))
+  <> command "fetch" (info fetchParams (progDesc "Fetch docs for a given query")))
+
+opts :: ParserInfo Command
+opts = info (params <**> helper)
+       (fullDesc
+       <> progDesc "A program to test HAL"
+       <> header "crawlerHAL-exe")
+
+main :: IO ()
+main = run =<< execParser opts
+  -- res <- getMetadataWith (generateRequestByStructID "artificial intelligence" imt) (Just 0) (Just 55)
+  -- case res of
+  --   (Left  err) -> print err
+  --   (Right val) -> print $ _docs val
+
+run :: Command -> IO ()
+run (Count (CountParams { cp_query })) = do
+  res <- getMetadataWithC (cp_query) (Just 0) Nothing Nothing
+  case res of
+    Left err -> putText $ show err
+    Right (cnt, _docsC) -> putText $ show cnt
+run (Fetch (FetchParams { fp_query })) = do
+  res <- getMetadataWithC (fp_query) (Just 0) Nothing Nothing
+  case res of
+    Left err -> putText $ show err
+    Right (_cnt, docsC) -> do
+      _ <- runConduit $
+           docsC
+        .| mapM_C (\(Corpus { .. }) -> putText $ "docid: " <> show _corpus_docid)
+        .| sinkList
+      pure ()
+
+
+-- data
 yearReq = [text|
 (language_t:en)
 AND (producedDateY_i:2018
@@ -61,11 +121,3 @@ imt = [
   ,"6279"
   ,"29212"
   ]
-
-main :: IO ()
-main = do
-  -- res <- getMetadataWith (generateRequestByStructID "artificial intelligence" imt) (Just 0) (Just 55)
-  res <- getMetadataWith (generateRequestByStructID "artificial intelligence" imt) (Just 0) (Just 55) (Just EN)
-  case res of
-    (Left  err) -> print err
-    (Right val) -> print $ _docs val

crawlerHAL.cabal

 cabal-version: 1.12
 
--- This file has been generated from package.yaml by hpack version 0.34.4.
+-- This file has been generated from package.yaml by hpack version 0.35.0.
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 5e660f75d650f1d72f17df082838eb0dc3d6bdd2f514b9931a97ddc17a47b5c0
+-- hash: d5f6b9788433a3873efd55684dfd55f4d85380f24209a6a8eb8ba80c82a330f8
 
 name:           crawlerHAL
 version:        0.1.0.0
@@ -58,6 +58,7 @@ library
     , iso639 >= 0.1.0.3 && < 0.2
     , lens >= 5.2.2 && < 5.3
     , neat-interpolation >= 0.5.1.3 && < 0.6
+    , optparse-applicative >= 0.18.1.0 && < 0.19
     , protolude >= 0.3.3 && < 0.4
     , scientific >= 0.3.7.0 && < 0.4
     , servant >= 0.19 && < 0.21
@@ -98,6 +99,7 @@ executable crawlerHAL-exe
     , iso639 >= 0.1.0.3 && < 0.2
     , lens >= 5.2.2 && < 5.3
     , neat-interpolation >= 0.5.1.3 && < 0.6
+    , optparse-applicative >= 0.18.1.0 && < 0.19
     , protolude >= 0.3.3 && < 0.4
     , scientific >= 0.3.7.0 && < 0.4
     , servant >= 0.19 && < 0.21
@@ -138,6 +140,7 @@ test-suite halCrawler-test
     , iso639 >= 0.1.0.3 && < 0.2
     , lens >= 5.2.2 && < 5.3
     , neat-interpolation >= 0.5.1.3 && < 0.6
+    , optparse-applicative >= 0.18.1.0 && < 0.19
     , protolude >= 0.3.3 && < 0.4
     , scientific >= 0.3.7.0 && < 0.4
     , servant >= 0.19 && < 0.21

src/HAL/Client.hs

@@ -22,7 +22,7 @@ type Search doc = "search"
   :> QueryParams "fq" Text
   -- pretty much clear, (Asc || Desc) + field you want to sort by
   :> QueryParam "sort" SortField
-  -- permit to start a the x result
+  -- permit to start at the x result
   :> QueryParam "start" Int
   -- use rows to make the request only return the x number of result
   :> QueryParam "rows" Integer

src/HAL/Doc/Corpus.hs

@@ -10,20 +10,19 @@ import Protolude
 import Servant.API (ToHttpApiData(..))
 
 data Corpus = Corpus
-  {
-    _corpus_docid :: Text,
-    _corpus_title :: [Text],
-    _corpus_abstract :: [Text],
-    _corpus_date :: Maybe Text,
-    _corpus_source :: Maybe Text,
-    _corpus_authors_names :: [Text],
-    _corpus_authors_affiliations :: [Text],
-    _corpus_struct_id :: [Int]
+  { _corpus_docid         :: Text
+  , _corpus_title         :: [Text]
+  , _corpus_abstract      :: [Text]
+  , _corpus_date          :: Maybe Text
+  , _corpus_source        :: Maybe Text
+  , _corpus_authors_names :: [Text]
+  , _corpus_authors_affiliations :: [Text]
+  , _corpus_struct_id     :: [Int]
   } deriving (Show, Generic)
 L.makeLenses ''Corpus
 
 instance Default Corpus where
-  def = Corpus "" def def def def def def def
+  def = Corpus "default Id" def def def def def def def
 
 instance FromJSON Corpus where
   parseJSON = withObject "Corpus" $