Use fq for lang filter

Also, some refactoring for app (add debug etc).
parent 5edaca5c
...@@ -5,21 +5,24 @@ module Main where ...@@ -5,21 +5,24 @@ module Main where
import Conduit ( sinkList, mapM_C, (.|), runConduit ) import Conduit ( sinkList, mapM_C, (.|), runConduit )
import Data.LanguageCodes (ISO639_1(..)) import Data.LanguageCodes (ISO639_1(..))
import Data.Text qualified as T import Data.Text qualified as T
import HAL (getMetadataWith, getMetadataWithC, getMetadataWithCursorC) import HAL (getMetadataWithCursorOptsC, countResultsOpts', HalCrawlerOptions(..), defaultHalOptions)
import HAL.Doc import HAL.Doc
import HAL.Doc.Corpus (Corpus(..)) import HAL.Doc.Corpus (Corpus(..))
import Network.HTTP.Client (newManager) import Network.HTTP.Client (newManager)
import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Client.TLS (tlsManagerSettings)
import Options.Applicative import Options.Applicative
import Prelude qualified
import Protolude import Protolude
data CountParams = CountParams data CountParams = CountParams
{ cp_query :: [ T.Text ] } { cp_query :: T.Text
, cp_lang :: Maybe ISO639_1 }
data FetchParams = FetchParams data FetchParams = FetchParams
{ fp_query :: [ T.Text ] { fp_query :: T.Text
, fp_limit :: Integer } , fp_limit :: Integer
, fp_lang :: Maybe ISO639_1 }
data Command = data Command =
Count CountParams Count CountParams
...@@ -28,13 +31,18 @@ data Command = ...@@ -28,13 +31,18 @@ data Command =
countParams :: Parser Command countParams :: Parser Command
countParams = Count <$> countParams = Count <$>
(CountParams (CountParams
<$> many (strArgument (metavar "query"))) <$> strArgument (metavar "query")
<*> optional (option (maybeReader readLang) (long "lang")))
fetchParams :: Parser Command fetchParams :: Parser Command
fetchParams = Fetch <$> fetchParams = Fetch <$>
(FetchParams (FetchParams
<$> many (strArgument (metavar "query")) <$> strArgument (metavar "query")
<*> option auto (long "limit")) <*> option auto (long "limit")
<*> optional(option (maybeReader readLang) (long "lang")))
readLang :: Prelude.String -> Maybe ISO639_1
readLang = readMaybe
params :: Parser Command params :: Parser Command
params = subparser params = subparser
...@@ -55,13 +63,15 @@ main = run =<< execParser opts ...@@ -55,13 +63,15 @@ main = run =<< execParser opts
-- (Right val) -> print $ _docs val -- (Right val) -> print $ _docs val
run :: Command -> IO () run :: Command -> IO ()
run (Count (CountParams { cp_query })) = do run (Count (CountParams { cp_query, cp_lang })) = do
res <- getMetadataWithC cp_query (Just 0) Nothing Nothing res <- countResultsOpts' opts cp_query cp_lang
case res of case res of
Left err -> putText $ show err Left err -> putText $ show err
Right (cnt, _docsC) -> putText $ show cnt Right cnt -> putText $ show cnt
run (Fetch (FetchParams { fp_query, fp_limit })) = do where
res <- getMetadataWithCursorC fp_query (Just fp_limit) Nothing opts = defaultHalOptions { _hco_debugLogs = True }
run (Fetch (FetchParams { fp_query, fp_limit, fp_lang })) = do
res <- getMetadataWithCursorOptsC opts fp_query (Just fp_limit) fp_lang
case res of case res of
Left err -> putText $ show err Left err -> putText $ show err
Right (_cnt, docsC) -> do Right (_cnt, docsC) -> do
...@@ -71,6 +81,7 @@ run (Fetch (FetchParams { fp_query, fp_limit })) = do ...@@ -71,6 +81,7 @@ run (Fetch (FetchParams { fp_query, fp_limit })) = do
.| sinkList .| sinkList
pure () pure ()
where where
opts = defaultHalOptions { _hco_debugLogs = True }
printCorpus Corpus { .. } = do printCorpus Corpus { .. } = do
putText $ "docid: " <> _corpus_docid <> " [" <> (T.intercalate " " _corpus_title) <> "]" putText $ "docid: " <> _corpus_docid <> " [" <> (T.intercalate " " _corpus_title) <> "]"
putText $ " " <> (T.intercalate " " _corpus_abstract) putText $ " " <> (T.intercalate " " _corpus_abstract)
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment