Commit 0e384787 authored by Przemyslaw Kaminski's avatar Przemyslaw Kaminski

[searx] parser works now

One issue is that it creates new list/docs/board/graph nodes.
parent fc4a45c7
Pipeline #2235 failed with stage
in 10 minutes and 25 seconds
...@@ -142,7 +142,7 @@ triggerSearxSearch user cId q l logStatus = do ...@@ -142,7 +142,7 @@ triggerSearxSearch user cId q l logStatus = do
, _fsp_query = q , _fsp_query = q
, _fsp_url = surl } , _fsp_url = surl }
printDebug "[triggerSearxSearch] res" res --printDebug "[triggerSearxSearch] res" res
case res of case res of
Left _ -> pure () Left _ -> pure ()
...@@ -152,7 +152,10 @@ triggerSearxSearch user cId q l logStatus = do ...@@ -152,7 +152,10 @@ triggerSearxSearch user cId q l logStatus = do
-- docs :: [Either Text HyperdataDocument] -- docs :: [Either Text HyperdataDocument]
let docs' = catMaybes $ rightToMaybe <$> docs let docs' = catMaybes $ rightToMaybe <$> docs
Prelude.mapM_ (\(HyperdataDocument { _hd_title, _hd_publication_year, _hd_publication_date }) -> do Prelude.mapM_ (\(HyperdataDocument { _hd_title, _hd_publication_year, _hd_publication_date }) -> do
printDebug "[triggerSearxSearch] doc time" $ (show _hd_title) <> " :: " <> (show _hd_publication_year) <> " :: " <> (show _hd_publication_date) printDebug "[triggerSearxSearch] doc time" $
"[title] " <> (show _hd_title) <>
" :: [publication_year] " <> (show _hd_publication_year) <>
" :: [publication_date] " <> (show _hd_publication_date)
) docs' ) docs'
_ <- flowDataText user (DataNew [docs']) (Multi EN) cId Nothing logStatus _ <- flowDataText user (DataNew [docs']) (Multi EN) cId Nothing logStatus
pure () pure ()
...@@ -161,7 +164,7 @@ triggerSearxSearch user cId q l logStatus = do ...@@ -161,7 +164,7 @@ triggerSearxSearch user cId q l logStatus = do
hyperdataDocumentFromSearxResult :: SearxResult -> Either T.Text HyperdataDocument hyperdataDocumentFromSearxResult :: SearxResult -> Either T.Text HyperdataDocument
hyperdataDocumentFromSearxResult (SearxResult { _sr_content, _sr_engine, _sr_pubdate, _sr_title }) = do hyperdataDocumentFromSearxResult (SearxResult { _sr_content, _sr_engine, _sr_pubdate, _sr_title }) = do
let mDate = parseTimeM False defaultTimeLocale "%Y-%m-%d %H:%M:%S" (T.unpack _sr_pubdate) :: Maybe Day let mDate = parseTimeM False defaultTimeLocale "%Y-%m-%d %H:%M:%S+0000" (T.unpack _sr_pubdate) :: Maybe Day
let mGregorian = toGregorian <$> mDate let mGregorian = toGregorian <$> mDate
Right HyperdataDocument { _hd_bdd = Just "Searx" Right HyperdataDocument { _hd_bdd = Just "Searx"
, _hd_doi = Nothing , _hd_doi = Nothing
...@@ -169,7 +172,7 @@ hyperdataDocumentFromSearxResult (SearxResult { _sr_content, _sr_engine, _sr_pub ...@@ -169,7 +172,7 @@ hyperdataDocumentFromSearxResult (SearxResult { _sr_content, _sr_engine, _sr_pub
, _hd_uniqId = Nothing , _hd_uniqId = Nothing
, _hd_uniqIdBdd = Nothing , _hd_uniqIdBdd = Nothing
, _hd_page = Nothing , _hd_page = Nothing
, _hd_title = Just _sr_title , _hd_title = Just $ ("[" <> _sr_pubdate <> "] ") <> _sr_title
, _hd_authors = Nothing , _hd_authors = Nothing
, _hd_institutes = Nothing , _hd_institutes = Nothing
, _hd_source = Just _sr_engine , _hd_source = Just _sr_engine
......
...@@ -279,7 +279,7 @@ instance ToNode HyperdataDocument where ...@@ -279,7 +279,7 @@ instance ToNode HyperdataDocument where
where where
n = maybe "No Title" (DT.take 255) (_hd_title h) n = maybe "No Title" (DT.take 255) (_hd_title h)
date = jour y m d date = jour y m d
y = maybe 0 fromIntegral $ _hd_publication_year h y = maybe 1 fromIntegral $ _hd_publication_year h
m = fromMaybe 1 $ _hd_publication_month h m = fromMaybe 1 $ _hd_publication_month h
d = fromMaybe 1 $ _hd_publication_day h d = fromMaybe 1 $ _hd_publication_day h
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment