From b56988d88e8025152b6749932f6e2e817d80463f Mon Sep 17 00:00:00 2001 From: Nicolas Pouillard <nicolas.pouillard@gmail.com> Date: Mon, 11 Jun 2018 16:26:20 +0200 Subject: [PATCH] Fix performance issue due to infinit result of splitEvery --- app/CleanCsvCorpus.hs | 10 ++++------ src/Gargantext/Prelude.hs | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/app/CleanCsvCorpus.hs b/app/CleanCsvCorpus.hs index 3f566a8b..f8fe9cbd 100644 --- a/app/CleanCsvCorpus.hs +++ b/app/CleanCsvCorpus.hs @@ -42,18 +42,16 @@ main = do let q = ["gratuit", "gratuité", "culture", "culturel"] (h,csvDocs) <- readCsv rPath - + putStrLn $ "Number of documents before:" <> show (V.length csvDocs) putStrLn $ "Mean size of docs:" <> show ( docsSize csvDocs) - + let docs = toDocs csvDocs let engine = insertDocs docs initialDocSearchEngine let docIds = S.query engine (map pack q) let docs' = fromDocs $ filterDocs docIds (V.fromList docs) - + putStrLn $ "Number of documents after:" <> show (V.length docs') putStrLn $ "Mean size of docs:" <> show (docsSize docs') - - writeCsv wPath (h, docs') - + writeCsv wPath (h, docs') diff --git a/src/Gargantext/Prelude.hs b/src/Gargantext/Prelude.hs index 6b12fdd6..0b92098a 100644 --- a/src/Gargantext/Prelude.hs +++ b/src/Gargantext/Prelude.hs @@ -109,7 +109,7 @@ ma = movingAverage 3 -- | splitEvery n == chunkAlong n n splitEvery :: Int -> [a] -> [[a]] -splitEvery _ [] = L.cycle [[]] +splitEvery _ [] = [] splitEvery n xs = let (h,t) = L.splitAt n xs in h : splitEvery n t -- 2.21.0