After merge fixes

parent 47951d0a
Pipeline #7431 failed with stages
in 14 minutes and 7 seconds
...@@ -419,7 +419,6 @@ insertMasterDocs ncs c lang hs = do ...@@ -419,7 +419,6 @@ insertMasterDocs ncs c lang hs = do
-- this will enable global database monitoring -- this will enable global database monitoring
-- maps :: IO Map Ngrams (Map NgramsType (Map NodeId Int)) -- maps :: IO Map Ngrams (Map NgramsType (Map NodeId Int))
mapNgramsDocs' :: HashMap.HashMap ExtractedNgrams (Map NgramsType (Map NodeId (Int, TermsCount)))
mapNgramsDocs' :: HashMap.HashMap ExtractedNgrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount))) mapNgramsDocs' :: HashMap.HashMap ExtractedNgrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount)))
<- mapNodeIdNgrams <- mapNodeIdNgrams
<$> documentIdWithNgrams <$> documentIdWithNgrams
...@@ -437,9 +436,6 @@ saveDocNgramsWith :: (IsDBCmd env err m) ...@@ -437,9 +436,6 @@ saveDocNgramsWith :: (IsDBCmd env err m)
-> HashMap.HashMap ExtractedNgrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount))) -> HashMap.HashMap ExtractedNgrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount)))
-> m () -> m ()
saveDocNgramsWith lId mapNgramsDocs' = do saveDocNgramsWith lId mapNgramsDocs' = do
--printDebug "[saveDocNgramsWith] mapNgramsDocs'" mapNgramsDocs'
let mapNgramsDocsNoCount = over (traverse . traverse . traverse) fst mapNgramsDocs'
terms2id <- insertExtractedNgrams $ HashMap.keys mapNgramsDocsNoCount
(terms2id :: HashMap.HashMap Text NgramsId) <- insertExtractedNgrams $ HashMap.keys mapNgramsDocs' (terms2id :: HashMap.HashMap Text NgramsId) <- insertExtractedNgrams $ HashMap.keys mapNgramsDocs'
let mapNgramsDocs :: HashMap.HashMap Ngrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount))) let mapNgramsDocs :: HashMap.HashMap Ngrams (Map NgramsType (Map NodeId (TermsWeight, TermsCount)))
...@@ -490,13 +486,6 @@ reIndexWith cId lId nt lts = do ...@@ -490,13 +486,6 @@ reIndexWith cId lId nt lts = do
-- Get all documents of the corpus -- Get all documents of the corpus
(docs :: [ContextOnlyId HyperdataDocument]) <- selectDocNodesOnlyId cId (docs :: [ContextOnlyId HyperdataDocument]) <- selectDocNodesOnlyId cId
docs <- selectDocNodes cId
let
-- fromListWith (<>)
ngramsByDoc = map (HashMap.fromListWith (Map.unionWith (Map.unionWith (\(_a,b) (_a',b') -> (1,b+b')))))
$ map (map (\((k, cnt), v) -> (SimpleNgrams (text2ngrams k), over (traverse . traverse) (\p -> (p, cnt)) v)))
$ map (docNgrams corpusLang nt ts) docs
-- Saving the indexation in database -- Saving the indexation in database
mapM_ (saveDocNgramsWith lId . ngramsByDoc corpusLang nt ts) docs mapM_ (saveDocNgramsWith lId . ngramsByDoc corpusLang nt ts) docs
......
...@@ -25,7 +25,6 @@ import Control.Concurrent (threadDelay) ...@@ -25,7 +25,6 @@ import Control.Concurrent (threadDelay)
import Control.Lens ((^.)) import Control.Lens ((^.))
import Control.Monad (void) import Control.Monad (void)
import Control.Monad.STM (atomically) import Control.Monad.STM (atomically)
import Control.Monad (void)
import Data.Aeson qualified as Aeson import Data.Aeson qualified as Aeson
import Data.ByteString qualified as BS import Data.ByteString qualified as BS
import Data.Text.Encoding qualified as TE import Data.Text.Encoding qualified as TE
...@@ -50,7 +49,7 @@ import Test.Hspec.Wai.Internal (withApplication) ...@@ -50,7 +49,7 @@ import Test.Hspec.Wai.Internal (withApplication)
import Test.Instances () import Test.Instances ()
import Test.Utils (waitForTChanValue, waitForTSem) import Test.Utils (waitForTChanValue, waitForTSem)
import Test.Utils.Notifications (withAsyncWSConnection) import Test.Utils.Notifications (withAsyncWSConnection)
import Test.Utils (protected, waitForTChanValue, waitForTSem, withValidLoginA) import Test.Utils (protected, withValidLoginA)
import Text.RawString.QQ (r) import Text.RawString.QQ (r)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment