Commit d97e2510 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[FIX] testing (WIP)

parent 9f5df649
Pipeline #1455 failed with stage
...@@ -39,7 +39,7 @@ import Gargantext.Database.Action.Metrics.NgramsByNode (getOccByNgramsOnlyFast') ...@@ -39,7 +39,7 @@ import Gargantext.Database.Action.Metrics.NgramsByNode (getOccByNgramsOnlyFast')
import Gargantext.Database.Admin.Types.Hyperdata.Document import Gargantext.Database.Admin.Types.Hyperdata.Document
import Gargantext.Database.Admin.Types.Node import Gargantext.Database.Admin.Types.Node
import Gargantext.Database.Query.Table.Ngrams (insertNgrams) import Gargantext.Database.Query.Table.Ngrams (insertNgrams)
import Gargantext.Database.Query.Table.Node (getDocumentsWithParentId) import Gargantext.Database.Query.Table.NodeNode (selectDocNodes)
import Gargantext.Database.Schema.Ngrams import Gargantext.Database.Schema.Ngrams
import Gargantext.Database.Schema.Node import Gargantext.Database.Schema.Node
import Gargantext.Database.Types (Indexed(..)) import Gargantext.Database.Types (Indexed(..))
...@@ -114,12 +114,16 @@ reIndexWith cId lId nt lts = do ...@@ -114,12 +114,16 @@ reIndexWith cId lId nt lts = do
<$> HashMap.toList <$> HashMap.toList
<$> getTermsWith identity [lId] nt lts <$> getTermsWith identity [lId] nt lts
-- printDebug "ts" ts --printDebug "ts" ts
-- Taking the ngrams with 0 occurrences only (orphans) -- Taking the ngrams with 0 occurrences only (orphans)
orphans <- HashMap.keys occs <- getOccByNgramsOnlyFast' cId lId nt ts
<$> HashMap.filter (==0)
<$> getOccByNgramsOnlyFast' cId lId nt ts let orphans = List.concat
$ map (\t -> case HashMap.lookup t occs of
Nothing -> [t]
Just n -> if n == 1 then [t] else [ ]
) ts
-- Getting the Id of orphan ngrams -- Getting the Id of orphan ngrams
mapTextNgramsId <- insertNgrams (map (text2ngrams . unNgramsTerm) orphans) mapTextNgramsId <- insertNgrams (map (text2ngrams . unNgramsTerm) orphans)
...@@ -127,7 +131,9 @@ reIndexWith cId lId nt lts = do ...@@ -127,7 +131,9 @@ reIndexWith cId lId nt lts = do
printDebug "orphans" orphans printDebug "orphans" orphans
-- Get all documents of the corpus -- Get all documents of the corpus
docs <- getDocumentsWithParentId cId docs <- selectDocNodes cId
printDebug "docs length" (List.length docs)
-- Checking Text documents where orphans match -- Checking Text documents where orphans match
-- TODO Tests here -- TODO Tests here
...@@ -143,6 +149,8 @@ reIndexWith cId lId nt lts = do ...@@ -143,6 +149,8 @@ reIndexWith cId lId nt lts = do
(List.cycle [Map.fromList $ [(nt, Map.singleton (doc ^. node_id) 1 )]]) (List.cycle [Map.fromList $ [(nt, Map.singleton (doc ^. node_id) 1 )]])
) docs ) docs
printDebug "ngramsByDoc" ngramsByDoc
-- Saving the indexation in database -- Saving the indexation in database
_ <- insertDocNgrams lId ( HashMap.fromList _ <- insertDocNgrams lId ( HashMap.fromList
$ catMaybes $ catMaybes
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment