Commit c092568a authored by Alexandre Delanoë's avatar Alexandre Delanoë

[PARAM] List cut of Candidate Ngrams

parent 336815f4
Pipeline #1400 failed with stage
...@@ -144,7 +144,7 @@ getGroupParams :: ( HasNodeError err ...@@ -144,7 +144,7 @@ getGroupParams :: ( HasNodeError err
=> GroupParams -> HashSet Ngrams -> m GroupParams => GroupParams -> HashSet Ngrams -> m GroupParams
getGroupParams gp@(GroupWithPosTag l a _m) ng = do getGroupParams gp@(GroupWithPosTag l a _m) ng = do
hashMap <- HashMap.fromList <$> selectLems l a (HashSet.toList ng) hashMap <- HashMap.fromList <$> selectLems l a (HashSet.toList ng)
printDebug "hashMap" hashMap -- printDebug "hashMap" hashMap
pure $ over gwl_map (\x -> x <> hashMap) gp pure $ over gwl_map (\x -> x <> hashMap) gp
getGroupParams gp _ = pure gp getGroupParams gp _ = pure gp
...@@ -201,8 +201,8 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do ...@@ -201,8 +201,8 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do
$ List.sortOn (viewScore . snd) $ List.sortOn (viewScore . snd)
$ HashMap.toList ns $ HashMap.toList ns
(groupedMonoHead, groupedMonoTail) = splitAt monoSize groupedMono (groupedMonoHead, _groupedMonoTail) = splitAt monoSize groupedMono
(groupedMultHead, groupedMultTail) = splitAt multSize groupedMult (groupedMultHead, groupedMultTail) = splitAt multSize groupedMult
------------------------- -------------------------
-- Filter 1 With Set NodeId and SpeGen -- Filter 1 With Set NodeId and SpeGen
...@@ -276,8 +276,8 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do ...@@ -276,8 +276,8 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do
monoInc_size = splitAt' $ monoSize * inclSize / 2 monoInc_size = splitAt' $ monoSize * inclSize / 2
(monoScoredInclHead, monoScoredInclTail) = monoInc_size $ (sortOn scored_genInc) monoScoredIncl (monoScoredInclHead, _monoScoredInclTail) = monoInc_size $ (sortOn scored_genInc) monoScoredIncl
(monoScoredExclHead, monoScoredExclTail) = monoInc_size $ (sortOn scored_speExc) monoScoredExcl (monoScoredExclHead, _monoScoredExclTail) = monoInc_size $ (sortOn scored_speExc) monoScoredExcl
multExc_size = splitAt' $ multSize * exclSize / 2 multExc_size = splitAt' $ multSize * exclSize / 2
(multScoredInclHead, multScoredInclTail) = multExc_size $ (sortOn scored_genInc) multScoredIncl (multScoredInclHead, multScoredInclTail) = multExc_size $ (sortOn scored_genInc) multScoredIncl
...@@ -285,25 +285,29 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do ...@@ -285,25 +285,29 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do
------------------------------------------------------------ ------------------------------------------------------------
-- Final Step building the Typed list -- Final Step building the Typed list
termListHead = maps <> cands -- Candidates Terms need to be filtered
where let
maps = setListType (Just MapTerm) maps = setListType (Just MapTerm)
$ monoScoredInclHead $ monoScoredInclHead
<> monoScoredExclHead <> monoScoredExclHead
<> multScoredInclHead <> multScoredInclHead
<> multScoredExclHead <> multScoredExclHead
cands = setListType (Just CandidateTerm) -- An original way to filter to start with
$ monoScoredInclTail cands = setListType (Just CandidateTerm)
<> monoScoredExclTail $ {- monoScoredInclTail
<> multScoredInclTail <> monoScoredExclTail
<> multScoredExclTail <> -} multScoredInclTail
<> multScoredExclTail
termListTail = (setListType (Just CandidateTerm)) (groupedMonoTail <> groupedMultTail)
cands' = setListType (Just CandidateTerm)
let result = Map.unionsWith (<>) {-$ groupedMonoTail
[ Map.fromList [( nt, toNgramsElement termListHead <>-} groupedMultTail
<> toNgramsElement termListTail
result = Map.unionsWith (<>)
[ Map.fromList [( nt, toNgramsElement maps
<> toNgramsElement cands
<> toNgramsElement cands'
<> toNgramsElement stopTerms <> toNgramsElement stopTerms
)] )]
] ]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment