[FIX] insertion is ok now.

src/Gargantext/Core/Text/List.hs

@@ -17,6 +17,7 @@ module Gargantext.Core.Text.List
 
 import Control.Lens hiding (both) -- ((^.), view, over, set, (_1), (_2))
 import Data.HashMap.Strict (HashMap)
+import Data.HashSet (HashSet)
 import Data.Map (Map)
 import Data.Monoid (mempty)
 import Data.Ord (Down(..))
@@ -36,6 +37,7 @@ import Gargantext.Database.Action.Metrics.NgramsByNode (getNodesByNgramsUser, ge
 import Gargantext.Database.Action.Metrics.TFICF (getTficf)
 import Gargantext.Database.Admin.Types.Node (NodeId)
 import Gargantext.Database.Prelude (CmdM)
+import Gargantext.Database.Query.Table.Ngrams (text2ngrams)
 import Gargantext.Database.Query.Table.Node (defaultList)
 import Gargantext.Database.Query.Table.NgramsPostag (selectLems)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError())
@@ -43,10 +45,11 @@ import Gargantext.Database.Query.Tree.Error (HasTreeError)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..), Ngrams(..))
 import Gargantext.Prelude
 import qualified Data.HashMap.Strict as HashMap
-import qualified Data.List as List
-import qualified Data.Map  as Map
-import qualified Data.Set  as Set
+import qualified Data.List    as List
+import qualified Data.Map     as Map
+import qualified Data.Set     as Set
 import qualified Gargantext.Data.HashMap.Strict.Utils as HashMap
+import qualified Data.HashSet as HashSet
 
 {-
 -- TODO maybe useful for later
@@ -138,9 +141,10 @@ getGroupParams :: ( HasNodeError err
                   , RepoCmdM env err m
                   , HasTreeError err
                   )
-               => GroupParams -> Set Ngrams -> m GroupParams
+               => GroupParams -> HashSet Ngrams -> m GroupParams
 getGroupParams gp@(GroupWithPosTag l a _m) ng = do
-  hashMap <- HashMap.fromList <$> selectLems l a (Set.toList ng)
+  hashMap <- HashMap.fromList <$> selectLems l a (HashSet.toList ng)
+  printDebug "hashMap" hashMap
   pure $ over gwl_map (\x -> x <> hashMap) gp
 getGroupParams gp _ = pure gp
 
@@ -172,9 +176,11 @@ buildNgramsTermsList user uCid mCid groupParams (nt, _mapListSize)= do
                                                       $ List.zip (HashMap.keys   allTerms)
                                                                  (List.cycle     [mempty])
                                            )
+  let ngramsKeys = HashMap.keysSet allTerms
 
-  let socialLists_Stemmed = addScoreStem groupParams (HashMap.keysSet allTerms) socialLists
-  printDebug "socialLists_Stemmed" socialLists_Stemmed
+  groupParams' <- getGroupParams groupParams (HashSet.map (text2ngrams . unNgramsTerm) ngramsKeys)
+  let socialLists_Stemmed = addScoreStem groupParams' ngramsKeys socialLists
+  --printDebug "socialLists_Stemmed" socialLists_Stemmed
   let groupedWithList = toGroupedTree socialLists_Stemmed allTerms
       (stopTerms, candidateTerms) = HashMap.partition ((== Just StopTerm) . viewListType)
                                   $ view flc_scores groupedWithList

src/Gargantext/Core/Text/List/Group/WithStem.hs

@@ -77,12 +77,14 @@ groupWith (GroupParams l _m _n _) t =
                   $ Text.intercalate " "
                   $ map (stem l)
                   -- . take n
-                  $ List.sort
+                  $ Set.toList
+                  $ Set.fromList
                   -- . (List.filter (\t -> Text.length t > m))
                   $ Text.splitOn " "
                   $ Text.replace "-" " "
                   $ unNgramsTerm t
 
+-- | This lemmatization group done with CoreNLP algo (or others)
 groupWith (GroupWithPosTag _ _ m) t = 
   case HashMap.lookup (unNgramsTerm t) m of
       Nothing -> t

src/Gargantext/Database/Query/Table/NgramsPostag.hs

@@ -72,14 +72,14 @@ insertNgramsPostag xs =
        let
           (ns, nps) =
             List.partition (\np -> np ^. np_form . ngramsTerms
-                                /= np ^. np_lem  . ngramsTerms
+                                == np ^. np_lem  . ngramsTerms
                            ) xs
 
        ns' <- insertNgrams (map (view np_form) ns)
 
        nps' <- HashMap.fromList
            <$> map (\(Indexed t i) -> (t,i))
-           <$> insertNgramsPostag' (map toInsert ns)
+           <$> insertNgramsPostag' (map toInsert nps)
 
        pure $ HashMap.union ns' nps'
 
@@ -134,13 +134,13 @@ queryInsertNgramsPostag = [sql|
   ------------------------------------------------
   , ins_postag AS (
     INSERT INTO ngrams_postag (lang_id, algo_id, postag, ngrams_id, lemm_id,score)
-    SELECT ir.lang_id, ir.algo_id, ir.postag, form.id, lem.id, count(*) as s
+    SELECT ir.lang_id, ir.algo_id, ir.postag, form.id, lem.id,1 -- count(*) as s
     FROM input_rows ir
       JOIN ins_form_ret  form ON form.terms = ir.form
       JOIN ins_lem_ret   lem  ON lem.terms  = ir.lem
-       GROUP BY ir.lang_id, ir.algo_id, ir.postag, form.id, lem.id    
-       ORDER BY s DESC
-       LIMIT 1
+       -- GROUP BY ir.lang_id, ir.algo_id, ir.postag, form.id, lem.id    
+       -- ORDER BY s DESC
+       -- LIMIT 1
       ON CONFLICT (lang_id,algo_id,postag,ngrams_id,lemm_id)
         DO UPDATE SET score = ngrams_postag.score + 1
     )