[OPTIM] Insert ngrams

src/Gargantext/Database/Action/Flow.hs

@@ -448,19 +448,20 @@ indexAllDocumentsWithPosTag = do
   corpusIds <- findNodesId rootId [NodeCorpus]
   docs      <- List.concat <$> mapM getDocumentsWithParentId corpusIds
 
-  printDebug "Nb of docs" (List.length docs)
+  _ <- mapM extractInsert (splitEvery 1000 docs)
 
+  pure ()
+
+extractInsert :: FlowCmdM env err m => [Node HyperdataDocument] -> m ()
+extractInsert docs = do
   let documentsWithId = map (\doc -> Indexed (doc ^. node_id) doc) docs
 
-  mapNgramsDocs' :: HashMap ExtractedNgrams (Map NgramsType (Map NodeId Int))
-                <- mapNodeIdNgrams
+  mapNgramsDocs' <- mapNodeIdNgrams
                 <$> documentIdWithNgrams
                     (extractNgramsT $ withLang (Multi EN) documentsWithId)
                     documentsWithId
 
-  _ <- mapM insertExtractedNgrams
-     $ splitEvery 500
-     $ HashMap.keys mapNgramsDocs'
+  _ <- insertExtractedNgrams $ HashMap.keys mapNgramsDocs'
 
   pure ()
 
@@ -471,7 +472,3 @@ indexAllDocumentsWithPosTag = do
 
 
 
-
-
-
-