Skip to content

haskell-gargantext
haskell-gargantext
Unverified Commit 441edde5 authored by Nicolas Pouillard's avatar Nicolas Pouillard
Browse files
Options

[FLOW] Use a Data.Set to avoid extra insertions of the same ngram

parent 081ec8fb
Pipeline #119 failed with stage
    Show whitespace changes
    Inline Side-by-side
    Showing with 2 additions and 1 deletion
    src/Gargantext/Database/Flow.hs
    View file @ 441edde5
    ...@@ -51,6 +51,7 @@ import Gargantext.Prelude ...@@ -51,6 +51,7 @@ import Gargantext.Prelude
    import Gargantext.Text.Parsers (parseDocs, FileFormat) import Gargantext.Text.Parsers (parseDocs, FileFormat)
    import System.FilePath (FilePath) import System.FilePath (FilePath)
    import qualified Data.Map as DM import qualified Data.Map as DM
    import qualified Data.Set as DS
    flowCorpus :: HasNodeError err => FileFormat -> FilePath -> CorpusName -> Cmd err CorpusId flowCorpus :: HasNodeError err => FileFormat -> FilePath -> CorpusName -> Cmd err CorpusId
    ...@@ -113,7 +114,7 @@ flowCorpus' NodeCorpus hyperdataDocuments (ids,masterUserId,masterCorpusId, user ...@@ -113,7 +114,7 @@ flowCorpus' NodeCorpus hyperdataDocuments (ids,masterUserId,masterCorpusId, user
    let maps = mapNodeIdNgrams docsWithNgrams let maps = mapNodeIdNgrams docsWithNgrams
    -- printDebug "maps" (maps) -- printDebug "maps" (maps)
    terms2id <- insertNgrams (map _ngramsT $ DM.keys maps) terms2id <- insertNgrams (DS.toList $ DS.map _ngramsT (DM.keysSet maps))
    let indexedNgrams = DM.mapKeys (indexNgramsT terms2id) maps let indexedNgrams = DM.mapKeys (indexNgramsT terms2id) maps
    -- printDebug "inserted ngrams" indexedNgrams -- printDebug "inserted ngrams" indexedNgrams
    _ <- insertToNodeNgrams indexedNgrams _ <- insertToNodeNgrams indexedNgrams
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment