Commit 6a3a5ca2 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[FIX] ngrams + ngramsPosTag insert

parent 7697da0d
......@@ -42,11 +42,10 @@ import Data.Text (Text)
import Data.Traversable
import GHC.Base (String)
import GHC.Generics (Generic)
import qualified Data.List as List
import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified Data.List as List
import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified Data.HashMap.Strict as HashMap
import Gargantext.Core
import Gargantext.Core.Text (sentences, HasText(..))
import Gargantext.Core.Text.Terms.Eleve (mainEleveWith, Tries, Token, buildTries, toToken)
......@@ -151,13 +150,14 @@ extracted2ngrams (EnrichedNgrams ng) = view np_form ng
insertExtractedNgrams :: [ ExtractedNgrams ] -> Cmd err (HashMap Text NgramsId)
insertExtractedNgrams ngs = do
let (s, e) = List.partition isSimpleNgrams ngs
m1 <- if List.null s
then pure HashMap.empty
else insertNgrams (map unSimpleNgrams s)
m2 <- if List.null e
then pure HashMap.empty
else insertNgramsPostag (map unEnrichedNgrams e)
pure $ m1 <> m2
m1 <- insertNgrams (map unSimpleNgrams s)
printDebug "others" m1
m2 <- insertNgramsPostag (map unEnrichedNgrams e)
printDebug "terms" m2
let result = HashMap.unions [m1, m2]
pure result
isSimpleNgrams :: ExtractedNgrams -> Bool
isSimpleNgrams (SimpleNgrams _) = True
......
......@@ -26,7 +26,8 @@ import Data.HashMap.Strict (HashMap)
import Data.ByteString.Internal (ByteString)
import Data.Text (Text)
import qualified Database.PostgreSQL.Simple as PGS
import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List
import qualified Data.HashMap.Strict as HashMap
import Gargantext.Core.Types
import Gargantext.Database.Prelude (runOpaQuery, Cmd)
......@@ -66,7 +67,10 @@ _dbGetNgramsDb = runOpaQuery queryNgramsTable
-- TODO-ACCESS: access must not be checked here but when insertNgrams is called.
insertNgrams :: [Ngrams] -> Cmd err (HashMap Text NgramsId)
insertNgrams ns = HashMap.fromList <$> map (\(Indexed i t) -> (t, i)) <$> (insertNgrams' ns)
insertNgrams ns =
if List.null ns
then pure HashMap.empty
else HashMap.fromList <$> map (\(Indexed i t) -> (t, i)) <$> (insertNgrams' ns)
-- TODO-ACCESS: access must not be checked here but when insertNgrams' is called.
insertNgrams' :: [Ngrams] -> Cmd err [Indexed Int Text]
......@@ -90,7 +94,7 @@ queryInsertNgrams = [sql|
RETURNING id,terms
)
SELECT terms, id
SELECT id, terms
FROM ins
UNION ALL
SELECT c.id, terms
......
......@@ -27,10 +27,10 @@ import Gargantext.Database.Schema.Ngrams
import Gargantext.Database.Schema.Prelude
import Gargantext.Database.Types
import Gargantext.Prelude
import qualified Data.HashMap.Strict as HashMap
import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List
import qualified Database.PostgreSQL.Simple as PGS
data NgramsPostag = NgramsPostag { _np_lang :: Lang
, _np_algo :: PosTagAlgo
, _np_postag :: POS
......@@ -64,9 +64,12 @@ toInsert (NgramsPostag l a p form lem) =
)
insertNgramsPostag :: [NgramsPostag] -> Cmd err (HashMap Text NgramsId)
insertNgramsPostag ns = HashMap.fromList
<$> map (\(Indexed t i) -> (t,i))
<$> insertNgramsPostag' (map toInsert ns)
insertNgramsPostag ns =
if List.null ns
then pure HashMap.empty
else HashMap.fromList
<$> map (\(Indexed t i) -> (t,i))
<$> insertNgramsPostag' (map toInsert ns)
insertNgramsPostag' :: [NgramsPostagInsert] -> Cmd err [Indexed Text Int]
insertNgramsPostag' ns = runPGSQuery queryInsertNgramsPostag (PGS.Only $ Values fields ns)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment