Commit 6a3a5ca2 authored by Alexandre Delanoë's avatar Alexandre Delanoë

[FIX] ngrams + ngramsPosTag insert

parent 7697da0d
...@@ -42,11 +42,10 @@ import Data.Text (Text) ...@@ -42,11 +42,10 @@ import Data.Text (Text)
import Data.Traversable import Data.Traversable
import GHC.Base (String) import GHC.Base (String)
import GHC.Generics (Generic) import GHC.Generics (Generic)
import qualified Data.List as List import qualified Data.List as List
import qualified Data.Set as Set import qualified Data.Set as Set
import qualified Data.Text as Text import qualified Data.Text as Text
import qualified Data.HashMap.Strict as HashMap import qualified Data.HashMap.Strict as HashMap
import Gargantext.Core import Gargantext.Core
import Gargantext.Core.Text (sentences, HasText(..)) import Gargantext.Core.Text (sentences, HasText(..))
import Gargantext.Core.Text.Terms.Eleve (mainEleveWith, Tries, Token, buildTries, toToken) import Gargantext.Core.Text.Terms.Eleve (mainEleveWith, Tries, Token, buildTries, toToken)
...@@ -151,13 +150,14 @@ extracted2ngrams (EnrichedNgrams ng) = view np_form ng ...@@ -151,13 +150,14 @@ extracted2ngrams (EnrichedNgrams ng) = view np_form ng
insertExtractedNgrams :: [ ExtractedNgrams ] -> Cmd err (HashMap Text NgramsId) insertExtractedNgrams :: [ ExtractedNgrams ] -> Cmd err (HashMap Text NgramsId)
insertExtractedNgrams ngs = do insertExtractedNgrams ngs = do
let (s, e) = List.partition isSimpleNgrams ngs let (s, e) = List.partition isSimpleNgrams ngs
m1 <- if List.null s m1 <- insertNgrams (map unSimpleNgrams s)
then pure HashMap.empty printDebug "others" m1
else insertNgrams (map unSimpleNgrams s)
m2 <- if List.null e m2 <- insertNgramsPostag (map unEnrichedNgrams e)
then pure HashMap.empty printDebug "terms" m2
else insertNgramsPostag (map unEnrichedNgrams e)
pure $ m1 <> m2 let result = HashMap.unions [m1, m2]
pure result
isSimpleNgrams :: ExtractedNgrams -> Bool isSimpleNgrams :: ExtractedNgrams -> Bool
isSimpleNgrams (SimpleNgrams _) = True isSimpleNgrams (SimpleNgrams _) = True
......
...@@ -26,7 +26,8 @@ import Data.HashMap.Strict (HashMap) ...@@ -26,7 +26,8 @@ import Data.HashMap.Strict (HashMap)
import Data.ByteString.Internal (ByteString) import Data.ByteString.Internal (ByteString)
import Data.Text (Text) import Data.Text (Text)
import qualified Database.PostgreSQL.Simple as PGS import qualified Database.PostgreSQL.Simple as PGS
import qualified Data.HashMap.Strict as HashMap import qualified Data.List as List
import qualified Data.HashMap.Strict as HashMap
import Gargantext.Core.Types import Gargantext.Core.Types
import Gargantext.Database.Prelude (runOpaQuery, Cmd) import Gargantext.Database.Prelude (runOpaQuery, Cmd)
...@@ -66,7 +67,10 @@ _dbGetNgramsDb = runOpaQuery queryNgramsTable ...@@ -66,7 +67,10 @@ _dbGetNgramsDb = runOpaQuery queryNgramsTable
-- TODO-ACCESS: access must not be checked here but when insertNgrams is called. -- TODO-ACCESS: access must not be checked here but when insertNgrams is called.
insertNgrams :: [Ngrams] -> Cmd err (HashMap Text NgramsId) insertNgrams :: [Ngrams] -> Cmd err (HashMap Text NgramsId)
insertNgrams ns = HashMap.fromList <$> map (\(Indexed i t) -> (t, i)) <$> (insertNgrams' ns) insertNgrams ns =
if List.null ns
then pure HashMap.empty
else HashMap.fromList <$> map (\(Indexed i t) -> (t, i)) <$> (insertNgrams' ns)
-- TODO-ACCESS: access must not be checked here but when insertNgrams' is called. -- TODO-ACCESS: access must not be checked here but when insertNgrams' is called.
insertNgrams' :: [Ngrams] -> Cmd err [Indexed Int Text] insertNgrams' :: [Ngrams] -> Cmd err [Indexed Int Text]
...@@ -90,7 +94,7 @@ queryInsertNgrams = [sql| ...@@ -90,7 +94,7 @@ queryInsertNgrams = [sql|
RETURNING id,terms RETURNING id,terms
) )
SELECT terms, id SELECT id, terms
FROM ins FROM ins
UNION ALL UNION ALL
SELECT c.id, terms SELECT c.id, terms
......
...@@ -27,10 +27,10 @@ import Gargantext.Database.Schema.Ngrams ...@@ -27,10 +27,10 @@ import Gargantext.Database.Schema.Ngrams
import Gargantext.Database.Schema.Prelude import Gargantext.Database.Schema.Prelude
import Gargantext.Database.Types import Gargantext.Database.Types
import Gargantext.Prelude import Gargantext.Prelude
import qualified Data.HashMap.Strict as HashMap import qualified Data.HashMap.Strict as HashMap
import qualified Data.List as List
import qualified Database.PostgreSQL.Simple as PGS import qualified Database.PostgreSQL.Simple as PGS
data NgramsPostag = NgramsPostag { _np_lang :: Lang data NgramsPostag = NgramsPostag { _np_lang :: Lang
, _np_algo :: PosTagAlgo , _np_algo :: PosTagAlgo
, _np_postag :: POS , _np_postag :: POS
...@@ -64,9 +64,12 @@ toInsert (NgramsPostag l a p form lem) = ...@@ -64,9 +64,12 @@ toInsert (NgramsPostag l a p form lem) =
) )
insertNgramsPostag :: [NgramsPostag] -> Cmd err (HashMap Text NgramsId) insertNgramsPostag :: [NgramsPostag] -> Cmd err (HashMap Text NgramsId)
insertNgramsPostag ns = HashMap.fromList insertNgramsPostag ns =
<$> map (\(Indexed t i) -> (t,i)) if List.null ns
<$> insertNgramsPostag' (map toInsert ns) then pure HashMap.empty
else HashMap.fromList
<$> map (\(Indexed t i) -> (t,i))
<$> insertNgramsPostag' (map toInsert ns)
insertNgramsPostag' :: [NgramsPostagInsert] -> Cmd err [Indexed Text Int] insertNgramsPostag' :: [NgramsPostagInsert] -> Cmd err [Indexed Text Int]
insertNgramsPostag' ns = runPGSQuery queryInsertNgramsPostag (PGS.Only $ Values fields ns) insertNgramsPostag' ns = runPGSQuery queryInsertNgramsPostag (PGS.Only $ Values fields ns)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment