[ngrams] specialCharNormalize to properly highlight accents

But this will never be the same with what the backend is doing.
parent 57939c9a
Pipeline #4160 failed with stage
in 0 seconds
......@@ -47,6 +47,7 @@ import Gargantext.Types (AsyncTask, AsyncTaskType(..), AsyncTaskWithType(..), CT
import Gargantext.Utils.Either (eitherMap)
--import Gargantext.Utils.KarpRabin (indicesOfAny)
import Gargantext.Utils.Reactix as R2
import Gargantext.Utils.String as GS
import Partial (crashWith)
import Partial.Unsafe (unsafePartial)
import Reactix as R
......@@ -82,7 +83,9 @@ normNgramInternal :: CTabNgramType -> String -> String
normNgramInternal CTabAuthors = identity
normNgramInternal CTabSources = identity
normNgramInternal CTabInstitutes = identity
normNgramInternal CTabTerms = S.toLower <<< R.replace wordBoundaryReg " "
normNgramInternal CTabTerms = GS.specialCharNormalize
<<< S.toLower
<<< R.replace wordBoundaryReg " "
normNgramWithTrim :: CTabNgramType -> String -> String
normNgramWithTrim nt = DSC.trim <<< normNgramInternal nt
......
export function _btoa(s) {
return btoa(unescape(encodeURIComponent(s)));
return btoa(unescape(encodeURIComponent(s)));
}
export function _specialCharNormalize(s) {
return s.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
}
......@@ -7,9 +7,14 @@ import Data.Int as Int
import Data.Number.Format as DNF
foreign import _btoa :: Fn1 String String
foreign import _specialCharNormalize :: Fn1 String String
btoa :: String -> String
btoa = runFn1 _btoa
intToString :: Int -> String
intToString = DNF.toString <<< Int.toNumber
-- | https://ricardometring.com/javascript-replace-special-characters
specialCharNormalize :: String -> String
specialCharNormalize = runFn1 _specialCharNormalize
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment