[ngrams] specialCharNormalize to properly highlight accents

But this will never be the same with what the backend is doing.
parent 57939c9a
Pipeline #4160 failed with stage
in 0 seconds
...@@ -47,6 +47,7 @@ import Gargantext.Types (AsyncTask, AsyncTaskType(..), AsyncTaskWithType(..), CT ...@@ -47,6 +47,7 @@ import Gargantext.Types (AsyncTask, AsyncTaskType(..), AsyncTaskWithType(..), CT
import Gargantext.Utils.Either (eitherMap) import Gargantext.Utils.Either (eitherMap)
--import Gargantext.Utils.KarpRabin (indicesOfAny) --import Gargantext.Utils.KarpRabin (indicesOfAny)
import Gargantext.Utils.Reactix as R2 import Gargantext.Utils.Reactix as R2
import Gargantext.Utils.String as GS
import Partial (crashWith) import Partial (crashWith)
import Partial.Unsafe (unsafePartial) import Partial.Unsafe (unsafePartial)
import Reactix as R import Reactix as R
...@@ -82,7 +83,9 @@ normNgramInternal :: CTabNgramType -> String -> String ...@@ -82,7 +83,9 @@ normNgramInternal :: CTabNgramType -> String -> String
normNgramInternal CTabAuthors = identity normNgramInternal CTabAuthors = identity
normNgramInternal CTabSources = identity normNgramInternal CTabSources = identity
normNgramInternal CTabInstitutes = identity normNgramInternal CTabInstitutes = identity
normNgramInternal CTabTerms = S.toLower <<< R.replace wordBoundaryReg " " normNgramInternal CTabTerms = GS.specialCharNormalize
<<< S.toLower
<<< R.replace wordBoundaryReg " "
normNgramWithTrim :: CTabNgramType -> String -> String normNgramWithTrim :: CTabNgramType -> String -> String
normNgramWithTrim nt = DSC.trim <<< normNgramInternal nt normNgramWithTrim nt = DSC.trim <<< normNgramInternal nt
......
export function _btoa(s) { export function _btoa(s) {
return btoa(unescape(encodeURIComponent(s))); return btoa(unescape(encodeURIComponent(s)));
}
export function _specialCharNormalize(s) {
return s.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
} }
...@@ -7,9 +7,14 @@ import Data.Int as Int ...@@ -7,9 +7,14 @@ import Data.Int as Int
import Data.Number.Format as DNF import Data.Number.Format as DNF
foreign import _btoa :: Fn1 String String foreign import _btoa :: Fn1 String String
foreign import _specialCharNormalize :: Fn1 String String
btoa :: String -> String btoa :: String -> String
btoa = runFn1 _btoa btoa = runFn1 _btoa
intToString :: Int -> String intToString :: Int -> String
intToString = DNF.toString <<< Int.toNumber intToString = DNF.toString <<< Int.toNumber
-- | https://ricardometring.com/javascript-replace-special-characters
specialCharNormalize :: String -> String
specialCharNormalize = runFn1 _specialCharNormalize
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment