[hyperdata] uniqId, uniqIdBdd removal

We have contexts.hash_id which is enough (computed by postgres as a
trigger).

src/Gargantext/API/GraphQL/Context.hs

@@ -25,10 +25,10 @@ import Data.Morpheus.Types
 import Data.Text (pack)
 import Data.Time.Format.ISO8601 (iso8601Show)
 import Gargantext.API.Admin.Types (HasSettings)
-import Gargantext.API.Errors.Types
+import Gargantext.API.Errors.Types ( BackendInternalError )
 import Gargantext.API.Prelude (GargM)
 import Gargantext.Core.Types.Search (HyperdataRow(..), toHyperdataRow)
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument)
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument )
 import Gargantext.Database.Admin.Types.Node (ContextTitle, NodeId(..), NodeTypeId, UserId, unNodeId, ContextId (..))
 import Gargantext.Database.Prelude (CmdCommon)
 import Gargantext.Database.Query.Table.NodeContext (getNodeContext, getContextsForNgramsTerms, ContextForNgramsTerms(..), {- getContextNgrams, -} getContextNgramsMatchingFTS)
@@ -71,8 +71,6 @@ data HyperdataRowDocumentGQL =
                           , hrd_source             :: Text
                           , hrd_title              :: Text
                           , hrd_url                :: Text
-                          , hrd_uniqId             :: Text
-                          , hrd_uniqIdBdd          :: Text
                           } deriving (Generic, GQLType, Show)
 
 data NodeContextGQL = NodeContextGQL
@@ -216,8 +214,6 @@ toHyperdataRowDocumentGQL hyperdata =
                                      , hrd_source             = _hr_source
                                      , hrd_title              = _hr_title
                                      , hrd_url                = _hr_url
-                                     , hrd_uniqId             = _hr_uniqId
-                                     , hrd_uniqIdBdd          = _hr_uniqIdBdd
                                      }
     HyperdataRowContact { } -> Nothing
 

src/Gargantext/API/Node/Corpus/Export.hs

@@ -22,21 +22,21 @@ import Data.Map.Strict qualified as Map
 import Data.Set qualified as Set
 import Data.Text (pack)
 import Gargantext.API.Ngrams.Tools (filterListWithRoot, mapTermListRoot, getRepo)
-import Gargantext.API.Ngrams.Types
-import Gargantext.API.Node.Corpus.Export.Types
+import Gargantext.API.Ngrams.Types ( NgramsTerm(unNgramsTerm) )
+import Gargantext.API.Node.Corpus.Export.Types ( Corpus(..) )
 import Gargantext.API.Node.Document.Export.Types qualified as DocumentExport
 import Gargantext.API.Prelude (GargNoServer)
-import Gargantext.Core.NodeStory
+import Gargantext.Core.NodeStory.Types ( NodeListStory )
 import Gargantext.Core.Types
 import Gargantext.Database.Action.Metrics.NgramsByContext (getNgramsByContextOnlyUser)
 import Gargantext.Database.Admin.Config (userMaster)
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Database.Prelude (Cmd)
-import Gargantext.Database.Query.Table.Node
+import Gargantext.Database.Query.Table.Node ( defaultList )
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
 import Gargantext.Database.Query.Table.Node.Select (selectNodesWithUsername)
 import Gargantext.Database.Query.Table.NodeContext (selectDocNodes)
-import Gargantext.Database.Schema.Context (_context_id, _context_hyperdata)
+import Gargantext.Database.Schema.Context (_context_id)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))
 import Gargantext.Prelude hiding (hash)
 import Gargantext.Prelude.Crypto.Hash (hash)
@@ -51,9 +51,7 @@ getCorpus :: CorpusId
 getCorpus cId lId nt' = do
 
   let
-    nt = case nt' of
-      Nothing -> NgramsTerms
-      Just  t -> t
+    nt = fromMaybe NgramsTerms nt'
 
   listId <- case lId of
     Nothing -> defaultList cId
@@ -75,10 +73,10 @@ getCorpus cId lId nt' = do
         ) ns (Map.map (Set.map unNgramsTerm) ngs)
           where
             d_hash :: Context HyperdataDocument -> Set Text -> Text
-            d_hash  a b = hash [ fromMaybe "" (_hd_uniqId $ _context_hyperdata a)
-                               , hash b
+            d_hash  _a b = hash [ -- fromMaybe "" (_hd_uniqId $ _context_hyperdata a),
+                                hash b
                                ]
-  pure $ addHeader ("attachment; filename=GarganText_corpus-" <> (pack $ show cId) <> ".json")
+  pure $ addHeader ("attachment; filename=GarganText_corpus-" <> pack (show cId) <> ".json")
     $ Corpus { _c_corpus = Map.elems r
              , _c_hash = hash $ List.map DocumentExport._d_hash $ Map.elems r }
 

src/Gargantext/API/Node/Corpus/Searx.hs

@@ -24,7 +24,7 @@ import Data.Time.Format (defaultTimeLocale, formatTime, parseTimeM)
 import Data.Tuple.Select (sel1, sel2, sel3)
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.NLP (HasNLPServer, nlpServerGet)
-import Gargantext.Core.NodeStory (HasNodeStory)
+import Gargantext.Core.NodeStory.Types ( HasNodeStory )
 import Gargantext.Core.Text.Corpus.API qualified as API
 import Gargantext.Core.Text.List (buildNgramsLists)
 import Gargantext.Core.Text.List.Group.WithStem ({-StopSize(..),-} GroupParams(..))
@@ -40,8 +40,7 @@ import Gargantext.Database.Admin.Types.Hyperdata.Corpus (HyperdataCorpus)
 import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
 import Gargantext.Database.Admin.Types.Node (CorpusId, ListId, NodeType(NodeTexts))
 import Gargantext.Database.Prelude (hasConfig)
-import Gargantext.Database.Query.Table.Node (getOrMkList)
-import Gargantext.Database.Query.Table.Node (insertDefaultNodeIfNotExists)
+import Gargantext.Database.Query.Table.Node (getOrMkList, insertDefaultNodeIfNotExists)
 import Gargantext.Database.Query.Table.Node.Error (HasNodeError)
 import Gargantext.Database.Query.Tree.Error (HasTreeError)
 import Gargantext.Database.Query.Tree.Root (getOrMk_RootWithCorpus)
@@ -54,7 +53,7 @@ import Prelude qualified
 
 langToSearx :: Lang -> Text
 langToSearx All = "en-US"
-langToSearx x   = (Text.toLower acronym) <> "-" <> acronym
+langToSearx x   = Text.toLower acronym <> "-" <> acronym
   where
     acronym = show x
 
@@ -137,7 +136,7 @@ insertSearxResponse user cId listId l (Right (SearxResponse { _srs_results })) =
   -- docs :: [Either Text HyperdataDocument]
   let docs = hyperdataDocumentFromSearxResult l <$> _srs_results
   --printDebug "[triggerSearxSearch] docs" docs
-  let docs' = catMaybes $ rightToMaybe <$> docs
+  let docs' = mapMaybe rightToMaybe docs
     {-
   Prelude.mapM_ (\(HyperdataDocument { _hd_title, _hd_publication_year, _hd_publication_date }) -> do
       printDebug "[triggerSearxSearch] doc time" $
@@ -215,16 +214,14 @@ hyperdataDocumentFromSearxResult l (SearxResult { _sr_content, _sr_engine, _sr_p
   Right HyperdataDocument { _hd_bdd = Just "Searx"
                           , _hd_doi = Nothing
                           , _hd_url = Nothing
-                          , _hd_uniqId = Nothing
-                          , _hd_uniqIdBdd = Nothing
                           , _hd_page = Nothing
                           , _hd_title = Just _sr_title
                           , _hd_authors = Nothing
                           , _hd_institutes = Nothing
                           , _hd_source = Just _sr_engine
                           , _hd_abstract = _sr_content
-                          , _hd_publication_date = T.pack <$> formatTime defaultTimeLocale "%Y-%m-%dT%H:%M:%S" <$> mDate
-                          , _hd_publication_year = fromIntegral <$> sel1 <$> mGregorian
+                          , _hd_publication_date = T.pack Prelude.. formatTime defaultTimeLocale "%Y-%m-%dT%H:%M:%S" Prelude.<$> mDate
+                          , _hd_publication_year = fromIntegral Prelude.. sel1 Prelude.<$> mGregorian
                           , _hd_publication_month = sel2 <$> mGregorian
                           , _hd_publication_day = sel3 <$> mGregorian
                           , _hd_publication_hour = Nothing

src/Gargantext/API/Node/DocumentUpload.hs

@@ -22,22 +22,22 @@ import Data.Swagger (ToSchema)
 import Data.Text qualified as T
 import Gargantext.API.Admin.EnvTypes (GargJob(..), Env)
 import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
-import Gargantext.API.Errors.Types
-import Gargantext.API.Prelude
+import Gargantext.API.Errors.Types ( BackendInternalError )
+import Gargantext.API.Prelude ( GargM )
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.NLP (nlpServerGet)
 import Gargantext.Core.Text.Corpus.Parsers.Date (mDateSplit)
 import Gargantext.Core.Text.Terms (TermType(..))
 import Gargantext.Core.Utils.Prefix (unCapitalize, dropPrefix)
 import Gargantext.Database.Action.Flow (addDocumentsToHyperCorpus)
-import Gargantext.Database.Action.Flow.Types
-import Gargantext.Database.Admin.Types.Hyperdata.Corpus
+import Gargantext.Database.Action.Flow.Types ( FlowCmdM )
+import Gargantext.Database.Admin.Types.Hyperdata.Corpus ( HyperdataCorpus )
 import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Node ( DocId, NodeId, NodeType(NodeCorpus) )
 import Gargantext.Database.Query.Table.Node (getClosestParentIdByType')
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
-import Servant
+import Servant ( JSON, Summary, type (:>), HasServer(ServerT) )
 
 
 data DocumentUpload = DocumentUpload
@@ -108,8 +108,6 @@ documentUpload nId doc = do
   let hd = HyperdataDocument { _hd_bdd = Nothing
                              , _hd_doi = Nothing
                              , _hd_url = Nothing
-                             , _hd_uniqId = Nothing
-                             , _hd_uniqIdBdd = Nothing
                              , _hd_page = Nothing
                              , _hd_title = Just $ if view du_title doc == "" then T.take 50 (view du_abstract doc) else view du_title doc
                              , _hd_authors = Just $ view du_authors doc

src/Gargantext/API/Node/DocumentsFromWriteNodes.hs

@@ -10,23 +10,22 @@ Portability : POSIX
 -}
 
 {-# LANGUAGE MonoLocalBinds     #-}
-{-# LANGUAGE TemplateHaskell    #-}
 {-# LANGUAGE TypeOperators      #-}
 
 module Gargantext.API.Node.DocumentsFromWriteNodes
       where
 
-import Conduit
+import Conduit ( yieldMany )
 import Control.Lens ((^.))
-import Data.Aeson
+import Data.Aeson ( genericParseJSON, defaultOptions, genericToJSON, FromJSON(parseJSON), ToJSON(toJSON) )
 import Data.List qualified as List
-import Data.Swagger
+import Data.Swagger ( ToSchema )
 import Data.Text qualified as T
-import Gargantext.API.Admin.Auth.Types
+import Gargantext.API.Admin.Auth.Types ( AuthenticatedUser, auth_node_id, auth_user_id )
 import Gargantext.API.Admin.EnvTypes (Env, GargJob(..))
 import Gargantext.API.Admin.Orchestrator.Types (JobLog(..), AsyncJobs)
 import Gargantext.API.Admin.Types (HasSettings)
-import Gargantext.API.Errors.Types
+import Gargantext.API.Errors.Types ( BackendInternalError )
 import Gargantext.API.Ngrams (commitStatePatch, Versioned(..))
 import Gargantext.API.Prelude (GargM)
 import Gargantext.Core (Lang(..))
@@ -39,13 +38,13 @@ import Gargantext.Core.Types.Individu (User(..))
 import Gargantext.Database.Action.Flow (flowDataText, DataText(..))
 import Gargantext.Database.Action.Flow.Types (FlowCmdM)
 import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
-import Gargantext.Database.Admin.Types.Hyperdata.Frame
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Hyperdata.Frame ( HyperdataFrame(..), getHyperdataFrameContents )
+import Gargantext.Database.Admin.Types.Node ( NodeId, Node, NodeType(..) )
 import Gargantext.Database.Query.Table.Node (getChildrenByType, getClosestParentIdByType', getNodeWith, getOrMkList)
 import Gargantext.Database.Schema.Node (node_hyperdata, node_name, node_date)
 import Gargantext.Prelude
 import Gargantext.Utils.Jobs (serveJobsAPI, MonadJobStatus(..))
-import Servant
+import Servant ( JSON, Summary, type (:>), HasServer(ServerT) )
 
 ------------------------------------------------------------------------
 type API = Summary " Documents from Write nodes."
@@ -106,7 +105,7 @@ documentsFromWriteNodes authenticatedUser nId Params { selection, lang, paragrap
              pure (node, contents)
          ) frameWrites
 
-  let paragraphs' = fromMaybe (7 :: Int) $ (readMaybe $ T.unpack paragraphs)
+  let paragraphs' = fromMaybe (7 :: Int) $ readMaybe (T.unpack paragraphs)
   let parsedE = (\(node, contents)
                   -> hyperdataDocumentFromFrameWrite lang paragraphs' (node, contents)) <$> frameWritesWithContents
   let parsed = List.concat $ rights parsedE
@@ -159,8 +158,6 @@ hyperdataDocumentFromFrameWrite lang paragraphSize (node, contents) =
       Right (List.map (\(t, ctxt) ->  HyperdataDocument { _hd_bdd = Just $ show Notes
                               , _hd_doi = Nothing
                               , _hd_url = Nothing
-                              , _hd_uniqId = Nothing
-                              , _hd_uniqIdBdd = Nothing
                               , _hd_page = Nothing
                               , _hd_title = Just t
                               , _hd_authors = Just authors'

src/Gargantext/Core/Ext/IMTUser.hs

@@ -16,13 +16,13 @@ Here is writtent a common interface.
 module Gargantext.Core.Ext.IMTUser -- (deserialiseImtUsersFromFile)
   where
 
-import Codec.Serialise
+import Codec.Serialise ( Serialise, deserialise )
 import Data.ByteString.Lazy qualified as BL
-import Data.Csv
+import Data.Csv ( (.:), header, decodeByNameWith, FromNamedRecord(..), Header )
 import Data.Text qualified as T
 import Data.Vector (Vector)
 import Data.Vector qualified as Vector
-import Gargantext.Core.Text.Corpus.Parsers.CSV
+import Gargantext.Core.Text.Corpus.Parsers.CSV ( csvDecodeOptions, Delimiter(Tab) )
 import Gargantext.Database.Admin.Types.Hyperdata.Contact
 import Gargantext.Prelude
 import System.FilePath.Posix (takeExtension)
@@ -156,11 +156,9 @@ imtUser2gargContact (IMTUser { id
                                          , _hc_where = [ou]
                                          , _hc_title = title
                                          , _hc_source = entite
-                                         , _hc_lastValidation = date_modification
-                                         , _hc_uniqIdBdd = Nothing
-                                         , _hc_uniqId = Nothing }
+                                         , _hc_lastValidation = date_modification }
   where
-    title = (<>) <$> (fmap (\p -> p <> " ") prenom) <*> nom
+    title = (<>) <$> fmap (\p -> p <> " ") prenom <*> nom
     qui = ContactWho { _cw_id = id
                      , _cw_firstName = prenom
                      , _cw_lastName = nom
@@ -182,7 +180,7 @@ imtUser2gargContact (IMTUser { id
     -- meta    = ContactMetaData (Just "IMT annuaire") date_modification'
     toList' Nothing  = []
     toList' (Just x) = [x]
-  
+
 
 
 

src/Gargantext/Core/Flow/Types.hs

@@ -15,7 +15,6 @@ Portability : POSIX
 module Gargantext.Core.Flow.Types where
 
 import Control.Lens
-import Gargantext.Database.Admin.Types.Hyperdata
 import Gargantext.Database.Admin.Types.Node
 import Gargantext.Database.Schema.Node (node_hash_id)
 import Gargantext.Prelude
@@ -25,14 +24,6 @@ class UniqId a
   where
     uniqId :: Lens' a (Maybe Hash)
 
-instance UniqId HyperdataDocument
-  where
-    uniqId = hd_uniqId
-
-instance UniqId HyperdataContact
-  where
-    uniqId = hc_uniqId
-
 instance UniqId (Node a)
   where
     uniqId = node_hash_id

src/Gargantext/Core/Text/Corpus/API/Arxiv.hs

@@ -99,8 +99,6 @@ toDoc l (Arxiv.Result { abstract
           ) = HyperdataDocument { _hd_bdd = Just "Arxiv"
                                 , _hd_doi = Just $ Text.pack doi
                                 , _hd_url = Just $ Text.pack url
-                                , _hd_uniqId = Nothing
-                                , _hd_uniqIdBdd = Nothing
                                 , _hd_page = Nothing
                                 , _hd_title = Just $ Text.pack title
                                 , _hd_authors = authors aus

src/Gargantext/Core/Text/Corpus/API/EPO.hs

@@ -9,7 +9,7 @@ Portability : POSIX
 -}
 module Gargantext.Core.Text.Corpus.API.EPO where
 
-import Conduit
+import Conduit ( ConduitT, (.|), mapC )
 import Data.LanguageCodes (ISO639_1)
 import Data.Map.Strict qualified as Map
 import Data.Text qualified as T
@@ -17,7 +17,7 @@ import EPO.API.Client.Types qualified as EPO
 import EPO.API.Client.Implementation qualified as EPO
 import Gargantext.Core (iso639ToText)
 import Gargantext.Core.Text.Corpus.Query qualified as Corpus
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Network.URI (parseURI)
 import Protolude
 import Servant.Client.Core (ClientError(ConnectionError))
@@ -48,8 +48,6 @@ toDoc lang (EPO.HyperdataDocument { .. }) =
   HyperdataDocument { _hd_bdd = Just "EPO"
                     , _hd_doi = Nothing
                     , _hd_url = Nothing
-                    , _hd_uniqId = Nothing
-                    , _hd_uniqIdBdd = Nothing
                     , _hd_page = Nothing
                     , _hd_title = Map.lookup lang titles
                     , _hd_authors = authors_

src/Gargantext/Core/Text/Corpus/API/Hal.hs

@@ -12,14 +12,12 @@ Portability : POSIX
 module Gargantext.Core.Text.Corpus.API.Hal
     where
 
-import Conduit
-import Data.Either
+import Conduit ( ConduitT, (.|), mapMC )
 import Data.LanguageCodes qualified as ISO639
 import Data.Map.Strict qualified as Map
-import Data.Maybe
-import Data.Text (pack, intercalate)
+import Data.Text (pack)
 import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Defaults qualified as Defaults
 import Gargantext.Prelude hiding (intercalate)
 import HAL qualified as HAL
@@ -30,7 +28,7 @@ import Servant.Client (ClientError)
 get :: Maybe ISO639.ISO639_1 -> Text -> Maybe Int -> IO [HyperdataDocument]
 get la q ml = do
   eDocs <- HAL.getMetadataWith [q] (Just 0) (fromIntegral <$> ml) la
-  either (panicTrace . pack . show) (\d -> mapM (toDoc' la) $ HAL._docs d) eDocs
+  either (panicTrace . pack . show) (mapM (toDoc' la) . HAL._docs) eDocs
 
 getC :: Maybe ISO639.ISO639_1 -> Text -> Maybe Int -> IO (Either ClientError (Maybe Integer, ConduitT () HyperdataDocument IO ()))
 getC la q ml = do
@@ -45,17 +43,15 @@ toDoc' la (HAL.Corpus { .. }) = do
   -- printDebug "[toDoc corpus] h" h
   let mDateS = maybe (Just $ pack $ show Defaults.year) Just _corpus_date
   let (utctime, (pub_year, pub_month, pub_day)) = Date.mDateSplit mDateS
-  let abstractDefault = intercalate " " _corpus_abstract
+  let abstractDefault = unwords _corpus_abstract
   let abstract = case la of
         Nothing -> abstractDefault
-        Just l  -> fromMaybe abstractDefault (intercalate " " <$> Map.lookup l _corpus_abstract_lang_map)
+        Just l  -> maybe abstractDefault unwords (Map.lookup l _corpus_abstract_lang_map)
   pure HyperdataDocument { _hd_bdd = Just "Hal"
                          , _hd_doi = Just $ pack $ show _corpus_docid
                          , _hd_url = Nothing
-                         , _hd_uniqId = Nothing
-                         , _hd_uniqIdBdd = Nothing
                          , _hd_page = Nothing
-                         , _hd_title = Just $ intercalate " " _corpus_title
+                         , _hd_title = Just $ unwords _corpus_title
                          , _hd_authors = Just $ foldl (\x y -> if x == "" then y else x <> ", " <> y) "" _corpus_authors_names
                          , _hd_institutes = Just $ foldl (\x y -> if x == "" then y else x <> ", " <> y) "" $ _corpus_authors_affiliations <> map show _corpus_struct_id
                          , _hd_source = Just $ maybe "Nothing" identity _corpus_source

src/Gargantext/Core/Text/Corpus/API/Isidore.hs

@@ -18,12 +18,12 @@ import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.Corpus.Parsers (cleanText)
 import Gargantext.Core.Text.Corpus.Parsers.CSV (writeDocs2Csv)
 import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Defaults qualified as Defaults
 import Gargantext.Prelude hiding (get)
 import Isidore qualified as Isidore
 import Isidore.Client
-import Servant.Client
+import Servant.Client ( ClientError(DecodeFailure) )
 
 -- | TODO work with the ServantErr
 get :: Lang -> Maybe Isidore.Limit
@@ -40,7 +40,7 @@ get la l q a = do
 
   iDocs <- either printErr _content <$> Isidore.get l q a
 
-  hDocs <- mapM (\d -> isidoreToDoc la d) (toIsidoreDocs iDocs)
+  hDocs <- mapM (isidoreToDoc la) (toIsidoreDocs iDocs)
   pure hDocs
 
 isidore2csvFile :: FilePath -> Lang -> Maybe Isidore.Limit
@@ -54,7 +54,7 @@ isidoreToDoc :: Lang -> IsidoreDoc -> IO HyperdataDocument
 isidoreToDoc l (IsidoreDoc t a d u s as) = do
   let
     author :: Author -> Text
-    author (Author fn ln) = (_name fn) <> ", " <> (_name ln)
+    author (Author fn ln) = _name fn <> ", " <> _name ln
     author (Authors aus) = Text.intercalate ". " $ map author aus
 
     creator2text :: Creator -> Text
@@ -66,21 +66,19 @@ isidoreToDoc l (IsidoreDoc t a d u s as) = do
     langText (OnlyText t2   ) = t2
     langText (ArrayText ts  ) = Text.intercalate " " $ map langText ts
 
-  let mDateS = maybe (Just $ Text.pack $ show Defaults.year) (Just) d
+  let mDateS = maybe (Just $ Text.pack $ show Defaults.year) Just d
   let (utcTime, (pub_year, pub_month, pub_day)) = Date.mDateSplit mDateS
 
   pure HyperdataDocument
          { _hd_bdd = Just "Isidore"
          , _hd_doi = Nothing
          , _hd_url = u
-         , _hd_uniqId = Nothing
-         , _hd_uniqIdBdd = Nothing
          , _hd_page = Nothing
          , _hd_title = Just $ cleanText $ langText t
          , _hd_authors = creator2text <$> as
          , _hd_institutes = Nothing
-         , _hd_source = Just $ maybe "Nothing" identity $ _sourceName <$> s
-         , _hd_abstract = cleanText <$> langText    <$> a
+         , _hd_source = Just $ maybe "Nothing" (identity . _sourceName) s
+         , _hd_abstract = cleanText . langText <$> a
          , _hd_publication_date = fmap (Text.pack . show) utcTime
          , _hd_publication_year = pub_year
          , _hd_publication_month = pub_month

src/Gargantext/Core/Text/Corpus/API/OpenAlex.hs

@@ -9,15 +9,15 @@ Portability : POSIX
 -}
 module Gargantext.Core.Text.Corpus.API.OpenAlex where
 
-import Conduit
+import Conduit ( ConduitT, (.|), mapC, takeC )
 import Data.LanguageCodes qualified as ISO639
-import qualified Data.Text as T
+import Data.Text qualified as T
 import Gargantext.Core (iso639ToText)
 import Gargantext.Core.Text.Corpus.Query as Corpus
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
 import Protolude
-import qualified OpenAlex as OA
-import qualified OpenAlex.Types as OA
+import OpenAlex qualified as OA
+import OpenAlex.Types qualified as OA
 import Servant.Client (ClientError)
 
 
@@ -37,8 +37,6 @@ toDoc (OA.Work { .. } ) =
   HyperdataDocument { _hd_bdd = Just "OpenAlex"
                     , _hd_doi = doi
                     , _hd_url = url
-                    , _hd_uniqId = Nothing
-                    , _hd_uniqIdBdd = Nothing
                     , _hd_page = firstPage biblio
                     , _hd_title = title
                     , _hd_authors = authors authorships

src/Gargantext/Core/Text/Corpus/API/Pubmed.hs

@@ -113,8 +113,6 @@ toDoc l (PubMedDoc.PubMed { pubmed_article = PubMedDoc.PubMedArticle t j as aus
           ) = HyperdataDocument { _hd_bdd = Just "PubMed"
                                 , _hd_doi = Nothing
                                 , _hd_url = Nothing
-                                , _hd_uniqId = Nothing
-                                , _hd_uniqIdBdd = Nothing
                                 , _hd_page = Nothing
                                 , _hd_title = t
                                 , _hd_authors = authors aus

src/Gargantext/Core/Text/Corpus/Parsers.hs

@@ -46,7 +46,7 @@ import Gargantext.Core.Text.Corpus.Parsers.JSON (parseJSONC, parseIstex)
 import Gargantext.Core.Text.Corpus.Parsers.RIS qualified as RIS
 import Gargantext.Core.Text.Corpus.Parsers.RIS.Presse (presseEnrich)
 import Gargantext.Core.Text.Corpus.Parsers.WOS qualified as WOS
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Database.Query.Table.Ngrams (NgramsType(..))
 import Gargantext.Prelude hiding (show, undefined)
 import Gargantext.Utils.Zip qualified as UZip
@@ -88,10 +88,10 @@ parseFormatC :: MonadBaseControl IO m
              -> m (Either Text (Integer, ConduitT () HyperdataDocument IO ()))
 parseFormatC CsvGargV3 Plain bs = do
   let eParsedC = parseCsvC $ DBL.fromStrict bs
-  pure ((\(len, parsedC) -> (len, transPipe (pure . runIdentity) parsedC)) <$> eParsedC)
+  pure (second (transPipe (pure . runIdentity)) <$> eParsedC)
 parseFormatC CsvHal    Plain bs = do
   let eParsedC = parseCsvC $ DBL.fromStrict bs
-  pure ((\(len, parsedC) -> (len, transPipe (pure . runIdentity) parsedC)) <$> eParsedC)
+  pure (second (transPipe (pure . runIdentity)) <$> eParsedC)
 parseFormatC Istex Plain bs = do
   ep <- liftBase $ parseIstex EN $ DBL.fromStrict bs
   pure $ (\p -> (1, yieldMany [p])) <$> ep
@@ -120,15 +120,15 @@ parseFormatC Iramuteq Plain bs = do
             , yieldMany docs
               .| mapC (map $ first Iramuteq.keys)
               .| mapC (map $ both decodeUtf8)
-              .| mapMC ((toDoc Iramuteq) . (map (second (DT.replace "_" " "))))
+              .| mapMC (toDoc Iramuteq . map (second (DT.replace "_" " ")))
             )
          )
               <$> eDocs
 parseFormatC JSON    Plain bs = do
   let eParsedC = parseJSONC $ DBL.fromStrict bs
-  pure ((\(len, parsedC) -> (len, transPipe (pure . runIdentity) parsedC)) <$> eParsedC)
+  pure (second (transPipe (pure . runIdentity)) <$> eParsedC)
 parseFormatC ft ZIP bs = liftBase $ UZip.withZipFileBS bs $ do
-  fileNames <- filter (filterZIPFileNameP ft) <$> DM.keys <$> getEntries
+  fileNames <- filter (filterZIPFileNameP ft) . DM.keys <$> getEntries
   printDebug "[parseFormatC] fileNames" fileNames
   fileContents <- mapM getEntry fileNames
   --printDebug "[parseFormatC] fileContents" fileContents
@@ -145,19 +145,19 @@ parseFormatC ft ZIP bs = liftBase $ UZip.withZipFileBS bs $ do
           let contents' = snd <$> contents
           let totalLength = sum lenghts
           pure $ Right ( totalLength
-                       , sequenceConduits contents' >> pure () ) -- .| mapM_C (printDebug "[parseFormatC] doc")
+                       , void (sequenceConduits contents') ) -- .| mapM_C (printDebug "[parseFormatC] doc")
     _ -> pure $ Left $ DT.intercalate "\n" errs
 parseFormatC _ _ _ = pure $ Left "Not implemented"
 
 
 filterZIPFileNameP :: FileType -> EntrySelector -> Bool
 filterZIPFileNameP Istex f = (takeExtension (unEntrySelector f) == ".json") &&
-                             ((unEntrySelector f) /= "manifest.json")
+                             (unEntrySelector f /= "manifest.json")
 filterZIPFileNameP _     _ = True
 
 
 etale :: [HyperdataDocument] -> [HyperdataDocument]
-etale = concat . (map etale')
+etale = concatMap etale'
   where
     etale' :: HyperdataDocument -> [HyperdataDocument]
     etale' h = map (\t -> h { _hd_abstract = Just t })
@@ -232,8 +232,6 @@ toDoc ff d = do
       let hd = HyperdataDocument { _hd_bdd = Just $ DT.pack $ show ff
                                  , _hd_doi = lookup "doi" d
                                  , _hd_url = lookup "URL" d
-                                 , _hd_uniqId = Nothing
-                                 , _hd_uniqIdBdd = Nothing
                                  , _hd_page = Nothing
                                  , _hd_title = lookup "title" d
                                  , _hd_authors = lookup "authors" d
@@ -293,7 +291,7 @@ runParser format text = pure $ runParser' format text
 
 runParser' :: FileType
           -> DB.ByteString
-          -> (Either Text [[(DB.ByteString, DB.ByteString)]])
+          -> Either Text [[(DB.ByteString, DB.ByteString)]]
 runParser' format text = first DT.pack $ parseOnly (withParser format) text
 
 openZip :: FilePath -> IO [DB.ByteString]
@@ -317,5 +315,5 @@ clean txt = DBC.map clean' txt
 --
 
 splitOn :: NgramsType -> Maybe Text -> Text -> [Text]
-splitOn Authors (Just "WOS") = (DT.splitOn "; ")
-splitOn _ _                  = (DT.splitOn ", ")
+splitOn Authors (Just "WOS") = DT.splitOn "; "
+splitOn _ _                  = DT.splitOn ", "

src/Gargantext/Core/Text/Corpus/Parsers/Book.hs

@@ -33,8 +33,8 @@ book2csv :: Int -> FileDir -> FileOut -> IO ()
 book2csv n f_in f_out = do
   files <- filesOf f_in
   texts <- readPublis f_in files
-  let publis = List.concat $ map (file2publi n) texts
-  let docs = map (\(y,p) -> publiToHyperdata y p) $ List.zip [1..] publis
+  let publis = concatMap (file2publi n) texts
+  let docs = zipWith publiToHyperdata [1..] publis
   DBL.writeFile f_out (hyperdataDocument2csv docs)
 
 filesOf :: FileDir -> IO [FilePath]
@@ -43,7 +43,7 @@ filesOf fd = List.sort                                        -- sort by filenam
         <$>  getDirectoryContents fd
 
 readPublis :: FileDir -> [FilePath] -> IO [(FilePath, Text)]
-readPublis fd fps = mapM (\fp -> DBL.readFile (fd <> fp) >>= \txt -> pure (fp, cs txt)) fps
+readPublis fd = mapM (\fp -> DBL.readFile (fd <> fp) >>= \txt -> pure (fp, cs txt))
 
 ------------------------------------------------------------------------
 -- Main Types
@@ -63,7 +63,7 @@ type FileDir = FilePath
 ---------------------------------------------------------------------
 
 file2publi :: Int -> (FilePath, Text) -> [Publi]
-file2publi n (fp,theText) = map (\(t,txt) -> Publi authors source t txt) theTexts
+file2publi n (fp,theText) = map (uncurry (Publi authors source)) theTexts
   where
     theTexts = text2titleParagraphs n theText
     FileInfo authors source = fileNameInfo fp
@@ -81,8 +81,6 @@ publiToHyperdata y (Publi a s t txt) =
        HyperdataDocument { _hd_bdd = Just "Book File"
                         , _hd_doi = Nothing
                         , _hd_url = Nothing
-                        , _hd_uniqId = Nothing
-                        , _hd_uniqIdBdd = Nothing
                         , _hd_page = Nothing
                         , _hd_title = Just t
                         , _hd_authors = Just (DT.concat a)

src/Gargantext/Core/Text/Corpus/Parsers/CSV.hs

@@ -14,8 +14,7 @@ CSV parser for Gargantext corpus files.
 
 module Gargantext.Core.Text.Corpus.Parsers.CSV where
 
-import Conduit
-import Control.Applicative
+import Conduit ( ConduitT, (.|), yieldMany, mapC )
 import Data.ByteString qualified as BS
 import Data.ByteString.Lazy qualified as BL
 import Data.Csv
@@ -24,9 +23,9 @@ import Data.Text qualified as T
 import Data.Time.Segment (jour)
 import Data.Vector (Vector)
 import Data.Vector qualified as V
-import Gargantext.Core.Text
-import Gargantext.Core.Text.Context
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Core.Text ( sentences, unsentences )
+import Gargantext.Core.Text.Context ( splitBy, SplitContext(..) )
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Prelude hiding (length, show)
 import Protolude
 
@@ -60,8 +59,6 @@ toDoc (CsvGargV3 did dt _ dpy dpm dpd dab dau) =
   HyperdataDocument { _hd_bdd = Just "CSV"
                     , _hd_doi = Just . pack . show $ did
                     , _hd_url = Nothing
-                    , _hd_uniqId = Nothing
-                    , _hd_uniqIdBdd = Nothing
                     , _hd_page = Nothing
                     , _hd_title = Just dt
                     , _hd_authors = Nothing
@@ -93,11 +90,11 @@ toDocs v = V.toList
                        (V.enumFromN 1 (V.length v'')) v''
           where
             v'' = V.foldl (\v' sep -> V.concatMap (splitDoc (docsSize v') sep) v') v seps
-            seps= (V.fromList [Paragraphs 1, Sentences 3, Chars 3])
+            seps= V.fromList [Paragraphs 1, Sentences 3, Chars 3]
 
 ---------------------------------------------------------------
 fromDocs :: Vector CsvGargV3 -> Vector CsvDoc
-fromDocs docs = V.map fromDocs' docs
+fromDocs = V.map fromDocs'
   where
     fromDocs' (CsvGargV3 { .. }) = CsvDoc { csv_title = d_title
                                           , csv_source = d_source
@@ -111,16 +108,11 @@ fromDocs docs = V.map fromDocs' docs
 -- | Split a document in its context
 -- TODO adapt the size of the paragraph according to the corpus average
 splitDoc :: Mean -> SplitContext -> CsvDoc -> Vector CsvDoc
-splitDoc m splt doc = let docSize = (T.length $ csv_abstract doc) in
-                          if docSize > 1000
-                            then
-                              if (mod (round m) docSize) >= 10
-                                then
-                                  splitDoc' splt doc
-                                else
-                                  V.fromList [doc]
-                            else
-                              V.fromList [doc]
+splitDoc m splt doc =
+ let docSize = (T.length $ csv_abstract doc) in
+ if (docSize > 1000) && (mod (round m) docSize >= 10)
+   then splitDoc' splt doc
+   else V.fromList [doc]
   where
     splitDoc' :: SplitContext -> CsvDoc -> Vector CsvDoc
     splitDoc' contextSize (CsvDoc { .. }) = V.fromList $ [firstDoc] <> nextDocs
@@ -152,7 +144,7 @@ unIntOrDec :: IntOrDec -> Int
 unIntOrDec (IntOrDec i) = i
 instance FromField IntOrDec where
   parseField s = case runParser (parseField s :: Parser Int) of
-    Left _err -> IntOrDec <$> floor <$> (parseField s :: Parser Double)
+    Left _err -> IntOrDec . floor <$> (parseField s :: Parser Double)
     Right n   -> pure $ IntOrDec n
 instance ToField IntOrDec where
   toField (IntOrDec i) = toField i
@@ -253,15 +245,15 @@ readByteStringStrict :: (FromNamedRecord a)
                      -> Delimiter
                      -> BS.ByteString
                      -> Either Text (Header, Vector a)
-readByteStringStrict d ff = (readByteStringLazy d ff) . BL.fromStrict
+readByteStringStrict d ff = readByteStringLazy d ff . BL.fromStrict
 
 ------------------------------------------------------------------------
 -- | TODO use readFileLazy
 readCSVFile :: FilePath -> IO (Either Text (Header, Vector CsvDoc))
 readCSVFile fp = do
-  result <- fmap (readCsvLazyBS Comma) $ BL.readFile fp
+  result <- readCsvLazyBS Comma <$> BL.readFile fp
   case result of
-    Left _err -> fmap (readCsvLazyBS Tab) $ BL.readFile fp
+    Left _err -> readCsvLazyBS Tab <$> BL.readFile fp
     Right res -> pure $ Right res
 
 
@@ -382,8 +374,6 @@ csvHal2doc (CsvHal { .. }) =
   HyperdataDocument { _hd_bdd = Just "CsvHal"
                     , _hd_doi = Just csvHal_doiId_s
                     , _hd_url = Just csvHal_url
-                    , _hd_uniqId = Nothing
-                    , _hd_uniqIdBdd = Nothing
                     , _hd_page = Nothing
                     , _hd_title = Just csvHal_title
                     , _hd_authors = Just csvHal_authors
@@ -407,8 +397,6 @@ csv2doc (CsvDoc { .. })
   = HyperdataDocument { _hd_bdd = Just "CsvHal"
                       , _hd_doi = Nothing
                       , _hd_url = Nothing
-                      , _hd_uniqId = Nothing
-                      , _hd_uniqIdBdd = Nothing
                       , _hd_page = Nothing
                       , _hd_title = Just csv_title
                       , _hd_authors = Just csv_authors
@@ -434,10 +422,10 @@ csv2doc (CsvDoc { .. })
 parseHal :: FilePath -> IO (Either Text [HyperdataDocument])
 parseHal fp = do
   r <- readCsvHal fp
-  pure $ (V.toList . V.map csvHal2doc . snd) <$> r
+  pure $ V.toList . V.map csvHal2doc . snd <$> r
 
 parseHal' :: BL.ByteString -> Either Text [HyperdataDocument]
-parseHal' bs = (V.toList . V.map csvHal2doc . snd) <$> readCsvHalLazyBS bs
+parseHal' bs = V.toList . V.map csvHal2doc . snd <$> readCsvHalLazyBS bs
 
 ------------------------------------------------------------------------
 
@@ -455,7 +443,7 @@ parseCsv' bs = do
     result = case readCsvLazyBS Comma bs of
       Left  _err -> readCsvLazyBS Tab bs
       Right res -> Right res
-  (V.toList . V.map csv2doc . snd) <$> result
+  V.toList . V.map csv2doc . snd <$> result
 
 parseCsvC :: BL.ByteString
           -> Either Text (Integer, ConduitT () HyperdataDocument Identity ())

src/Gargantext/Core/Text/Corpus/Parsers/Gitlab.hs

@@ -13,12 +13,12 @@ module Gargantext.Core.Text.Corpus.Parsers.Gitlab (
   Issue(..), gitlabIssue2hyperdataDocument, readFile_Issues, readFile_IssuesAsDocs
 ) where
 
-import Data.Aeson
+import Data.Aeson ( FromJSON(parseJSON), decode, (.:), (.:?), withObject )
 import Data.ByteString.Lazy qualified as DBL
 import Data.Text qualified as DT
 import Data.Time
 import Gargantext.Core (Lang(..))
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Prelude
 
 data Issue = Issue { _issue_id      :: !Int
@@ -42,8 +42,6 @@ gitlabIssue2hyperdataDocument issue = HyperdataDocument
     { _hd_bdd = Nothing
     , _hd_doi = Nothing
     , _hd_url = Nothing
-    , _hd_uniqId = Nothing
-    , _hd_uniqIdBdd = Nothing
     , _hd_page = Nothing
     , _hd_title = Just (_issue_title issue)
     , _hd_authors = Nothing

src/Gargantext/Core/Text/Corpus/Parsers/GrandDebat.hs

@@ -31,8 +31,8 @@ import Data.ByteString.Lazy qualified as DBL
 import Data.JsonStream.Parser qualified as P
 import Data.Text qualified as Text
 import Gargantext.Core (Lang(..))
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..), ToHyperdataDocument, toHyperdataDocument)
-import Gargantext.Database.GargDB
+import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..), ToHyperdataDocument, toHyperdataDocument)
+import Gargantext.Database.GargDB ( ReadFile(..) )
 import Gargantext.Prelude
 
 data GrandDebatReference = GrandDebatReference
@@ -43,14 +43,14 @@ data GrandDebatReference = GrandDebatReference
   , createdAt :: !(Maybe Text)
   , publishedAt   :: !(Maybe Text)
   , updatedAt     :: !(Maybe Text)
-  
+
   , trashed       :: !(Maybe Bool)
   , trashedStatus :: !(Maybe Text)
-  
+
   , authorId      :: !(Maybe Text)
   , authorType    :: !(Maybe Text)
   , authorZipCode :: !(Maybe Text)
-  
+
   , responses     :: !(Maybe [GrandDebatResponse])
   }
   deriving (Show, Generic)
@@ -77,8 +77,6 @@ instance ToHyperdataDocument GrandDebatReference
       HyperdataDocument { _hd_bdd = Just "GrandDebat"
                         , _hd_doi = id
                         , _hd_url = Nothing
-                        , _hd_uniqId = Nothing
-                        , _hd_uniqIdBdd = Nothing
                         , _hd_page = Nothing
                         , _hd_title = title
                         , _hd_authors = authorType
@@ -94,12 +92,10 @@ instance ToHyperdataDocument GrandDebatReference
                         , _hd_publication_second = Nothing
                         , _hd_language_iso2 = Just $ Text.pack $ show FR }
         where
-          toAbstract = (Text.intercalate " . ") . ((filter (/= "")) . (map toSentence))
+          toAbstract = Text.intercalate " . " . (filter (/= "") . map toSentence)
           toSentence (GrandDebatResponse _id _qtitle _qvalue r) = case r of
             Nothing -> ""
-            Just r' -> case Text.length r' > 10 of
-                True  -> r'
-                False -> ""
+            Just r' -> if Text.length r' > 10 then r' else ""
 
 instance ReadFile [GrandDebatReference]
   where

src/Gargantext/Core/Text/Corpus/Parsers/Isidore.hs

@@ -20,14 +20,14 @@ TODO:
 
 module Gargantext.Core.Text.Corpus.Parsers.Isidore where
 
-import Control.Lens hiding (contains)
+import Control.Lens ( (^.), (.~) )
 import Data.ByteString.Lazy (ByteString)
-import Data.RDF hiding (triple, Query)
+import Data.RDF ( Node(LNode, UNode), LValue(PlainLL, TypedL, PlainL) )
 import Data.Text qualified as T
-import Database.HSparql.Connection
+import Database.HSparql.Connection ( BindingValue(..), EndPoint, structureContent )
 import Database.HSparql.QueryGenerator
 import Gargantext.Core (Lang)
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
 import Gargantext.Prelude hiding (ByteString)
 import Network.Wreq (getWith, Response, defaults, header, param, responseStatus, responseBody)
 import Prelude qualified
@@ -115,7 +115,7 @@ unbound _ Unbound         = Nothing
 unbound _ (Bound (UNode x)) = Just x
 unbound _ (Bound (LNode (TypedL x _))) = Just x
 unbound _ (Bound (LNode (PlainL x)))   = Just x
-unbound l (Bound (LNode (PlainLL x l')))   = if l' == (T.toLower $ show l) then Just x else Nothing
+unbound l (Bound (LNode (PlainLL x l')))   = if l' == T.toLower (show l) then Just x else Nothing
 unbound _ _ = Nothing
 
 bind2doc :: Lang -> [BindingValue] -> HyperdataDocument
@@ -123,8 +123,6 @@ bind2doc l [ link', date, langDoc, authors, _source, publisher, title, abstract
   HyperdataDocument { _hd_bdd = Just "Isidore"
                     , _hd_doi = Nothing
                     , _hd_url = unbound l link'
-                    , _hd_uniqId = Nothing
-                    , _hd_uniqIdBdd = Nothing
                     , _hd_page = Nothing
                     , _hd_title = unbound l title
                     , _hd_authors = unbound l authors

src/Gargantext/Core/Text/Corpus/Parsers/JSON/Istex.hs

@@ -19,11 +19,10 @@ module Gargantext.Core.Text.Corpus.Parsers.JSON.Istex where
 import Data.Text qualified as T
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.Corpus.Parsers.Date qualified as Date
-import Gargantext.Database.Admin.Types.Hyperdata (HyperdataDocument(..))
+import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
 import Gargantext.Defaults qualified as Defaults
-import Gargantext.Prelude hiding (length, show)
+import Gargantext.Prelude hiding (length)
 import ISTEX.Client qualified as ISTEX
-import Protolude
 
 
 -- | TODO remove dateSplit here
@@ -37,13 +36,11 @@ toDoc la (ISTEX.Document i t a ab d s) = do
   pure $ HyperdataDocument { _hd_bdd       = Just "Istex"
                            , _hd_doi       = Just i
                            , _hd_url       = Nothing
-                           , _hd_uniqId    = Nothing
-                           , _hd_uniqIdBdd = Nothing
                            , _hd_page      = Nothing
                            , _hd_title     = t
-                           , _hd_authors = Just $ foldl (\x y -> if x == "" then y else x <> ", " <> y) "" (map ISTEX._author_name a)
-                           , _hd_institutes = Just $ foldl (\x y -> if x == "" then y else x <> ", " <> y) "" (concat $ (map ISTEX._author_affiliations) a)
-                           , _hd_source = Just $ foldl (\x y -> if x == "" then y else x <> ", " <> y) "" (ISTEX._source_title s)
+                           , _hd_authors = Just $ foldl' (\x y -> if x == "" then y else x <> ", " <> y) "" (map ISTEX._author_name a)
+                           , _hd_institutes = Just $ foldl' (\x y -> if x == "" then y else x <> ", " <> y) "" (concatMap ISTEX._author_affiliations a)
+                           , _hd_source = Just $ foldl' (\x y -> if x == "" then y else x <> ", " <> y) "" (ISTEX._source_title s)
                            , _hd_abstract = ab
                            , _hd_publication_date = fmap (T.pack . show) utctime
                            , _hd_publication_year = pub_year

src/Gargantext/Core/Text/Corpus/Parsers/Wikidata.hs

@@ -21,11 +21,11 @@ module Gargantext.Core.Text.Corpus.Parsers.Wikidata where
 import Control.Lens (makeLenses, (^.) )
 import Data.List qualified as List
 import Data.Text (concat)
-import Database.HSparql.Connection
+import Database.HSparql.Connection ( BindingValue, EndPoint, selectQueryRaw )
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Text.Corpus.Parsers.Date (mDateSplit)
 import Gargantext.Core.Text.Corpus.Parsers.Isidore (unbound)
-import Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler
+import Gargantext.Core.Text.Corpus.Parsers.Wikidata.Crawler ( crawlPage )
 import Gargantext.Database.Admin.Types.Hyperdata.Document (HyperdataDocument(..))
 import Gargantext.Prelude hiding (concat)
 import Prelude qualified
@@ -57,11 +57,9 @@ wikiPageToDocument m wr = do
 
   let bdd    = Just "wikidata"
       doi    = Nothing
-      url    = (wr ^. wr_url)
-      uniqId    = Nothing
-      uniqIdBdd = Nothing
+      url    = wr ^. wr_url
       page      = Nothing
-      title     = (wr ^. wr_title)
+      title     = wr ^. wr_title
       authors    = Nothing
       institutes = Nothing
       source     = Nothing
@@ -83,8 +81,6 @@ wikiPageToDocument m wr = do
   pure $ HyperdataDocument { _hd_bdd = bdd
                            , _hd_doi = doi
                            , _hd_url = url
-                           , _hd_uniqId = uniqId
-                           , _hd_uniqIdBdd = uniqIdBdd
                            , _hd_page = page
                            , _hd_title = title
                            , _hd_authors = authors

src/Gargantext/Core/Types.hs

@@ -167,7 +167,7 @@ instance Semigroup TokenTag where
 
 instance Monoid TokenTag where
   mempty = TokenTag [] empty Nothing Nothing
-  mconcat = foldl mappend mempty
+  mconcat = foldl' mappend mempty
   -- mappend t1 t2 = (<>) t1 t2
 
 

src/Gargantext/Core/Types/Search.hs

@@ -9,22 +9,21 @@ Portability : POSIX
 -}
 
 
-{-# LANGUAGE DeriveAnyClass           #-}
-
 module Gargantext.Core.Types.Search where
 
 import Data.Aeson hiding (defaultTaggedObject)
-import Data.Swagger hiding (fieldLabelModifier, Contact)
+import Data.Swagger ( ToSchema(..), genericDeclareNamedSchema )
 import Data.Text qualified as Text
 import Data.Time (UTCTime)
 import Gargantext.Core.Utils.Prefix (dropPrefix, unCapitalize, unPrefixSwagger)
-import Gargantext.Database.Admin.Types.Hyperdata (ContactWhere(..), HyperdataContact(..), HyperdataDocument(..), ContactWho(..))
-import Gargantext.Database.Admin.Types.Node
+import Gargantext.Database.Admin.Types.Hyperdata.Contact ( ContactWhere(..), HyperdataContact(..), ContactWho(..) )
+import Gargantext.Database.Admin.Types.Hyperdata.Document ( HyperdataDocument(..) )
+import Gargantext.Database.Admin.Types.Node ( NodeId )
 import Gargantext.Database.Query.Facet.Types (Facet(..), FacetDoc, FacetPaired(..))
 import Gargantext.Defaults qualified as Defaults
 import Gargantext.Prelude
 import Gargantext.Utils.Aeson (defaultTaggedObject)
-import Test.QuickCheck.Arbitrary
+import Test.QuickCheck.Arbitrary ( Arbitrary(arbitrary) )
 
 
 data Row =
@@ -93,8 +92,6 @@ data HyperdataRow =
                        , _hr_source             :: !Text
                        , _hr_title              :: !Text
                        , _hr_url                :: !Text
-                       , _hr_uniqId             :: !Text
-                       , _hr_uniqIdBdd          :: !Text
                        }
   | HyperdataRowContact { _hr_firstname :: !Text
                         , _hr_lastname  :: !Text
@@ -148,9 +145,7 @@ instance ToHyperdataRow HyperdataDocument where
       , _hr_publication_second = fromMaybe 0 _hd_publication_second
       , _hr_source = fromMaybe "" _hd_source
       , _hr_title = fromMaybe "Title" _hd_title
-      , _hr_url = fromMaybe "" _hd_url
-      , _hr_uniqId = fromMaybe "" _hd_uniqId
-      , _hr_uniqIdBdd = fromMaybe "" _hd_uniqIdBdd }
+      , _hr_url = fromMaybe "" _hd_url }
 
 instance ToHyperdataRow HyperdataContact where
   toHyperdataRow (HyperdataContact { _hc_who = Just (ContactWho _ fn ln _ _ _), _hc_where = ou} ) =

src/Gargantext/Database/Action/Flow/Types.hs

@@ -11,7 +11,6 @@ Portability : POSIX
 
 {-# LANGUAGE ConstraintKinds         #-}
 {-# LANGUAGE ConstrainedClassMethods #-}
-{-# LANGUAGE ConstraintKinds         #-}
 {-# LANGUAGE InstanceSigs            #-}
 {-# LANGUAGE TemplateHaskell         #-}
 
@@ -52,9 +51,7 @@ type FlowCmdM env err m =
   , MonadLogger m
   )
 
-type FlowCorpus a = ( AddUniqId      a
-                    , UniqId         a
-                    , UniqParameters a
+type FlowCorpus a = ( UniqParameters a
                     , InsertDb       a
                     , ExtractNgramsT a
                     , HasText        a

src/Gargantext/Database/Admin/Types/Hyperdata/Contact.hs

@@ -16,7 +16,6 @@ Portability : POSIX
 {-# LANGUAGE FlexibleContexts           #-}
 {-# LANGUAGE FunctionalDependencies     #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{-# LANGUAGE MultiParamTypeClasses      #-}
 {-# LANGUAGE NoImplicitPrelude          #-}
 {-# LANGUAGE OverloadedStrings          #-}
 {-# LANGUAGE RankNTypes                 #-}
@@ -32,7 +31,7 @@ import Gargantext.API.GraphQL.Utils qualified as GAGU
 import Gargantext.Core.Text (HasText(..))
 import Gargantext.Database.Admin.Types.Hyperdata.Prelude
 import Gargantext.Prelude
-import Gargantext.Utils.UTCTime
+import Gargantext.Utils.UTCTime ( NUTCTime(..) )
 
 --------------------------------------------------------------------------------
 data HyperdataContact =
@@ -42,8 +41,6 @@ data HyperdataContact =
                    , _hc_title  :: Maybe Text -- TODO remove (only demo)
                    , _hc_source :: Maybe Text -- TODO remove (only demo)
                    , _hc_lastValidation  :: Maybe Text -- TODO UTCTime
-                   , _hc_uniqIdBdd       :: Maybe Text
-                   , _hc_uniqId          :: Maybe Text
   } deriving (Eq, Show, Generic)
 
 instance GQLType HyperdataContact where
@@ -61,9 +58,7 @@ defaultHyperdataContact =
     , _hc_where = [defaultContactWhere]
     , _hc_title =Just "Title"
     , _hc_source = Just "Source"
-    , _hc_lastValidation = Just "TODO lastValidation date"
-    , _hc_uniqIdBdd = Just "DO NOT expose this"
-    , _hc_uniqId = Just "DO NOT expose this" }
+    , _hc_lastValidation = Just "TODO lastValidation date" }
 
 hyperdataContact :: FirstName -> LastName -> HyperdataContact
 hyperdataContact fn ln =
@@ -73,9 +68,7 @@ hyperdataContact fn ln =
     , _hc_where = []
     , _hc_title = Nothing
     , _hc_source = Nothing
-    , _hc_lastValidation = Nothing
-    , _hc_uniqIdBdd = Nothing
-    , _hc_uniqId = Nothing }
+    , _hc_lastValidation = Nothing }
 
 -- TOD0 contact metadata (Type is too flat)
 data ContactMetaData =
@@ -94,9 +87,7 @@ arbitraryHyperdataContact =
     , _hc_where = []
     , _hc_title = Nothing
     , _hc_source = Nothing
-    , _hc_lastValidation = Nothing
-    , _hc_uniqIdBdd = Nothing
-    , _hc_uniqId = Nothing }
+    , _hc_lastValidation = Nothing }
 
 
 data ContactWho = 
@@ -188,7 +179,7 @@ instance ToSchema ContactMetaData where
 
 -- | Arbitrary instances
 instance Arbitrary HyperdataContact where
-  arbitrary = elements [HyperdataContact Nothing Nothing [] Nothing Nothing Nothing Nothing Nothing]
+  arbitrary = elements [ HyperdataContact Nothing Nothing [] Nothing Nothing Nothing ]
 
 -- | Specific Gargantext instance
 instance Hyperdata HyperdataContact

src/Gargantext/Database/Admin/Types/Hyperdata/Document.hs

@@ -29,11 +29,6 @@ import Gargantext.Database.Admin.Types.Hyperdata.Prelude
 data HyperdataDocument = HyperdataDocument { _hd_bdd                :: !(Maybe Text)
                                            , _hd_doi                :: !(Maybe Text)
                                            , _hd_url                :: !(Maybe Text)
-                                           -- | Unique MD5 hash of the document
-                                           , _hd_uniqId             :: !(Maybe Text)
-                                           -- | Used as unique ID per source (can be same doc in Openalex, HAL, etc)
-                                           --   I think it's currently not used.
-                                           , _hd_uniqIdBdd          :: !(Maybe Text)
                                            , _hd_page               :: !(Maybe Int)
                                            , _hd_title              :: !(Maybe Text)
                                            , _hd_authors            :: !(Maybe Text)
@@ -61,7 +56,7 @@ instance HasText HyperdataDocument
 defaultHyperdataDocument :: HyperdataDocument
 defaultHyperdataDocument = case decode docExample of
   Just hp -> hp
-  Nothing -> HyperdataDocument Nothing Nothing Nothing Nothing
+  Nothing -> HyperdataDocument Nothing Nothing
                                Nothing Nothing Nothing Nothing
                                Nothing Nothing Nothing Nothing
                                Nothing Nothing Nothing Nothing
@@ -110,7 +105,8 @@ instance ToHyperdataDocument HyperdataDocument
 
 ------------------------------------------------------------------------
 instance Eq HyperdataDocument where
-  (==) h1 h2 = (==) (_hd_uniqId h1) (_hd_uniqId h2)
+  (==) h1 h2 = _hd_title h1 == _hd_title h2
+            && _hd_abstract h1 == _hd_abstract h2
 
 ------------------------------------------------------------------------
 instance Ord HyperdataDocument where
@@ -129,7 +125,7 @@ arbitraryHyperdataDocuments =
                             ] :: [(Text, Text)])
   where
     toHyperdataDocument' (t1,t2) =
-      HyperdataDocument Nothing Nothing Nothing Nothing Nothing Nothing (Just t1)
+      HyperdataDocument Nothing Nothing Nothing Nothing (Just t1)
                       Nothing Nothing (Just t2) Nothing Nothing Nothing Nothing Nothing
                       Nothing Nothing Nothing   Nothing
 

src/Gargantext/Database/Query/Table/Node/Document/Insert.hs

@@ -57,9 +57,6 @@ the concatenation of the parameters defined by @shaParameters@.
 module Gargantext.Database.Query.Table.Node.Document.Insert
   where
 
-import Control.Lens (set, view)
-import Control.Lens.Cons ( _head )
-import Control.Lens.Prism ( _Just )
 import Data.Aeson (toJSON, ToJSON)
 import Data.Text qualified as DT (pack, concat, take, filter, toLower)
 import Data.Time.Segment (jour)
@@ -108,18 +105,18 @@ instance InsertDb HyperdataDocument
                       , toField p
                       , toField $ maybe "No Title" (DT.take 255)  (_hd_title h)
                       , toField $ _hd_publication_date h -- TODO USE UTCTime
-                      , (toField . toJSON) (addUniqId h)
+                      -- , (toField . toJSON) (addUniqId h)
                       ]
 
 instance InsertDb HyperdataContact
   where
-    insertDb' u p h = [ toField ("" :: Text)
+    insertDb' u p _h = [ toField ("" :: Text)
                       , toField $ toDBid NodeContact
                       , toField u
                       , toField p
                       , toField $ maybe "Contact" (DT.take 255) (Just "Name") -- (_hc_name h)
                       , toField $ jour 0 1 1 -- TODO put default date
-                      , (toField . toJSON) (addUniqId h)
+                      -- , (toField . toJSON) (addUniqId h)
                       ]
 
 instance ToJSON a => InsertDb (Node a)
@@ -194,39 +191,39 @@ class AddUniqId a
   where
     addUniqId :: a -> a
 
-class UniqParameters a
-  where
-    uniqParameters :: ParentId -> a -> Text
+-- instance AddUniqId HyperdataDocument
+--   where
+--     addUniqId = addUniqIdsDoc
+--       where
+--         addUniqIdsDoc :: HyperdataDocument -> HyperdataDocument
+--         addUniqIdsDoc doc = set hd_uniqIdBdd (Just shaBdd)
+--                           $ set hd_uniqId    (Just shaUni) doc
+--           where
+--             shaUni = hash $ DT.concat $ map ($ doc) shaParametersDoc
+--             shaBdd = hash $ DT.concat $ map ($ doc) ([maybeText . _hd_bdd] <> shaParametersDoc)
+
+--             shaParametersDoc :: [HyperdataDocument -> Text]
+--             shaParametersDoc = [ filterText . maybeText . _hd_title
+--                                , filterText . maybeText . _hd_abstract
+--                                , filterText . maybeText . _hd_source
+--                         --       , \d -> maybeText (_hd_publication_date d)
+--                                ]
 
-instance AddUniqId HyperdataDocument
+class UniqParameters a
   where
-    addUniqId = addUniqIdsDoc
-      where
-        addUniqIdsDoc :: HyperdataDocument -> HyperdataDocument
-        addUniqIdsDoc doc = set hd_uniqIdBdd (Just shaBdd)
-                          $ set hd_uniqId    (Just shaUni) doc
-          where
-            shaUni = hash $ DT.concat $ map ($ doc) shaParametersDoc
-            shaBdd = hash $ DT.concat $ map ($ doc) ([maybeText . _hd_bdd] <> shaParametersDoc)
-
-            shaParametersDoc :: [HyperdataDocument -> Text]
-            shaParametersDoc = [ filterText . maybeText . _hd_title
-                               , filterText . maybeText . _hd_abstract
-                               , filterText . maybeText . _hd_source
-                        --       , \d -> maybeText (_hd_publication_date d)
-                               ]
+    uniqParameters :: a -> Text
 
 instance UniqParameters HyperdataDocument
   where
-    uniqParameters _ h = filterText $  DT.concat $ map maybeText $ [_hd_title h, _hd_abstract h, _hd_source h]
+    uniqParameters h = filterText $  DT.concat $ map maybeText $ [_hd_title h, _hd_abstract h, _hd_source h]
 
 instance UniqParameters HyperdataContact
   where
-    uniqParameters _ _ = ""
+    uniqParameters _ = ""
 
 instance UniqParameters (Node a)
   where
-    uniqParameters _ _ = undefined
+    uniqParameters _ = undefined
 
 
 filterText :: Text -> Text
@@ -237,30 +234,30 @@ instance (UniqParameters a, ToJSON a, HasDBid NodeType) => AddUniqId (Node a)
   where
     addUniqId (Node nid _ t u p n d h)  = Node nid (Just newHash) t u p n d h
       where
-        newHash = "\\x" <> hash (uniqParameters (fromMaybe 0 p) h)
+        newHash = "\\x" <> hash (uniqParameters h)
 
 
     ---------------------------------------------------------------------------
 -- * Uniqueness of document definition
 -- TODO factorize with above (use the function below for tests)
 
-instance AddUniqId HyperdataContact
-  where
-    addUniqId = addUniqIdsContact
-
-addUniqIdsContact :: HyperdataContact -> HyperdataContact
-addUniqIdsContact hc = set hc_uniqIdBdd (Just shaBdd)
-                     $ set hc_uniqId    (Just shaUni) hc
-  where
-    shaUni = hash $ DT.concat $ map ($ hc) shaParametersContact
-    shaBdd = hash $ DT.concat $ map ($ hc) ([maybeText . view hc_bdd] <> shaParametersContact)
-
-    -- | TODO add more shaparameters
-    shaParametersContact :: [HyperdataContact -> Text]
-    shaParametersContact = [ maybeText . view (hc_who   . _Just . cw_firstName              )
-                           , maybeText . view (hc_who   . _Just . cw_lastName               )
-                           , maybeText . view (hc_where . _head . cw_touch . _Just . ct_mail)
-                           ]
+-- instance AddUniqId HyperdataContact
+--   where
+--     addUniqId = addUniqIdsContact
+
+-- addUniqIdsContact :: HyperdataContact -> HyperdataContact
+-- addUniqIdsContact hc = set hc_uniqIdBdd (Just shaBdd)
+--                      $ set hc_uniqId    (Just shaUni) hc
+--   where
+--     shaUni = hash $ DT.concat $ map ($ hc) shaParametersContact
+--     shaBdd = hash $ DT.concat $ map ($ hc) ([maybeText . view hc_bdd] <> shaParametersContact)
+
+--     -- | TODO add more shaparameters
+--     shaParametersContact :: [HyperdataContact -> Text]
+--     shaParametersContact = [ maybeText . view (hc_who   . _Just . cw_firstName              )
+--                            , maybeText . view (hc_who   . _Just . cw_lastName               )
+--                            , maybeText . view (hc_where . _head . cw_touch . _Just . ct_mail)
+--                            ]
 
 
 maybeText :: Maybe Text -> Text

test/Test/Database/Operations/DocumentSearch.hs

@@ -65,7 +65,6 @@ exampleDocument_01 = either error id $ parseEither parseJSON $ [aesonQQ|
 exampleDocument_02 :: HyperdataDocument
 exampleDocument_02 = either error id $ parseEither parseJSON $ [aesonQQ|
   { "doi":""
-  , "uniqId": "1405.3072v3"
   , "bdd": "Arxiv"
   , "publication_day":6
   , "language_iso2":"EN"
@@ -90,7 +89,6 @@ exampleDocument_03 = either error id $ parseEither parseJSON $ [aesonQQ|
   , "url": "http://arxiv.org/pdf/1405.3072v2"
   , "title": "Haskell for OCaml programmers"
   , "source": ""
-  , "uniqId": "1405.3072v2"
   , "authors": "Raphael Poss, Herbert Ballerina"
   , "abstract": "  This introduction to Haskell is written to optimize learning by programmers who already know OCaml. "
   , "institutes": ""
@@ -108,7 +106,6 @@ exampleDocument_04 = either error id $ parseEither parseJSON $ [aesonQQ|
   , "url": "http://arxiv.org/pdf/1407.5670v1"
   , "title": "Rust for functional programmers"
   , "source": ""
-  , "uniqId": "1407.5670v1"
   , "authors": "Raphael Poss"
   , "abstract": "  This article provides an introduction to Rust , a systems language by Mozilla , to programmers already familiar with Haskell , OCaml or other functional languages. " , "institutes": ""
   , "language_iso2": "EN"