Merge branch 'dev' into dev-file-upload

gargantext.ini_toModify

 [gargantext]
+# Needed to instantiate the first users and first data
 MASTER_USER = gargantua
 
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY      = PASSWORD_TO_CHANGE
 
-# Frames
+# Data path to local files
+DATA_FILEPATH = FILEPATH_TO_CHANGE
+
+# FRAMES
 FRAME_WRITE_URL = URL_TO_CHANGE
 FRAME_CALC_URL  = URL_TO_CHANGE
 
+FRAME_SEARX_URL  = URL_TO_CHANGE
+FRAME_ISTEX_URL  = URL_TO_CHANGE
+
 [network]
 # Emails From address (sent by smtp)
 MAIL = username@gargantext.org

src/Gargantext/Config.hs

@@ -22,12 +22,17 @@ import GHC.Generics (Generic)
 import Control.Lens (makeLenses)
 
 
-data GargConfig = GargConfig { _gc_masteruser :: Text
-                             , _gc_secretkey  :: Text
-                             , _gc_frame_write_url :: Text
-                             , _gc_frame_calc_url  :: Text
+data GargConfig = GargConfig { _gc_masteruser       :: !Text
+                             , _gc_secretkey        :: !Text
+                             , _gc_datafilepath     :: !Text
+
+                             , _gc_frame_write_url  :: !Text
+                             , _gc_frame_calc_url   :: !Text
+
+                             , _gc_frame_searx_url  :: !Text
+                             , _gc_frame_istex_url  :: !Text
                              }
-  deriving (Generic)
+  deriving (Generic, Show)
 
 makeLenses ''GargConfig
 
@@ -45,8 +50,17 @@ readConfig fp = do
 
   pure $ GargConfig (val "MASTER_USER")
                     (val "SECRET_KEY")
+                    (val "DATA_FILEPATH")
                     (val "FRAME_WRITE_URL")
                     (val "FRAME_CALC_URL")
+                    (val "FRAME_SEARX_URL")
+                    (val "FRAME_ISTEX_URL")
 
 defaultConfig :: GargConfig
-defaultConfig = GargConfig "gargantua" "secret" "https://frame_write.url" "https://frame_calc.url"
+defaultConfig = GargConfig "gargantua"
+                           "secret"
+                           "data/"
+                           "https://frame_write.url"
+                           "https://frame_calc.url"
+                           "https://frame_searx.url"
+                           "https://frame_istex.url"

src/Gargantext/Database/Action/Metrics/NgramsByNode.hs

@@ -19,14 +19,12 @@ module Gargantext.Database.Action.Metrics.NgramsByNode
 import Data.Map.Strict (Map, fromListWith, elems, toList, fromList)
 import Data.Map.Strict.Patch (PatchMap, Replace, diff)
 import Data.Set (Set)
-import qualified Data.Ord as DO (Down(..))
 import Data.Text (Text)
 import Data.Tuple.Extra (second, swap)
 import Database.PostgreSQL.Simple.SqlQQ (sql)
 import Database.PostgreSQL.Simple.Types (Values(..), QualifiedIdentifier(..))
 import Debug.Trace (trace)
 import Gargantext.Core (Lang(..))
-import Gargantext.Core.Types (Ordering(..))
 import Gargantext.Database.Admin.Config (nodeTypeId)
 import Gargantext.Database.Admin.Types.Node -- (ListId, CorpusId, NodeId)
 import Gargantext.Database.Prelude (Cmd, runPGSQuery)
@@ -58,12 +56,6 @@ ngramsGroup l _m _n = Text.intercalate " "
                   . Text.replace "-" " "
 
 
-sortTficf :: Ordering
-          -> (Map Text (Double, Set Text))
-          -> [   (Text,(Double, Set Text))]
-sortTficf Down = List.sortOn (DO.Down . fst . snd) . toList
-sortTficf Up   = List.sortOn (fst . snd) . toList
-
 
 getTficf :: UserCorpusId
          -> MasterCorpusId

src/Gargantext/Prelude/Utils.hs

@@ -41,6 +41,14 @@ data NodeToHash = NodeToHash { nodeType :: NodeType
 type FolderPath = FilePath
 type FileName   = FilePath
 
+-- | toPath example of use:
+-- toPath 2 "gargantexthello"
+-- ("ga/rg","antexthello")
+-- 
+-- toPath 3 "gargantexthello"
+-- ("gar/gan","texthello")
+
+
 toPath :: Int -> Text -> (FolderPath,FileName)
 toPath n x = (Text.unpack $ Text.intercalate "/" [x1,x2], Text.unpack xs)
   where

src/Gargantext/Text/List.hs

@@ -21,7 +21,8 @@ import Gargantext.API.Ngrams (NgramsElement, mkNgramsElement, RootParent(..), mS
 -- import Gargantext.API.Ngrams.Tools (getCoocByNgrams', Diagonal(..))
 import Gargantext.Core (Lang(..))
 import Gargantext.Core.Types (ListType(..), MasterCorpusId, UserCorpusId, Ordering(..))
-import Gargantext.Database.Action.Metrics.NgramsByNode (getTficf, sortTficf, ngramsGroup, getNodesByNgramsUser, groupNodesByNgramsWith)
+import Gargantext.Database.Action.Metrics.NgramsByNode (getTficf, ngramsGroup, getNodesByNgramsUser, groupNodesByNgramsWith)
+import Gargantext.Text.Metrics.TFICF (sortTficf)
 import Gargantext.Database.Prelude (Cmd)
 import Gargantext.Database.Schema.Ngrams (NgramsType(..))
 import Gargantext.Prelude
@@ -199,7 +200,6 @@ toGargList stop l n = case stop n of
     False -> (l, n)
 
 
-
 isStopTerm :: StopSize -> Text -> Bool
 isStopTerm (StopSize n) x = Text.length x < n || any isStopChar (Text.unpack x)
   where

src/Gargantext/Text/Metrics/TFICF.hs

@@ -19,14 +19,20 @@ module Gargantext.Text.Metrics.TFICF ( TFICF
                                      , Total(..)
                                      , Count(..)
                                      , tficf
+                                     , sortTficf
                                      )
   where
 
 import Data.Text (Text)
 import Gargantext.Prelude
+import Data.Set (Set)
+import Gargantext.Core.Types (Ordering(..))
+import Data.Map.Strict (Map, toList)
+import qualified Data.Ord as DO (Down(..))
+import qualified Data.List as List
 
 path :: Text
-path = "Gargantext.Text.Metrics.TFICF"
+path = "[G.T.Metrics.TFICF]"
 
 type TFICF = Double
 
@@ -42,8 +48,14 @@ tficf :: TficfContext Count Total
       -> TFICF
 tficf (TficfInfra (Count ic) (Total it) )
       (TficfSupra (Count sc) (Total st) )
-            | it >= ic && st >= sc = (ic/it) / log (sc/st)
+            | it >= ic && st >= sc && it <= st = (ic/it) / log (sc/st)
             | otherwise            = panic $ "[ERR]" <> path <>" Frequency impossible"
 tficf _ _ = panic $ "[ERR]" <> path <> "Undefined for these contexts"
 
 
+sortTficf :: Ordering
+          -> (Map Text (Double, Set Text))
+          -> [   (Text,(Double, Set Text))]
+sortTficf Down = List.sortOn (DO.Down . fst . snd) . toList
+sortTficf Up   = List.sortOn (fst . snd) . toList
+