[ngrams] fixes for list JSON import

src/Gargantext/API/Admin/Auth.hs

@@ -173,7 +173,7 @@ type ForgotPasswordAPI = Summary "Forgot password POST API"
                            :> QueryParam "uuid" Text
                            :> Get '[JSON] ForgotPasswordGet
 
-  
+
 forgotPassword :: GargServer ForgotPasswordAPI
      -- => ForgotPasswordRequest -> Cmd' env err ForgotPasswordResponse
 forgotPassword = forgotPasswordPost :<|> forgotPasswordGet
@@ -211,18 +211,18 @@ forgotPasswordGetUser :: (HasSettings env, HasConnectionPool env, HasJoseError e
 forgotPasswordGetUser (UserLight { .. }) = do
   -- pick some random password
   password <- liftBase gargPass
-  
+
   -- set it as user's password
   hashed <- liftBase $ Auth.hashPassword $ Auth.mkPassword password
   let hashed' = Auth.unPasswordHash hashed
   let userPassword = UserLight { userLight_password = GargPassword hashed', .. }
   _ <- updateUserPassword userPassword
-  
+
   -- display this briefly in the html
-  
+
   -- clear the uuid so that the page can't be refreshed
   _ <- updateUserForgotPasswordUUID $ UserLight { userLight_forgot_password_uuid = Nothing, .. }
-    
+
   pure $ ForgotPasswordGet password
 
 forgotUserPassword :: (HasConnectionPool env, HasConfig env, HasMail env)

src/Gargantext/API/Ngrams/List.hs

@@ -82,10 +82,10 @@ type JSONAPI = Summary "Update List"
           :> "add"
           :> "form"
           :> "async"
-            :> AsyncJobs JobLog '[FormUrlEncoded] WithFile JobLog
+            :> AsyncJobs JobLog '[FormUrlEncoded] WithJsonFile JobLog
 
 jsonApi :: ServerT JSONAPI (GargM Env GargError)
-jsonApi = postAsync
+jsonApi = jsonPostAsync
 
 ----------------------
 type CSVAPI = Summary "Update List (legacy v3 CSV)"
@@ -203,14 +203,8 @@ toIndexedNgrams m t = Indexed <$> i <*> n
     n = Just (text2ngrams t)
 
 ------------------------------------------------------------------------
-type PostAPI = Summary "Update List"
-        :> "add"
-        :> "form"
-        :> "async"
-        :> AsyncJobs JobLog '[FormUrlEncoded] WithFile JobLog
-
-postAsync :: ListId -> ServerT PostAPI (GargM Env GargError)
-postAsync lId =
+jsonPostAsync :: ServerT JSONAPI (GargM Env GargError)
+jsonPostAsync lId =
   serveJobsAPI UpdateNgramsListJobJSON $ \f log' ->
       let
         log'' x = do
@@ -220,10 +214,10 @@ postAsync lId =
 
 postAsync' :: FlowCmdM env err m
           => ListId
-          -> WithFile
+          -> WithJsonFile
           -> (JobLog -> m ())
           -> m JobLog
-postAsync' l (WithFile _ m _) logStatus = do
+postAsync' l (WithJsonFile m _) logStatus = do
 
   logStatus JobLog { _scst_succeeded = Just 0
                    , _scst_failed    = Just 0
@@ -235,10 +229,10 @@ postAsync' l (WithFile _ m _) logStatus = do
   -- printDebug "Done" r
 
   logStatus JobLog { _scst_succeeded = Just 1
-              , _scst_failed    = Just 0
-              , _scst_remaining = Just 1
-              , _scst_events    = Just []
-              }
+                   , _scst_failed    = Just 0
+                   , _scst_remaining = Just 1
+                   , _scst_events    = Just []
+                   }
 
 
   corpus_node <- getNode l -- (Proxy :: Proxy HyperdataList)
@@ -253,12 +247,6 @@ postAsync' l (WithFile _ m _) logStatus = do
 
 
 ------------------------------------------------------------------------
-type CSVPostAPI = Summary "Update List (legacy v3 CSV)"
-        :> "csv"
-        :> "add"
-        :> "form"
-        :> "async"
-        :> AsyncJobs JobLog '[FormUrlEncoded] WithFile JobLog
 
 readCsvText :: Text -> [(Text, Text, Text)]
 readCsvText t = case eDec of

src/Gargantext/API/Ngrams/List/Types.hs

@@ -18,10 +18,12 @@ module Gargantext.API.Ngrams.List.Types where
 
 --import Control.Lens hiding (elements, Indexed)
 import Data.Aeson
+import qualified Data.ByteString.Lazy as BSL
 import Data.Swagger (ToSchema, declareNamedSchema, genericDeclareNamedSchema)
 import Data.Text
+import qualified Data.Text.Encoding as E
 import Servant.Job.Utils (jsonOptions)
-import Web.FormUrlEncoded (FromForm, ToForm)
+import Web.FormUrlEncoded (FromForm(..), ToForm, parseUnique)
 
 import Protolude
 
@@ -29,7 +31,7 @@ import Gargantext.API.Ngrams.Types (NgramsList)
 import Gargantext.API.Node.Corpus.New.Types (FileType(..))
 import Gargantext.Core.Utils.Prefix (unPrefixSwagger)
 
-------------------------------------------------------------------------
+
 
 data WithFile = WithFile
   { _wf_filetype :: !FileType
@@ -38,7 +40,7 @@ data WithFile = WithFile
   } deriving (Eq, Show, Generic)
 
 --makeLenses ''WithFile
-instance FromForm WithFile
+instance FromForm WithFile where
 instance ToForm WithFile
 instance FromJSON WithFile where
   parseJSON = genericParseJSON $ jsonOptions "_wf_"
@@ -47,6 +49,30 @@ instance ToJSON WithFile where
 instance ToSchema WithFile where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wf_")
 
+------------------------------------------------------------------------
+
+data WithJsonFile = WithJsonFile
+  { _wjf_data     :: !NgramsList
+  , _wjf_name     :: !Text
+  } deriving (Eq, Show, Generic)
+
+instance FromForm WithJsonFile where
+  fromForm f = do
+    d' <- parseUnique "_wjf_data" f
+    d <- case eitherDecode' (BSL.fromStrict $ E.encodeUtf8 d') of
+      Left s -> Left $ pack s
+      Right v -> Right v
+    n <- parseUnique "_wjf_name" f
+    pure $ WithJsonFile { _wjf_data = d
+                        , _wjf_name = n }
+instance ToForm WithJsonFile
+instance FromJSON WithJsonFile where
+  parseJSON = genericParseJSON $ jsonOptions "_wjf_"
+instance ToJSON WithJsonFile where
+  toJSON = genericToJSON $ jsonOptions "_wjf_"
+instance ToSchema WithJsonFile where
+  declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wjf_")
+
 
 ------------------------------------------------------------------------
 
@@ -65,4 +91,3 @@ instance ToJSON WithTextFile where
   toJSON = genericToJSON $ jsonOptions "_wtf_"
 instance ToSchema WithTextFile where
   declareNamedSchema = genericDeclareNamedSchema (unPrefixSwagger "_wtf_")
-

src/Gargantext/API/Node/Corpus/Annuaire.hs

@@ -88,4 +88,3 @@ addToAnnuaireWithForm _cid (AnnuaireWithForm { _wf_filetype }) logStatus = do
                    , _scst_remaining = Just 0
                    , _scst_events    = Just []
                    }
-

src/Gargantext/API/Node/Corpus/Searx.hs

@@ -175,7 +175,7 @@ triggerSearxSearch user cId q l logStatus = do
       listId <- getOrMkList cId uId
       pure listId
     Just listId -> pure listId
-    
+
   printDebug "[triggerSearxSearch] listId" listId
 
   manager <- liftBase $ newManager tlsManagerSettings
@@ -220,4 +220,3 @@ hyperdataDocumentFromSearxResult l (SearxResult { _sr_content, _sr_engine, _sr_p
                           , _hd_publication_minute = Nothing
                           , _hd_publication_second = Nothing
                           , _hd_language_iso2 = Just $ T.pack $ show l }
-

src/Gargantext/API/Node/Update.hs

@@ -271,7 +271,7 @@ instance FromJSON  UpdateNodeParams where
 
 instance ToJSON    UpdateNodeParams where
   toJSON = genericToJSON (defaultOptions { sumEncoding = GUA.defaultTaggedObject })
-  
+
 instance ToSchema  UpdateNodeParams
 instance Arbitrary UpdateNodeParams where
   arbitrary = do

src/Gargantext/Database/Schema/Ngrams.hs

@@ -89,11 +89,12 @@ data NgramsType = Authors | Institutes | Sources | NgramsTerms
 instance Serialise NgramsType
 instance FromJSON NgramsType
   where
-    parseJSON (String "Authors")    = pure Authors
-    parseJSON (String "Institutes") = pure Institutes
-    parseJSON (String "Sources")    = pure Sources
-    parseJSON (String "Terms")      = pure NgramsTerms
-    parseJSON _                     = mzero
+    parseJSON (String "Authors")     = pure Authors
+    parseJSON (String "Institutes")  = pure Institutes
+    parseJSON (String "Sources")     = pure Sources
+    parseJSON (String "Terms")       = pure NgramsTerms
+    parseJSON (String "NgramsTerms") = pure NgramsTerms
+    parseJSON _                      = mzero
 
 instance FromJSONKey NgramsType where
    fromJSONKey = FromJSONKeyTextParser (parseJSON . String)