Skip to content

H
haskell-gargantext
Commit 4a80fa7b authored by Alexandre Delanoë's avatar Alexandre Delanoë
Browse files
Options

[MIGRATION] gargantext-upgrade can be used.

parent cfbd39f8
Show whitespace changes
Inline Side-by-side
Showing with 275 additions and 45 deletions
bin/gargantext-upgrade/Main.hs
View file @ 4a80fa7b
......@@ -16,65 +16,57 @@ Import a corpus binary.
module Main where
import Data.Text (Text)
import Data.Either (Either(..))
import Database.PostgreSQL.Simple.SqlQQ (sql)
import GHC.IO.Exception (IOException)
import Gargantext.API.Admin.EnvTypes (DevEnv)
import Gargantext.API.Dev (withDevEnv, runCmdDev)
import Gargantext.API.Prelude (GargError)
import Gargantext.API.Node () -- instances only
-- import Gargantext.API.Ngrams.Tools (getRepo)
import Gargantext.Database.Query.Table.Node (getOrMkList)
import Data.Either (Either(..))
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataCorpus)
import Gargantext.Database.Prelude (Cmd'', )
import Gargantext.Database.Action.Flow (getOrMkRoot, getOrMk_RootWithCorpus)
import Gargantext.Core.NodeStory
import Gargantext.Core.Types.Individu (User(..), arbitraryNewUsers, NewUser(..), arbitraryUsername, GargPassword(..))
import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(..), readConfig)
import System.Environment (getArgs)
import Prelude (getLine)
import GHC.IO.Exception (IOException)
import Database.PostgreSQL.Simple.SqlQQ (sql)
import Gargantext.Database.Prelude (Cmd, execPGSQuery)
import Gargantext.API.Prelude (GargError)
import Gargantext.Core (HasDBid(toDBid))
import Gargantext.Core.Types.Individu (User(..))
import Gargantext.Database.Action.Flow (getOrMk_RootWithCorpus)
import Gargantext.Database.Admin.Config (userMaster, corpusMasterName)
import Gargantext.Database.Admin.Trigger.Init
import Gargantext.Database.Admin.Types.Hyperdata (HyperdataCorpus)
import Gargantext.Database.Admin.Types.Node (NodeType(NodeDocument, NodeContact))
import Gargantext.Database.Prelude (Cmd'', Cmd, execPGSQuery)
import Gargantext.Database.Query.Table.Node (getOrMkList)
import Gargantext.Prelude
import Gargantext.Prelude.Config (GargConfig(..), readConfig)
import Prelude (getLine)
import System.Environment (getArgs)
import qualified Data.List as List (cycle, concat, take, unlines)
main :: IO ()
main = do
putStrLn "Manual method:"
putStrLn "Upgrade your GarganText instance with the script:"
putStrLn "./bin/psql gargantext.ini < devops/postgres/upgrade/0.0.5/schema.sql"
putStrLn "Then press enter key when you are done."
_ok <- getLine
[iniPath] <- getArgs
cfg <- readConfig iniPath
let secret = _gc_secretkey cfg
let ___ = putStrLn $ List.concat $ List.take 72 $ List.cycle ["_"]
___
putStrLn "GarganText upgrade to version 0.0.5"
___
let
sqlUpdateTriggerHash :: Cmd'' DevEnv IOException Int64
sqlUpdateTriggerHash = do
execPGSQuery query ()
where
query = [sql|
UPDATE nodes SET typename = typename;
|]
params@[iniPath] <- getArgs
_ <- if length params /= 1
then panic "Usage: ./gargantext-upgrade gargantext.ini"
else pure ()
let
sqlNodes2Context :: Cmd'' DevEnv IOException Int64
sqlNodes2Context = do
execPGSQuery query ()
where
query = [sql|
UPDATE nodes SET typename = typename;
|]
putStrLn $ List.unlines
[ "Your Database defined in gargantext.ini will be upgraded."
, "We stronlgy recommend you to make a backup using pg_dump."
, ""
, "If you encounter issues, please report your bugs here:"
, "https://gitlab.iscpif.fr/gargantext/haskell-gargantext/issues/101"
, ""
, "Press ENTER if you want to continue, CTRL+C if you want to stop."
]
_ok <- getLine
cfg <- readConfig iniPath
let secret = _gc_secretkey cfg
let
contextsTriggers :: Cmd GargError ()
......@@ -89,8 +81,179 @@ main = do
withDevEnv iniPath $ \env -> do
-- First upgrade the Database Schema
_ <- runCmdDev env sqlSchema
-- Then upgrade the triggers
_ <- runCmdDev env (initFirstTriggers secret :: Cmd GargError [Int64])
_ <- runCmdDev env (contextsTriggers :: Cmd GargError ())
-- Move nodes to contexts table
_ <- runCmdDev env sqlNodes2Contexts
-- Update the hashes
_ <- runCmdDev env sqlUpdateTriggerHash
putStrLn "Uprade done with success"
___
putStrLn "Uprade done with success !"
___
pure ()
sqlUpdateTriggerHash :: Cmd'' DevEnv IOException Int64
sqlUpdateTriggerHash = do
execPGSQuery query ()
where
query = [sql|
UPDATE nodes SET typename = typename;
UPDATE contexts SET typename = typename;
|]
sqlNodes2Contexts :: Cmd'' DevEnv IOException Int64
sqlNodes2Contexts = do
execPGSQuery query (toDBid NodeDocument,toDBid NodeContact)
where
query = [sql|
-- WITH docs (id,hash_id,typename,user_id,parent_id,name,date,hyperdata, search)
WITH docs AS (SELECT * from nodes WHERE nodes.typename IN (?,?)),
inserted (id, hash_id) AS (
INSERT INTO contexts (hash_id,typename,user_id,parent_id,name,date,hyperdata, search)
SELECT d.hash_id,d.typename,d.user_id,NULL,d.name,d.date,d.hyperdata,search FROM docs AS d
RETURNING contexts.id, contexts.hash_id
),
indexed (node_id, context_id) AS (
SELECT docs.id, inserted.id from inserted
JOIN docs on docs.hash_id = inserted.hash_id
),
-- nodes_nodes -> nodes_contexts
nodes_contexts_query AS (
INSERT INTO nodes_contexts (node_id, context_id,score, category)
SELECT nn.node1_id,i.context_id,nn.score,nn.category FROM nodes_nodes nn
JOIN indexed i ON i.node_id = nn.node2_id
),
-- nodes_nodes_ngrams -> contexts_nodes_ngrams
contexts_nodes_ngrams_query AS (
INSERT INTO context_node_ngrams
SELECT i.context_id, nnn.node1_id, nnn.ngrams_id, nnn.ngrams_type, nnn.weight FROM node_node_ngrams nnn
JOIN indexed i ON i.node_id = nnn.node2_id
),
---- nodes_nodes_ngrams2 -> contexts_nodes_ngrams2
context_node_ngrams2_query AS (
INSERT INTO context_node_ngrams2
SELECT i.context_id, nnn2.nodengrams_id, nnn2.weight FROM node_node_ngrams2 nnn2
JOIN indexed i ON i.node_id = nnn2.node_id
)
-- WITH CASCADE it should update others tables
DELETE FROM nodes n
USING indexed i WHERE i.node_id = n.id
;
UPDATE contexts SET parent_id = id;
|]
sqlSchema :: Cmd'' DevEnv IOException Int64
sqlSchema = do
execPGSQuery query ()
where
query = [sql|
-- TODO typename -> type_id
CREATE TABLE public.contexts (
id SERIAL,
hash_id CHARACTER varying(66) DEFAULT ''::character varying NOT NULL,
typename INTEGER NOT NULL,
user_id INTEGER NOT NULL,
parent_id INTEGER REFERENCES public.contexts(id) ON DELETE CASCADE ,
name CHARACTER varying(255) DEFAULT ''::character varying NOT NULL,
date TIMESTAMP with time zone DEFAULT now() NOT NULL,
hyperdata jsonb DEFAULT '{}'::jsonb NOT NULL,
search tsvector,
PRIMARY KEY (id),
FOREIGN KEY (user_id) REFERENCES public.auth_user(id) ON DELETE CASCADE
);
ALTER TABLE public.contexts OWNER TO gargantua;
-- To attach contexts to a Corpus
CREATE TABLE public.nodes_contexts (
node_id INTEGER NOT NULL REFERENCES public.nodes(id) ON DELETE CASCADE,
context_id INTEGER NOT NULL REFERENCES public.contexts(id) ON DELETE CASCADE,
score REAL ,
category INTEGER ,
PRIMARY KEY (node_id, context_id)
);
ALTER TABLE public.nodes_contexts OWNER TO gargantua;
---------------------------------------------------------------
CREATE TABLE public.context_node_ngrams (
context_id INTEGER NOT NULL REFERENCES public.contexts (id) ON DELETE CASCADE,
node_id INTEGER NOT NULL REFERENCES public.nodes (id) ON DELETE CASCADE,
ngrams_id INTEGER NOT NULL REFERENCES public.ngrams (id) ON DELETE CASCADE,
ngrams_type INTEGER ,
weight double precision,
PRIMARY KEY (context_id, node_id, ngrams_id, ngrams_type)
);
ALTER TABLE public.context_node_ngrams OWNER TO gargantua;
CREATE TABLE public.context_node_ngrams2 (
context_id INTEGER NOT NULL REFERENCES public.contexts (id) ON DELETE CASCADE,
nodengrams_id INTEGER NOT NULL REFERENCES public.node_ngrams (id) ON DELETE CASCADE,
weight double precision,
PRIMARY KEY (context_id, nodengrams_id)
);
ALTER TABLE public.context_node_ngrams2 OWNER TO gargantua;
CREATE INDEX ON public.contexts USING gin (hyperdata);
CREATE INDEX ON public.contexts USING btree (user_id, typename, parent_id);
CREATE INDEX ON public.contexts USING btree (id, typename, date ASC);
CREATE INDEX ON public.contexts USING btree (id, typename, date DESC);
CREATE INDEX ON public.contexts USING btree (typename, id);
CREATE UNIQUE INDEX ON public.contexts USING btree (hash_id);
-- To make the links between Corpus Node and its contexts
CREATE UNIQUE INDEX ON public.nodes_contexts USING btree (node_id, context_id);
CREATE INDEX ON public.nodes_contexts USING btree (node_id, context_id, category);
------------------------------------------------------------------------
CREATE UNIQUE INDEX ON public.context_node_ngrams USING btree (context_id, node_id, ngrams_id, ngrams_type);
CREATE INDEX ON public.context_node_ngrams USING btree (context_id, node_id);
CREATE INDEX ON public.context_node_ngrams USING btree (ngrams_id, node_id);
CREATE INDEX ON public.context_node_ngrams USING btree (ngrams_type);
CREATE INDEX ON public.context_node_ngrams2 USING btree (context_id);
CREATE INDEX ON public.context_node_ngrams2 USING btree (nodengrams_id);
CREATE INDEX ON public.context_node_ngrams2 USING btree (context_id, nodengrams_id);
DROP TABLE if EXISTS public.node_nodengrams_nodengrams;
DROP TRIGGER if EXISTS trigger_count_delete2 ON nodes_nodes;
DROP TRIGGER if EXISTS trigger_count_update_add ON nodes_nodes;
DROP TRIGGER if EXISTS trigger_delete_count ON nodes_nodes;
DROP TRIGGER if EXISTS trigger_insert_count ON nodes_nodes;
|]
devops/postgres/empty 0 → 100755
View file @ 4a80fa7b
#!/bin/bash
# sudo su postgres
# postgresql://$USER:$PW@localhost/$DB
PW="C8kdcUrAQy66U"
DB="gargandb1"
USER="gargantua"
#psql -c "CREATE USER \"${USER}\""
#psql -c "ALTER USER \"${USER}\" with PASSWORD '${PW}'"
psql -c "DROP DATABASE IF EXISTS \"${DB}\""
createdb "${DB}"
psql -c "ALTER DATABASE \"${DB}\" OWNER to \"${USER}\""
devops/postgres/upgrade/0.0.5/nodes2contexts.sql 0 → 100644
View file @ 4a80fa7b
-- to delete
-- DELETE FROM contexts;
-- WITH docs (id,hash_id,typename,user_id,parent_id,name,date,hyperdata, search)
WITH docs AS (SELECT * from nodes WHERE nodes.typename IN (4,41)),
inserted (id, hash_id) AS (
INSERT INTO contexts (hash_id,typename,user_id,parent_id,name,date,hyperdata, search)
SELECT d.hash_id,d.typename,d.user_id,NULL,d.name,d.date,d.hyperdata,search FROM docs AS d
RETURNING contexts.id, contexts.hash_id
),
indexed (node_id, context_id) AS (
SELECT docs.id, inserted.id from inserted
JOIN docs on docs.hash_id = inserted.hash_id
),
-- nodes_nodes -> nodes_contexts
nodes_contexts_query AS (
INSERT INTO nodes_contexts (node_id, context_id,score, category)
SELECT nn.node1_id,i.context_id,nn.score,nn.category FROM nodes_nodes nn
JOIN indexed i ON i.node_id = nn.node2_id
),
-- nodes_nodes_ngrams -> contexts_nodes_ngrams
contexts_nodes_ngrams_query AS (
INSERT INTO context_node_ngrams
SELECT i.context_id, nnn.node1_id, nnn.ngrams_id, nnn.ngrams_type, nnn.weight FROM node_node_ngrams nnn
JOIN indexed i ON i.node_id = nnn.node2_id
),
---- nodes_nodes_ngrams2 -> contexts_nodes_ngrams2
context_node_ngrams2_query AS (
INSERT INTO context_node_ngrams2
SELECT i.context_id, nnn2.nodengrams_id, nnn2.weight FROM node_node_ngrams2 nnn2
JOIN indexed i ON i.node_id = nnn2.node_id
)
-- WITH CASCADE it should update others tables
DELETE FROM nodes n
USING indexed i WHERE i.node_id = n.id
;
UPDATE contexts SET parent_id = id;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment