Commit 6a0506a4 authored by sim's avatar sim

Remove old install stuff

parent 591915ef
# Install Instructions for Gargamelle
**Gargamelle** is the gargantext platform toolbox: it installs a full gargantext system with minimal modules inside a **docker** container.
First you need to get the source code to install it
The destination folder will be `/srv/gargantext`:
* docs contains all information on gargantext
(`/srv/gargantext/docs/`)
* install contains all the installation files
`/srv/gargantext/install/`
Help needed ?
See [http://gargantext.org/about](http://gargantext.org/about) and [tools](./contribution_guide.md) for the community
## Get the source code
by cloning gargantext into /srv/gargantext
``` bash
git clone ssh://git@gitlab.iscpif.fr:20022/humanities/gargantext.git /srv/gargantext \
&& cd /srv/gargantext \
&& git fetch origin install \
&& git checkout install \
```
## Install
``` bash
# go into the directory
user@computer: cd /srv/gargantext/
# get inside installation folder
user@computer: cd install
# execute the installation script
user@computer: ./install
```
During installation an admin account for gargantext will be created by asking you a username and a password
Remember it to access to the Gargantext plateform
## Run
Once you're done with the installation, **Gargantext** platform will be available at `http://localhost:8000`
simply by running the `start` executable file
``` bash
# go into the directory
user@computer: cd /srv/gargantext/
# run the start command
user@computer: ./start
# type ctrl+d or "exit" command to exit
```
Then open up a chromium browser and go to `http://localhost:8000`
Click on "Enter Gargantext"
Login in with your created username and password
Enjoy! ;)
#!/bin/bash
sudo apt-get update
sudo apt-get install \
apt-transport-https \
ca-certificates \
curl \
gnupg2 \
software-properties-common
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -
sudo apt-key fingerprint 0EBFCD88
echo "Should be: Key fingerprint = 9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88"
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/debian \
$(lsb_release -cs) \
stable"
sudo apt-get update
sudo apt-get install docker-ce
sudo docker run hello-world
#!/bin/bash
#configure the base image gargamelle
#echo ':::: BUILD ::::'
sudo docker build -t gargamelle:latest ./gargamelle
echo ':::: CONFIGURE ::::'
sudo docker run \
-v /srv/:/srv/ \
-p 8000:8000 \
-p 5432 \
-it gargamelle:latest \
/bin/bash -c "./psql_configure.sh; ./django_configure.sh ; exit"
sudo docker rm -f `docker ps -a | grep -v CONTAINER | awk '{print $1 }'`
#!/bin/bash
sudo docker run \
-v /srv/:/srv/\
-p 8000:8000 \
-p 5432 \
-it gargamelle:latest \
/bin/bash -c "service postgresql start; /bin/su gargantua -c 'source /env_3-5/bin/activate && /srv/gargantext/manage.py runserver 0.0.0.0:8000'"
sudo docker rm -f `docker ps -a | grep -v CONTAINER | awk '{print $1 }'`
#A quick way to recover space (assuming site is running)
sudo docker rm `docker ps -a | grep Exited | awk '{print $1 }'`
sudo docker rmi `docker images -aq`
# list all containers
docker ps -a
# remove a container
docker rm $container_id
#!/bin/bash
### Update and install base dependencies
echo "############ DEBIAN LIBS ###############"
apt-get update && \
apt-get install -y \
apt-utils ca-certificates locales \
sudo aptitude gcc g++ wget git vim \
build-essential make \
postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
postgresql-server-dev-9.6 libpq-dev libxml2 \
postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
nginx rabbitmq-server uwsgi uwsgi-core uwsgi-plugin-python3
### Configure timezone and locale
echo "########### LOCALES & TZ #################"
echo "Europe/Paris" > /etc/timezone
dpkg-reconfigure --frontend=noninteractive tzdata
#ENV TZ "Europe/Paris"
sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \
sed -i -e 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen && \
locale-gen && \
update-locale LANG=fr_FR.UTF-8 && \
update-locale LANGUAGE=fr_FR.UTF-8 && \
update-locale LC_ALL=fr_FR.UTF-8
# Not: using LC_ALL is discouraged
# see warning: https://wiki.debian.org/Locale
### Install main dependencies and python packages based on Debian distrib
echo "############# PYTHON DEPENDENCIES ###############"
apt-get update && apt-get install -y \
libxml2-dev xml-core libgfortran-6-dev \
libpq-dev \
python3.5 \
python3.5-dev \
python3-six python3-numpy python3-setuptools \
python3-numexpr \
python3-pip \
libxml2-dev libxslt-dev zlib1g-dev libigraph0-dev
#libxslt1-dev
# UPDATE AND CLEAN
apt-get update && apt-get autoclean
#NB: removing /var/lib will avoid to significantly fill up your /var/ folder on your native system
########################################################################
### PYTHON ENVIRONNEMENT (as ROOT)
########################################################################
#adduser --disabled-password --gecos "" gargantua
cd /srv/
pip3 install virtualenv
virtualenv /srv/env_3-5 -p /usr/bin/python3.5
echo '/srv/gargantext' > /srv/env_3-5/lib/python3.5/site-packages/gargantext.pth
echo 'alias venv="source /srv/env_3-5/bin/activate"' >> ~/.bashrc
# CONFIG FILES
# su gargantua -c 'all commands below'
# but nltk needs right access (so done in root, not really good for safety)
source /srv/env_3-5/bin/activate && pip3 install -r /srv/gargantext/install/gargamelle/requirements.txt && \
pip3 install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1 && \
python3 -m nltk.downloader averaged_perceptron_tagger -d /usr/local/share/nltk_data
chown gargantua:gargantua -R /srv/env_3-5
#######################################################################
## POSTGRESQL DATA (as ROOT)
#######################################################################
sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargandata\'%" /etc/postgresql/9.6/main/postgresql.conf
echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.6/main/pg_hba.conf
echo "listen_addresses='*'" >> /etc/postgresql/9.6/main/postgresql.conf
###########################################################
# Gargamelle WEB
###########################################################
#Build an image starting with debian:stretch image
# wich contains all the source code of the app
FROM debian:stretch
MAINTAINER ISCPIF <gargantext@iscpif.fr>
USER root
### Update and install base dependencies
RUN echo "############ DEBIAN LIBS ###############"
RUN apt-get update && \
apt-get install -y \
apt-utils ca-certificates locales \
sudo aptitude gcc g++ wget git vim \
build-essential make \
postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
postgresql-server-dev-9.6 libpq-dev libxml2 \
postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
### Configure timezone and locale
RUN echo "########### LOCALES & TZ #################"
RUN echo "Europe/Paris" > /etc/timezone
ENV TZ "Europe/Paris"
RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \
sed -i -e 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen && \
dpkg-reconfigure --frontend=noninteractive locales && \
echo 'LANG="fr_FR.UTF-8"' > /etc/default/locale
ENV LANG fr_FR.UTF-8
ENV LANGUAGE fr_FR.UTF-8
ENV LC_ALL fr_FR.UTF-8
### Install main dependencies and python packages based on Debian distrib
RUN echo "############# PYTHON DEPENDENCIES ###############"
RUN apt-get update && apt-get install -y \
libxml2-dev xml-core libgfortran-6-dev \
libpq-dev \
python3.5 \
python3-dev \
# for numpy, pandas and numpyperf
python3-six python3-numpy python3-setuptools \
python3-numexpr \
# python dependencies
python3-pip \
# for lxml
libxml2-dev libxslt-dev \
libxslt1-dev zlib1g-dev
# UPDATE AND CLEAN
RUN apt-get update && apt-get autoclean &&\
rm -rf /var/lib/apt/lists/*
#NB: removing /var/lib will avoid to significantly fill up your /var/ folder on your native system
########################################################################
### PYTHON ENVIRONNEMENT (as ROOT)
########################################################################
RUN adduser --disabled-password --gecos "" gargantua
RUN pip3 install virtualenv
RUN virtualenv /env_3-5
RUN echo 'alias venv="source /env_3-5/bin/activate"' >> ~/.bashrc
# CONFIG FILES
ADD requirements.txt /
ADD psql_configure.sh /
ADD django_configure.sh /
RUN . /env_3-5/bin/activate && pip3 install -r requirements.txt && \
pip3 install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1 && \
python3 -m nltk.downloader averaged_perceptron_tagger -d /usr/local/share/nltk_data
RUN chown gargantua:gargantua -R /env_3-5
########################################################################
### POSTGRESQL DATA (as ROOT)
########################################################################
RUN sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargandata\'%" /etc/postgresql/9.5/main/postgresql.conf
RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.5/main/pg_hba.conf
RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
EXPOSE 5432 8000
# VOLUME ["/srv/",]
#!/bin/bash
#configure the base image gargamelle
docker build -t gargamelle:latest ./gargamelle
#2 option with this image:
# configure the container
# run the image with the app in it
#!/bin/bash
PATH="/srv/gargantext/install/gargamelle"
#./folders_configure.sh;
#./psql_configure.sh;
/bin/bash "$PATH/django_configure.sh";
/bin/bash -c "Configuration Ok""
#!/bin/bash
# opens a console + virtualenv inside the already active docker container
# (to use after start)
sudo docker exec -it gargamelle_box bash --rcfile 'env_3-5/bin/activate'
#!/bin/bash
sudo docker run \
--name=gargamelle_box \
-v /srv/gargantext:/srv/gargantext \
-v /srv/gargandata:/srv/gargandata \
-v /srv/gargantext_lib:/srv/gargantext_lib \
-p 8000:8000 \
-p 5432 \
-it gargamelle:latest \
/bin/bash -c "service postgresql start; /bin/su gargantua -c 'source /env_3-5/bin/activate && /srv/gargantext/manage.py runserver 0.0.0.0:8000' && bin/bash"
sudo docker rm gargamelle_box
#!/bin/bash
##################################################
# __| |(_) __ _ _ __ __ _ ___
# / _` || |/ _` | '_ \ / _` |/ _ \
# | (_| || | (_| | | | | (_| | (_) |
# \__,_|/ |\__,_|_| |_|\__, |\___/
# |__/ |___/
##################################################
#configure django migrations
##################################################
echo "::::: DJANGO :::::"
#echo "Starting Postgres"
#/usr/sbin/service postgresql start
su gargantua -c 'source /srv/env_3-5/bin/activate &&\
echo "Activated env" &&\
/srv/gargantext/manage.py makemigrations &&\
/srv/gargantext/manage.py migrate && \
echo "migrations ok" &&\
/srv/gargantext/dbmigrate.py && \
/srv/gargantext/dbmigrate.py && \
/srv/gargantext/dbmigrate.py && \
/srv/gargantext/manage.py createsuperuser'
service postgresql stop
#!/bin/bash
chown -R gargantua:gargantua /srv/gargantext
##
# You should look at the following URL's in order to grasp a solid understanding
# of Nginx configuration files in order to fully unleash the power of Nginx.
# http://wiki.nginx.org/Pitfalls
# http://wiki.nginx.org/QuickStart
# http://wiki.nginx.org/Configuration
#
# Generally, you will want to move this file somewhere, and start with a clean
# file but keep this around for reference. Or just disable in sites-enabled.
#
# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
##
# the upstream component nginx needs to connect to
upstream gargantext {
server unix:///tmp/gargantext.sock; # for a file socket
#server 127.0.0.1:8001; # for a web port socket (we'll use this first)
}
# Default server configuration
#
server {
listen 80 default_server;
listen [::]:80 default_server;
# SSL configuration
#
# listen 443 ssl default_server;
# listen [::]:443 ssl default_server;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
client_max_body_size 800M;
client_body_timeout 12;
client_header_timeout 12;
keepalive_timeout 15;
send_timeout 10;
root /var/www/html;
# Add index.php to the list if you are using PHP
#index index.html index.htm index.nginx-debian.html;
server_name _ localhost ;
# Django media
location /media {
alias /srv/gargantext_media; # your Django project's media files - amend as required
}
location /static {
alias /srv/gargantext_static; # your Django project's static files - amend as required
}
# Finally, send all non-media requests to the Django server.
location / {
uwsgi_pass gargantext;
include uwsgi_params;
}
#access_log off;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
}
-- CNRS Copyrights 2017
-- See Gargantext Licence for details
-- Maintainers: team@gargantext.org
-- USAGE
-- psql gargandb < occ_growth.sql
-- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
-- EXEMPLE USAGE
-- SELECT * FROM OCC_HIST(182856, 183859, 183866, '1800-03-15 17:00:00+01', '2000-03-15 17:00:00+01', '2017-03-15 17:00:00+01')
-- OCC_HIST_PART :: Corpus.id -> GroupList.id -> Start -> End
DROP FUNCTION OCC_HIST_PART(integer, integer, timestamp without time zone, timestamp without time zone);
-- DROP for tests
CREATE OR REPLACE FUNCTION OCC_HIST_PART(int, int, timestamp, timestamp) RETURNS TABLE (ng_id int, score float8)
AS $$
-- EXPLAIN ANALYZE
SELECT
COALESCE(gr.ngram1_id, ng1.ngram_id) as ng_id,
SUM(ng1.weight) as score
from nodes n
-- BEFORE
INNER JOIN nodes as n1 ON n1.id = n.id
INNER JOIN nodes_ngrams ng1 ON ng1.node_id = n1.id
-- Limit with timestamps: ]start, end]
INNER JOIN nodes_hyperdata nh1 ON nh1.node_id = n1.id
AND nh1.value_utc > $3
AND nh1.value_utc <= $4
-- Group List
LEFT JOIN nodes_ngrams_ngrams gr ON ng1.ngram_id = gr.ngram2_id
AND gr.node_id = $2
WHERE
n.typename = 4
AND n.parent_id = $1
GROUP BY 1
$$
LANGUAGE SQL;
DROP FUNCTION OCC_HIST(integer, integer, integer, timestamp without time zone, timestamp without time zone, timestamp without time zone);
-- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
CREATE OR REPLACE FUNCTION OCC_HIST(int, int, int, timestamp, timestamp, timestamp) RETURNS TABLE (ng_id int, score numeric)
AS $$
WITH OCC1 as (SELECT * from OCC_HIST_PART($1, $2, $4, $5))
, OCC2 as (SELECT * from OCC_HIST_PART($1, $2, $5, $6))
, GROWTH as (SELECT ml.ngram_id as ngram_id
, COALESCE(OCC1.score, null) as score1
, COALESCE(OCC2.score, null) as score2
FROM nodes_ngrams ml
LEFT JOIN OCC1 ON OCC1.ng_id = ml.ngram_id
LEFT JOIN OCC2 ON OCC2.ng_id = ml.ngram_id
WHERE ml.node_id = $3
ORDER by score2 DESC)
SELECT ngram_id, COALESCE(ROUND(CAST((100 * (score2 - score1) / COALESCE((score2 + score1), 1)) as numeric), 2), 0) from GROWTH
$$
LANGUAGE SQL;
-- BEHAVIORAL TEST (should be equal to occ in terms table)
-- WITH OCC as (SELECT * from OCC_HIST(182856, 183859, '1800-03-15 17:00:00+01', '2300-03-15 17:00:00+01'))
-- SELECT ng_id, score from OCC
-- INNER JOIN nodes_ngrams ml on ml.ngram_id = ng_id
-- AND ml.node_id = 183866
-- ORDER BY score DESC;
#!/bin/bash
#######################################################################
## ____ _
## | _ \ ___ ___| |_ __ _ _ __ ___ ___
## | |_) / _ \/ __| __/ _` | '__/ _ \/ __|
## | __/ (_) \__ \ || (_| | | | __/\__ \
## |_| \___/|___/\__\__, |_| \___||___/
## |___/
#######################################################################
echo "::::: POSTGRESQL :::::"
su postgres -c 'pg_dropcluster 9.4 main --stop'
#done in docker but redoing it
rm -rf /srv/gargandata && mkdir /srv/gargandata && chown postgres:postgres /srv/gargandata
su postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /srv/gargandata/'
su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /srv/gargandata/ -l /srv/gargandata/journal_applicatif start'
su postgres -c 'pg_createcluster -D /srv/gargandata 9.6 main '
su postgres -c 'pg_ctlcluster -D /srv/gargandata 9.6 main start '
su postgres -c 'pg_ctlcluster 9.6 main start'
service postgresql start
su postgres -c "psql -c \"CREATE user gargantua WITH PASSWORD 'C8kdcUrAQy66U'\""
su postgres -c "createdb -O gargantua gargandb"
echo "Postgres configured"
#service postgresql stop
# try bottleneck
eventlet==0.20.1
amqp==1.4.9
anyjson==0.3.3
billiard==3.3.0.23
celery==3.1.25
chardet==2.3.0
dateparser==0.3.5
Django==1.10.5
django-celery==3.2.1
django-pgfields==1.4.4
django-pgjsonb==0.0.23
djangorestframework==3.5.3
djangorestframework-jwt==1.9.0
html5lib==0.9999999
python-igraph>=0.7.1
jdatetime==1.7.2
kombu==3.0.37 # messaging
langdetect==1.0.6 #detectinglanguage
nltk==3.1
numpy==1.13.1
psycopg2==2.6.2
pycountry==1.20
python-dateutil==2.4.2
pytz==2016.10 # timezones
PyYAML==3.11
RandomWords==0.1.12
ujson==1.35
umalqurra==0.2 # arabic calendars (?? why use ??)
networkx==1.11
pandas==0.18.0
six==1.10.0
lxml==3.5.0
requests-futures==0.9.7
bs4==0.0.1
requests==2.10.0
alembic>=0.9.2
SQLAlchemy==1.1.14
SQLAlchemy-Searchable==0.10.4
SQLAlchemy-Utils==0.32.16
#!/bin/bash
#name:01-setup
#TODO clone the repo into /srv/gargantext/ and reduce the different steps
#git clone ssh://git@gitlab.iscpif.fr:20022/humanities/gargantext.git /srv/gargantext \
# && cd /srv/gargantext \
# && git fetch origin stable \
# && git checkout stable
function clone_repo {
echo "* CLONE"
#echo "/home/"$USER"/ssh_keys/"$USER;
read -e -r -p "Please provide you ssh keys full directory:" response;
ssh-agent bash -c 'ssh-add ' $response '';
git clone ssh://git@gitlab.iscpif.fr:20022/humanities/gargantext.git -o /srv/gargantext;
cd /srv/gargantext
git fetch origin stable
git pull
git checkout -b $USER'-stable';
echo "Switching to your own branch";
}
function download_lib {
echo "* DOWNLOAD additionnal lib"
echo "(May take a while, be patient or come back later when finished;)"
sudo wget -P install/ http://dl.gargantext.org/gargantext_lib.tar.bz2
#here gargantext_lib.tar.bz2 inside this directory
}
function uncompress_lib {
cd /srv/gargantext/install/
echo "* UNCOMPRESSING additionnal lib"
sudo tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib
sudo chown -R gargantua:gargantua /srv/gargantext_lib
}
#~ read -r -p "Do you want to clone the project? [y/N] " response
#~ case $response in
#~ [yY][eE][sS]|[yY])
#~ clone_repo
#~ ;;
#~ *)
#~ cd /srv/gargantext/
#~ ;;
#~ esac
echo "::: CREATE GROUP :::";
if grep -q 'gargantua' /etc/groups
then
echo "Using existing group 'gargantua'"
else
sudo groupadd gargantua
fi
# adding the users to the group
current_user=$(who -m | cut -d' ' -f1)
sudo usermod -G gargantua $current_user
sudo usermod -G gargantua gargantua
# changing the group of the sourcedir
sudo chown -R :gargantua /srv/gargantext
echo "::: SETUP ENV :::";
for dir in "/srv/gargantext_lib" "/srv/gargantext_static" "/srv/gargantext_media"; do
sudo mkdir -p $dir ;
sudo chown gargantua:gargantua $dir ;
done;
#here gargantext_lib.tar.bz2 inside this directory
#sudo wget http://dl.gargantext.org/gargantext_lib.tar.bz2 && \
#sudo tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
#&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
#&& echo ":::::::::::::::::Done::::::::::::::::::::::::::";
echo ':::: BUILD ::::'
sudo docker build -t gargamelle:latest ./gargamelle
echo ':::: CONFIGURE ::::'
sudo docker run \
--name=gargamelle_box \
-v /srv/gargantext:/srv/gargantext \
-v /srv/gargandata:/srv/gargandata \
-v /srv/gargantext_lib:/srv/gargantext_lib \
-p 8000:8000 \
-p 5432 \
-it gargamelle:latest \
/bin/bash -c "./psql_configure.sh; ./django_configure.sh ; exit"
sudo docker rm gargamelle_box
# creating the "start" copy + giving it normal ownership (because we're probably sudo)
cp ./run /srv/gargantext/start
chown $current_user:gargantua /srv/gargantext/start
#!/bin/bash
sudo docker run \
--name=gargamelle_box \
-v /srv/gargantext:/srv/gargantext \
-v /srv/gargandata:/srv/gargandata \
-v /srv/gargantext_lib:/srv/gargantext_lib \
-p 8000:8000 \
-p 5432 \
-it gargamelle:latest \
/bin/bash -c "service postgresql start; /bin/su gargantua -c 'source /env_3-5/bin/activate && /srv/gargantext/manage.py runserver 0.0.0.0:8000'"
sudo docker rm gargamelle_box
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment