Automatically handle scrapyd configuration file

.gitignore

@@ -6,3 +6,4 @@ gargantext.ini
 postgrest.conf
 *.log
 scrapyd
+scrapy.cfg

scrapy.cfg

-gargantext.ini
\ No newline at end of file

tools/mkconf.sh

@@ -54,10 +54,23 @@ VENV=$(pipenv --venv)
 # PostgREST configuration file path
 [ -z "$POSTGREST_CONF" ] && POSTGREST_CONF=postgrest.conf
 
+# Scrapy & scrapyd configuration file path
+SCRAPY_CONF=scrapy.cfg
+
 # Configuration template paths
 GARGANTEXT_TEMPLATE=tools/conf/gargantext.template.ini
 POSTGREST_TEMPLATE=tools/conf/postgrest.template.conf
 
+echo "▸ Make symbolic link $SCRAPY_CONF to $GARGANTEXT_CONF for scrapyd..."
+if [ -e "$SCRAPY_CONF" ]; then
+    if [ -L "$SCRAPY_CONF" ]; then
+        rm -f $SCRAPY_CONF
+    else
+        echo "WARNING: $SCRAPY_CONF exists and is not a symlink, won't touch it."
+    fi
+fi
+[ -e "$SCRAPY_CONF" ] || ln -s $GARGANTEXT_CONF $SCRAPY_CONF
+
 # Check for configuration file existence
 if [ -f "$GARGANTEXT_CONF" -a -z "$FORCE" ]; then
     echo -e "Configuration file $GARGANTEXT_CONF already exists, you may" \

tools/mkenvs.sh

@@ -26,6 +26,8 @@ $DJANGO_VAR
 $SCRAPY_VAR
 # Paths of configuration files, you're welcome to change that; when a simple
 # filename is given, it'll be searched in current directory.
+# IMPORTANT: if you change this don't forget to update scrapy.cfg symbolic link
+#            by running ./tools/mkconf.sh
 GARGANTEXT_CONF=$GARGANTEXT_CONF
 POSTGREST_CONF=$POSTGREST_CONF
 EOF