merged the two python services to run them together via run.sh and adapted the...

merged the two python services to run them together via run.sh and adapted the docs... this new version takes almost all functionalities from david chavalarias's and samuel castillo's work, cleaned up and integrated with the new registration service and a standard docker+nginx packaging for deployment

parametres_comex.ini → config/parametres_comex.ini

@@ -8,11 +8,11 @@ COMEX_PORT  = 9090
 
 ################################################################################
 [routes]
-USR_PREFIX = /regcomex
-# API_PREFIX = /comexAPI
+PREFIX = /services
+USR_ROUTE = /user
+API_ROUTE = /api
 
-# NB: prefixes should match your nginx/apache reverse proxy conf
-# API_PREFIX still unused but will be after merge with legacy comex
+# NB: prefix should match your nginx/apache reverse proxy conf
 
 ################################################################################
 [services]

doc/nginx_conf.md

@@ -28,20 +28,25 @@ sudo nano comex.conf
 
 This below is a full config exemple you can paste in nano:
   - it serves the comex app (legacy php), in `/`
-  - it also serves registration app, in `/regcomex`  
+  - it also serves registration app, in `/services/user/register`  
 
 
 ```ini
-# Full server config: php comex as root and regcomex as subpath
-# =============================================================
+# Full server config: php comex as root and api + reg as services subpath
+# ========================================================================
 server {
     listen 80 ;
     listen [::]:80 ;
 
-    server_name _;
+    server_name communityexplorer.org;
 
     # adapt path to your php docroot
-    root /home/romain/comex/www ;
+    root /home/me/comex2 ;
+
+    # get the logs in a custom place
+    # (adapt paths)
+    access_log /home/me/somewhere/access.log ;
+    error_log /home/me/somewhere/error.log ;
 
     location / {
         index     index.html index.php ;
@@ -52,13 +57,13 @@ server {
         fastcgi_pass unix:/run/php/php7.0-fpm.sock;
 
         # here we adapted $documentroot to our real php docroot
-        fastcgi_param SCRIPT_FILENAME /home/romain/comex/www/$fastcgi_script_name;
-        #                             -----------------------
+        fastcgi_param SCRIPT_FILENAME /home/me/comex2/$fastcgi_script_name;
+        #                             ----------------
     }
 
-    # no root here => independant app
+    # no root here => independant app serving both services/user (formerly know as regcomex) and services/api (ex formerly known as comex_install)
     # (but /locationpath must match this app's default route)
-    location /regcomex {
+    location /services {
         # point to gunicorn server
         proxy_pass http://0.0.0.0:9090;
         proxy_redirect     off;
@@ -68,16 +73,11 @@ server {
         proxy_set_header   X-Real-IP $remote_addr;
         proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header   X-Forwarded-Host $server_name;
-
-        # get the logs in a custom place
-        # (adapt paths)
-        access_log /home/romain/comex/regcomex/logs/nginx/access.log ;
-        error_log /home/romain/comex/regcomex/logs/nginx/error.log debug;
     }
 
     # faster static serving
     location /static {
-        alias  /home/romain/comex/regcomex/static/;
+        alias  /home/me/comex2/static/;
         autoindex on;
     }
 

doc/permissions_setup.md

@@ -2,13 +2,10 @@ To remember the setup
 
 #### Dir permissions
 ```
-# in regcomex dir
+# in comex2 dir
 chown -R rloth:www-data .
 
-# cgi executables
-chmod 754 cgi-bin/*               # <=> u+rwx, g+rx, o+r
-
 # writeable data
 chmod 774 data
-chmod 774 data/registered.db      # <=> u+rwx, g+rwx, o+r
+chmod 774 data/shared_mysql_data/  # <=> u+rwx, g+rwx, o+r
 ```

index.html

@@ -200,7 +200,7 @@
                         <!--<a id="printname" href="#"> <strong>PRINT</strong></a>-->
                     </p>
                     <br/><br/>
-<i class="icon-user"></i>  <strong>You cannot find yourself</strong> in the database, want update your data or keywords ? <a href="/regcomex">Edit your profile</a>, the database is updated instantly.
+<i class="icon-user"></i>  <strong>You cannot find yourself</strong> in the database, want update your data or keywords ? <a href="/services/user/register">Edit your profile</a>, the database is updated instantly.
 
                                                  <button id="register" class="btn btn-mini btn-info">
                             Register

php_library/parametres.php

@@ -6,7 +6,7 @@ $min_num_friends=0;// nombre minimal de voisin que doit avoir un scholar pour ê
 // $compress='No';
 
 /* parametres externes: nom de l'hôte SQL*/
-$params = parse_ini_file("parametres_comex.ini");
+$params = parse_ini_file("config/parametres_comex.ini");
 
 /* tout pour le MySQL */
 $host =  $params['SQL_HOST'];

services/comex_main_backend.py

@@ -29,17 +29,20 @@ from MySQLdb     import connect, ProgrammingError
 from re          import sub
 from os          import path
 from traceback   import format_tb
+from json        import dumps
 
 if __package__ == 'services':
     # when we're run via import
     from services.user import comex_user
     from services.text import keywords
     from services.tools import read_config
+    from services.db_to_tina_api.extractDataCustom import MyExtractor as MySQL
 else:
     # when this script is run directly
     from user          import comex_user
     from text          import keywords
     from tools         import read_config
+    from db_to_tina_api.extractDataCustom import MyExtractor as MySQL
 
 # ============= read config ============
 
@@ -136,9 +139,9 @@ MIN_KW = 5
 # /!\ Routes are not prefixed by nginx in prod so we do it ourselves /!\
 # -----------------------------------------------------------------------
 
-
-@app.route(config['USR_PREFIX'] + "/", methods=['GET','POST'])
-def one_big_form():
+# /services/user/register
+@app.route(config['PREFIX'] + config['USR_ROUTE'] + '/register', methods=['GET','POST'])
+def register():
     if request.method == 'GET':
         return render_template(
             "base_form.html",
@@ -221,6 +224,33 @@ def one_big_form():
                                 message = "")
 
 
+# API to provide json extracts of the DB to tinaweb
+# (originally @ moma/legacy_php_comex/tree/master/comex_install)
+# (original author S. Castillo)
+# /services/api/
+@app.route(config['PREFIX'] + config['API_ROUTE'])
+def api_main():
+
+    db=MySQL(config['SQL_HOST'])
+
+    if 'qtype' in request.args:
+        if request.args['qtype'] == "filters":
+            # all the other args are a series of constraints for filtering
+            # ex: qtype=filters&keywords[]=complex%20networks&keywords[]=complexity%20theory&countries[]=France&countries[]=USA
+            scholars = db.getScholarsList("filter",request.query_string.decode())
+        else:
+            unique_id = request.args['unique_id']
+            scholars = db.getScholarsList("unique_id",unique_id)
+    else:
+        raise TypeError("API query is missing qtype (should be 'filters' or 'uid')")
+
+    if scholars and len(scholars):
+        db.extract(scholars)
+    # < / Data Extraction > #
+
+    graphArray = db.buildJSON_sansfa2(db.Graph)
+    return dumps(graphArray)
+
 
 ########### SUBS ###########
 def re_hash(userinput, salt="verylonverylongverylonverylongverylonverylong"):

services/db_to_tina_api/README.md

 #### Structure
-  - `main.py`: exposes the web app and routes to the correct functions according to request's GET parameters
   - `extractDataCustom.py`: starting from a query or a scholar's id, retrieves all related kw+neighbors data from DB, and builds a graph's json with the metadata and custom html in node's `content` property
   - `converter.py`: normalizes country names and codes
 
 #### Requirements
-  - flask
-  - networkx
+  see global documentation
+  (this module no longer need to be ran independantly, now part of main run.sh)
 
 #### History
-
 Several modules related to graph community region extraction and graph force atlas layout have been moved to `graph_manipulation_unused` dir, because they are replaced by a client-side FA2 in tinawebJS.
 
 Extraction logic originally developed by [S. Castillo](https://github.com/PkSM3/)  
-python3 port by [R. Loth](https://github.com/rloth/)
+python3 port and merge with regcomex into ../comex_main_backend by [R. Loth](https://github.com/rloth/)
 
 Copyright 2014-2016 ISCPIF/CNRS - UPS 3611

services/db_to_tina_api/extractDataCustom.py

@@ -3,12 +3,15 @@ from networkx  import Graph, DiGraph
 from random    import randint
 from math      import floor
 from cgi       import escape
-from converter import CountryConverter
 from pprint    import pprint
 from re        import sub
 from traceback import format_tb
 from urllib.parse import unquote
 
+
+from .converter import CountryConverter
+
+
 whoswhofilters_to_sqlnames = {
     "keywords": "keywords.kwstr",
     "countries": "scholars.country",
@@ -124,7 +127,6 @@ class MyExtractor:
                 # print("filter: REST query is", filter_dict)
 
                 if "query" in filter_dict and filter_dict["query"] == "*":
-                    print("bombaclass")
                     # query is "*" <=> all scholars
                     sql_query = """
                         SELECT doors_uid
@@ -450,8 +452,8 @@ class MyExtractor:
     def buildJSON_sansfa2(self,graph,coordsRAW=None):
 
         inst = CountryConverter("","","","")
-        ISO=inst.getCountries("countries_ISO3166.txt")
-        Alternatives=inst.getCountries("countries_alternatives.txt")
+        ISO=inst.getCountries("services/db_to_tina_api/countries_ISO3166.txt")
+        Alternatives=inst.getCountries("services/db_to_tina_api/countries_alternatives.txt")
         inst.createInvertedDicts(ISO,Alternatives)
 
         nodesA=0

services/db_to_tina_api/main.py

-"""
-comex helper backend to create json graphs from sqlite3 db
-
-TODO integrate with new regcomex server
-"""
-from extractDataCustom import MyExtractor as MySQL
-
-from flask        import Flask, request
-from json         import dumps
-from os           import environ
-from configparser import ConfigParser
-# ============= app creation ==============
-app = Flask(__name__)
-
-# ============= read environ ==============
-MY_SQL_HOST = environ.get('SQL_HOST', None)
-
-if not MY_SQL_HOST:
-    ini = ConfigParser()
-    ini.read("../../parametres_comex.ini")
-    MY_SQL_HOST = ini['services']['SQL_HOST']
-
-# ================= views =================
-# @app.route("/getJSON")        # route renamed
-@app.route("/comexAPI")
-def main():
-
-    # db=SQLite('../community.db')
-    db=MySQL(MY_SQL_HOST)
-
-    if 'qtype' in request.args:
-        if request.args['qtype'] == "filters":
-            # all the other args are a series of constraints for filtering
-            # ex: qtype=filters&keywords[]=complex%20networks&keywords[]=complexity%20theory&countries[]=France&countries[]=USA
-            scholars = db.getScholarsList("filter",request.query_string.decode())
-        else:
-            unique_id = request.args['unique_id']
-            scholars = db.getScholarsList("unique_id",unique_id)
-    else:
-        raise TypeError("API query is missing qtype (should be 'filters' or 'uid')")
-
-    if scholars and len(scholars):
-        db.extract(scholars)
-    # < / Data Extraction > #
-
-    graphArray = db.buildJSON_sansfa2(db.Graph)
-    return dumps(graphArray)
-
-
-if __name__ == "__main__":
-    print(" * Using %s:3306 as SQL_HOST" % MY_SQL_HOST)
-    app.run(host="0.0.0.0", port=8484)

services/db_to_tina_api/nginx_conf_suggestion.md

-
-### A possible connection of comex helper app (comex_install)
-
-Here is the simplest possible nginx configuration to make comex_install answer http requests for /comexAPI (previously known as "/getJSON")
-
-```
-
-  # for comex_install helper app
-  location /comexAPI {
-      proxy_pass http://0.0.0.0:8484;
-
-      proxy_redirect     off;
-      proxy_set_header   Host $host;
-      proxy_set_header   X-Real-IP $remote_addr;
-      proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header   X-Forwarded-Host $server_name;
-  }
-
-```

services/tools.py

@@ -12,7 +12,9 @@ CONFIGMENU = [
             {"sec": 'main',       "var":'DEBUG_FLAG',   "def": True         },
             {"sec": 'main',       "var":'COMEX_HOST',   "def": '0.0.0.0'    },
             {"sec": 'main',       "var":'COMEX_PORT',   "def": '9090'       },
-            {"sec": 'routes',     "var":'USR_PREFIX',   "def": '/regcomex'  },
+            {"sec": 'routes',     "var":'PREFIX',       "def": '/services'  },
+            {"sec": 'routes',     "var":'USR_ROUTE',    "def": '/user/'     },
+            {"sec": 'routes',     "var":'API_ROUTE',    "def": '/api/'      },
             {"sec": 'services',   "var":'SQL_HOST',     "def": '172.17.0.2' },
             {"sec": 'services',   "var":'SQL_PORT',     "def": '3306'       },
             {"sec": 'services',   "var":'DOORS_HOST',   "def": '0.0.0.0'    },

setup/dockers/flask_iscpif_regcomex/Dockerfile → setup/dockers/comex2_services/Dockerfile

-# Regcomex minimal dockerfile
-# ----------------------------
+# Comex2 minimal dockerfile
+# --------------------------
 # based on https://github.com/egeland/docker-gunicorn-python3-flask by F Egeland
 
 FROM ubuntu:xenial
@@ -15,18 +15,19 @@ RUN dpkg-reconfigure locales
 
 # Install global dependencies and utilities
 RUN apt update && apt autoclean
+# if without nginx add php7.0-fpm php7.0-mysql
 RUN apt install -y python3-pip libmysqlclient-dev git nano tree iputils-ping
 RUN pip3 install --upgrade pip
 
-# for regcomex itself ----------------------------------------------------------
+# for comex2 itself ----------------------------------------------------------
 # pull the comex server from repository
-RUN git clone https://github.com/moma/regcomex.git
+RUN git clone https://github.com/moma/comex2.git
 
 # Install project dependencies
-RUN pip3 install -r regcomex/setup/requirements.txt
+RUN pip3 install -r comex2/setup/requirements.txt
 
 # main CMD to start the app
-WORKDIR /regcomex
+WORKDIR /comex2
 
 # flask dev server
 # CMD python3 server_comex_registration.py

setup/dockers/docker-compose.yml

-# NB we must have already built 'flask_iscpif_regcomex' and 'minidoors' images
-#                                =====================       =========
     version: '2'
     services:
 
@@ -22,20 +20,24 @@
       comex_test:
         ports:
          - "32790:9090"
-        build: ./flask_iscpif_regcomex
-        image: flask_iscpif_regcomex:latest
+        build: ./comex2_services
+        image: comex2_services:latest
         depends_on:
          - comex_db_test
          - doors_test
         links:
          - comex_db_test:comex_db_backend
          - doors_test:doors_backend
-        environment:
-         - HOST=0.0.0.0                # script will bind server to this address
-         - DEBUG_FLAG=false
-         - SQL_HOST=comex_db_backend   # script will read/write sql to this host
-         - DOORS_HOST=134.158.75.71
-         - DOORS_PORT=80               # forwarded port we're accessing via ajax in tests
 
-      # POSSIBLE add php legacy communityexplorer site
-      # (currently served in parallel: see doc/nginx_conf.md)
+        # links the outside config parametres_comex.ini to the inside
+        # /!\ this erases the contained config with the outside one /!\
+        volumes:
+         - ../../config:/comex2/config
+
+        # # moved from env vars to new config vars via volume config
+        # environment:
+        #  - HOST=0.0.0.0                # script will bind server to this address
+        #  - DEBUG_FLAG=false
+        #  - SQL_HOST=comex_db_backend   # script will read/write sql to this host
+        #  - DOORS_HOST=134.158.75.71
+        #  - DOORS_PORT=80               # forwarded port we're accessing via ajax in tests

setup/dockers/mysql_prerequisites.md

 ## Sql config tips
-If you have an sql running, just fill your sql hostname in `$SQL_HOST` and skip to [Set up the tables](https://github.com/moma/regcomex/blob/master/doc/table_specifications.md).
+If you have an sql running, just fill your sql hostname in `$SQL_HOST` and skip to [Set up the tables](https://github.com/moma/comex2/blob/master/doc/table_specifications.md).
 
 #### Prerequisites
 We'll use docker and jq here

setup/regcomex_config.ini

-# REGCOMEX SH MINIMAL CONFIG
-# ==========================
-
-export HOST=localhost
-export SQL_HOST=172.18.0.2
-export DOORS_HOST=localhost
-export DOORS_PORT=8989
-export DEBUG_FLAG=false

setup/requirements.txt

 gunicorn
 flask
 mysqlclient  # provides MySQLdb
+networkx     # POSS could be replaced by faster igraph

static/js/whoswho.js

@@ -159,7 +159,7 @@ $(document).ready(function() {
     return popfilter("tagged", "tags", []);
   });
   $("#register").click(function() {
-    return window.open("/regcomex");
+    return window.open("/services/user/register");
   });
   $("#searchname").scholarcomplete({
     minLength: 2,

static/tinawebJS/settings_explorerjs.js

@@ -19,8 +19,8 @@ var iwantograph = "";
 var bridge={};
 external="";
 //external="http://tina.iscpif.fr/explorerjs/";//Just if you want to use the server-apps from tina.server
-bridge["forFilteredQuery"] = "comexAPI" //external+"php/bridgeClientServer_filter.php";
-bridge["forNormalQuery"] = "comexAPI" //external+"php/bridgeClientServer.php";
+bridge["forFilteredQuery"] = "services/api" //external+"php/bridgeClientServer_filter.php";
+bridge["forNormalQuery"] = "services/api" //external+"php/bridgeClientServer.php";