finish up service modules refacto

README.md

@@ -164,7 +164,7 @@ More info in `doc/` directory
 This project was developed over several years with 3 main development phases.
 
 Scholars statistics and directory structure originally developed by D. Chavalarias (scholars data exploration in the `php_library` folder).
-Graph extraction logic originally developed by [S. Castillo](https://github.com/PkSM3/) (SubsetExtractor in the `dbdatapi` module)
+Graph extraction logic originally developed by [S. Castillo](https://github.com/PkSM3/) (BipartiteExtractor in the `dbdatapi` module)
 Ports to python3+MySQL, user registration and merge of the various layers into one deployable docker by [R. Loth](https://github.com/rloth/)
 
 ##### Copyright

services/db_to_tina_api/README.md

-#### Structure
-  - `extractDataCustom.py`: starting from a query or a scholar's id, retrieves all related kw+neighbors data from DB, and builds a graph's json with the metadata and custom html in node's `content` property
-  - `converter.py`: normalizes country names and codes
-
-#### Requirements
-  see global documentation
-  (this module no longer need to be ran independantly, now part of main run.sh)
-
-#### History
-Several modules related to graph community region extraction and graph force atlas layout have been moved to `graph_manipulation_unused` dir, because they are replaced by a client-side FA2 in tinawebJS.
-
-Extraction logic originally developed by [S. Castillo](https://github.com/PkSM3/)  
-python3 port and merge with regcomex into ../comex_main_backend by [R. Loth](https://github.com/rloth/)
-
-Copyright 2014-2016 ISCPIF/CNRS - UPS 3611

services/dbdatapi.py

@@ -18,11 +18,11 @@ from traceback import format_tb
 if __package__ == 'services':
     from services.tools import mlog, REALCONFIG
     from services.dbcrud  import connect_db
-    from services.text.converter import CountryConverter
+    from services.text.utils import CountryConverter
 else:
     from tools          import mlog, REALCONFIG
     from dbcrud         import connect_db
-    from text.converter import CountryConverter
+    from text.utils     import CountryConverter
 
 
 FIELDS_FRONTEND_TO_SQL = {
@@ -219,7 +219,16 @@ def find_scholar(some_key, some_str_value, cmx_db = None):
 
 
 
-class SubsetExtractor:
+class BipartiteExtractor:
+    """
+    JSON FILTERS => SQL SELECT => scholars subset
+                                       ||
+                                       VV
+                                     keywords
+                                       ||
+                                       VV
+                                     neighboors
+    """
 
     def __init__(self,dbhost):
         self.connection=connect(

services/main.py

@@ -25,11 +25,11 @@ __status__    = "Dev"
 
 
 # ============== imports ==============
-from re           import sub, match
+from re           import sub
 from os           import path
 from json         import dumps
 from datetime     import timedelta
-from urllib.parse import urlparse, urljoin, unquote
+from urllib.parse import unquote
 from flask        import Flask, render_template, request, \
                          redirect, url_for, session
 from flask_login  import fresh_login_required, login_required, \
@@ -42,9 +42,8 @@ if __package__ == 'services':
     from services          import tools, dbcrud, dbdatapi
     from services.user     import User, login_manager, \
                                   doors_login, doors_register
-    from services.dbdatapi import SubsetExtractor
-    # TODO move sanitize there
-    # from services.text  import keywords, sanitize
+    from services.dbdatapi import BipartiteExtractor
+    from services.text.utils import sanitize
 else:
     # when this script is run directly
     print("*** comex services (dev server mode) ***")
@@ -52,8 +51,8 @@ else:
     import tools, dbcrud, dbdatapi
     from user           import User, login_manager, \
                                doors_login, doors_register
-    from db_to_tina_api.extractDataCustom import MyExtractor
-    # from text           import keywords, sanitize
+    from dbdatapi       import BipartiteExtractor
+    from text.utils      import sanitize
 
 # ============= app creation ============
 config = tools.REALCONFIG
@@ -162,6 +161,9 @@ def unauthorized():
     )
 
 
+def reroute(function_name_str):
+    return redirect(url_for(function_name_str, _external=True))
+
 
 # ============= views =============
 
@@ -180,18 +182,10 @@ def rootindex():
                             "rootindex.html"
                           )
 
-# # /test_base
-# @app.route('/test_base')
-# def test_base():
-#     return render_template(
-#         "base_layout.html"
-#     )
-
-
 # /services/
 @app.route(config['PREFIX']+'/')
 def services():
-    return redirect(url_for('login', _external=True))
+    return reroute('login')
 
 # /services/api/aggs
 @app.route(config['PREFIX'] + config['API_ROUTE'] + '/aggs')
@@ -226,7 +220,7 @@ def graph_api():
     (original author S. Castillo)
     """
     if 'qtype' in request.args:
-        graphdb = SubsetExtractor(config['SQL_HOST'])
+        graphdb = BipartiteExtractor(config['SQL_HOST'])
         scholars = graphdb.getScholarsList(
                     request.args['qtype'],
                     tools.restparse(
@@ -269,7 +263,7 @@ def user_api():
 # /services/user/
 @app.route(config['PREFIX'] + config['USR_ROUTE']+'/', methods=['GET'])
 def user():
-    return redirect(url_for('login', _external=True))
+    return reroute('login')
 
 
 # /services/user/login/
@@ -380,7 +374,7 @@ def login():
             elif user.empty:
                 mlog('DEBUG',"empty user redirected to profile")
                 # we go straight to empty profile for the person to create infos
-                return(redirect(url_for('profile', _external=True)))
+                return reroute('profile')
 
             # normal call, normal user
             else:
@@ -388,7 +382,7 @@ def login():
                 next_url = request.args.get('next', None)
 
                 if not next_url:
-                    return(redirect(url_for('profile', _external=True)))
+                    return reroute('profile')
                 else:
                     next_url = unquote(next_url)
                     mlog("DEBUG", "login with next_url:", next_url)
@@ -404,7 +398,7 @@ def login():
                     else:
                         # server name is different than ours
                         # in next_url so we won't go there
-                        return(redirect(url_for('rootindex', _external=True)))
+                        return reroute('rootindex')
 
 
 # /services/user/logout/
@@ -412,7 +406,7 @@ def login():
 def logout():
     logout_user()
     mlog('INFO', 'logged out previous user')
-    return redirect(url_for('rootindex', _external=True))
+    return reroute('rootindex')
 
 # /services/user/profile/
 @app.route(config['PREFIX'] + config['USR_ROUTE'] + '/profile/', methods=['GET', 'POST'])
@@ -458,7 +452,8 @@ def profile():
                  "executing DELETE scholar's data at the request of user %s" % str(the_id_to_delete))
             logout_user()
             dbcrud.rm_scholar(the_id_to_delete)
-            return(redirect(url_for('rootindex', _external=True)))
+
+            return reroute('rootindex')
 
 
         else:
@@ -863,53 +858,11 @@ def read_record_from_request(request):
     if hasattr(request, "files") and 'pic_file' in request.files and request.files['pic_file']:
         new_fname = tools.pic_blob_to_filename(request.files['pic_file'])
         clean_records['pic_fname'] = new_fname
-        mlog("DEBUG", "new_fname", new_fname)
+        mlog("INFO", "new_fname", new_fname)
 
     return clean_records
 
 
-# TODO move to text submodules
-def sanitize(value, specific_type=None):
-    """
-    simple and radical: leaves only alphanum and '@' '.' '-' ':' ',' '(', ')', '#', ' '
-
-    One of the main goals is to remove ';'
-    POSS better
-
-
-    args:
-        @value: any string to santize
-
-        @specific_type: None or 'url' or 'date'
-    """
-    vtype = type(value)
-    str_val = str(value)
-    clean_val = sub(r'^\s+', '', str_val)
-    clean_val = sub(r'\s+$', '', clean_val)
-
-    if not specific_type:
-        san_val = sub(r'[^\w@\.:,()# -]', '_', clean_val)
-    elif specific_type == "sbool":
-        # DB uses int(0) or int(1)
-        if match('^[01]$',clean_val):
-            san_val = int(clean_val)
-        else:
-            san_val = 0
-        # NB san_val_bool = bool(san_val)
-
-    elif specific_type == "surl":
-        san_val = sub(r'[^\w@\.: -/]', '_', clean_val)
-    elif specific_type == "sdate":
-        san_val = sub(r'[^0-9/-:]', '_', clean_val)
-
-    if vtype not in [int, str]:
-        raise ValueError("Value has an incorrect type %s" % str(vtype))
-    else:
-        # cast back to orginal type
-        san_typed_val = vtype(san_val)
-        return san_typed_val
-
-
 ########### MAIN ###########
 # this can only be used for debug
 # (in general use comex-run.sh to run the app)

services/text/keywords.py

+# TODO a keywords class gathering to factorize all cleanup and split operations

services/text/converter.py → services/text/utils.py

 from sqlite3  import connect, Row
+from re       import sub, match
 
 if __package__ == "services.text":
     from services.tools import mlog
@@ -6,6 +7,48 @@ else:
     from tools          import mlog
 
 
+def sanitize(value, specific_type=None):
+    """
+    simple and radical: leaves only alphanum and '@' '.' '-' ':' ',' '(', ')', '#', ' '
+
+    One of the main goals is to remove ';'
+    POSS better
+
+
+    args:
+        @value: any string to santize
+
+        @specific_type: None or 'url' or 'date'
+    """
+    vtype = type(value)
+    str_val = str(value)
+    clean_val = sub(r'^\s+', '', str_val)
+    clean_val = sub(r'\s+$', '', clean_val)
+
+    if not specific_type:
+        san_val = sub(r'[^\w@\.:,()# -]', '_', clean_val)
+    elif specific_type == "sbool":
+        # DB uses int(0) or int(1)
+        if match('^[01]$',clean_val):
+            san_val = int(clean_val)
+        else:
+            san_val = 0
+        # NB san_val_bool = bool(san_val)
+
+    elif specific_type == "surl":
+        san_val = sub(r'[^\w@\.: -/]', '_', clean_val)
+    elif specific_type == "sdate":
+        san_val = sub(r'[^0-9/-:]', '_', clean_val)
+
+    if vtype not in [int, str]:
+        raise ValueError("Value has an incorrect type %s" % str(vtype))
+    else:
+        # cast back to orginal type
+        san_typed_val = vtype(san_val)
+        return san_typed_val
+
+
+
 class CountryConverter:
     def __init__(self,dbname,dbtable,dbcolumnID,dbcolumnName):
 

services/tools.py

@@ -124,6 +124,18 @@ read_config()
 
 # ============================ other tools =====================================
 
+from urllib.parse import urlparse, urljoin, unquote
+def is_safe_url(target, host_url):
+    """
+    Checks if url is ok for redirects
+    cf. http://flask.pocoo.org/snippets/62/
+    """
+    ref_url = urlparse(host_url)
+    test_url = urlparse(urljoin(host_url, target))
+    return (test_url.scheme in ('http', 'https')
+            and ref_url.netloc == test_url.netloc)
+
+
 def re_hash(userinput, salt="verylonverylongverylonverylongverylonverylong"):
     """
     Build the captcha's verification hash server side