split db.json configuration file into servermenu and associated DBs

server_menu.json remains at project root while associated DBs are in new files called project_conf.json in each project dir

00.DOCUMENTATION/A-Introduction/README.md

@@ -96,4 +96,4 @@ There are two exceptions are the html page and the companion backends:
         -----------------------------------------------
       ```
 
-    4) finally, to use these backends as related documents search engines for a given graph, you'll need to configure your `db.json` file [as explained here](https://github.com/moma/ProjectExplorer/blob/master/00.DOCUMENTATION/A-Introduction/servermenu_config.md#more-relateddocs-settings). You can also find real-life examples in this distribution's `db.json` file.
+    4) finally, to use these backends as related documents search engines for a given graph, you'll need to configure your `project_conf.json` file [as explained here](https://github.com/moma/ProjectExplorer/blob/master/00.DOCUMENTATION/A-Introduction/project_config.md#more-relateddocs-settings). You can also find real-life examples in this distribution's `data/test/project_conf.json` file.

00.DOCUMENTATION/A-Introduction/servermenu_config.md → 00.DOCUMENTATION/A-Introduction/project_config.md

-## Servermenu Configuration
+## Project Configuration
 
-The servermenu file associates some metadata to each graph file.
+The directories under `data/` are called project directories and should contain:
+  - a graph file (ending in `.gexf` or `.json`)
+  - a `project_conf.json` to declare nodetypes and optionally link DBs to each graph file.
+  - optionally the said associated database of documents
 
-It is used if `sourcemode="servermenu"` is found in the url params or the settings file.
-
-By default, the file is called `./db.json` (this can be modified in settings under `TW.conf.paths.sourceMenu`)
-
-The `db.json` file of this distribution contains many examples.
+See for example `data/test/project_conf.json` in the project dir `test`.
 
 ------------------------------------------------------
 #### Minimal Config
 
-One minimal servermenu entry contains:
-  - a data dir path (**the project**)
-  - for each project: a list of graph files subpaths (**the graph source**)
-  - for each graph file: a list of expected node types starting by 'node0' (**the nodetypes**)
+One minimal entry contains for each graph file of the project dir : a list of expected node types starting by 'node0' (**the nodetypes**)
 
 ```json
-  "$$data/yourprojectdir": {
-    "graphs": {
-      "$$graph_source.gexf":{
-        "node0": {"name": "$$a_typename_of_nodes"}
-      },
-      "$$another_graph_source.json":{
-        "node0": {"name": "$$a_typename_of_nodes"}
-      }
-    }
+{
+  "$$graph_source.gexf":{
+    "node0": {"name": "$$a_typename_of_nodes"}
+  },
+  "$$another_graph_source.json":{
+    "node0": {"name": "$$a_typename_of_nodes"}
   }
+}
 ```
 The value **typename_of_nodes** should match the `type` or `category` attribute value of your nodes in the source gexf or json. It acts as a filter specifying the nodes that will be displayed in the ProjectExplorer GUI.
 
 ##### For a bipartite graph
 If the source file has 2 types of nodes, the config should look like this:
 ```json
-  "$$data/yourprojectdir": {
-    "graphs": {
-      "$$source_file.ext":{
-        "node0": {"name": "$$typename_of_term_nodes"},
-        "node1": {"name": "$$typename_of_context_nodes"}
-      }
-    }
+{
+  "$$source_file.ext":{
+    "node0": {"name": "$$typename_of_term_nodes"},
+    "node1": {"name": "$$typename_of_context_nodes"}
   }
+}
 ```
 
 NB: giving an empty string value to `node1.name` property will group all other found types in an "other" category.
@@ -49,12 +41,10 @@ Having a node0.name entry and optionally a node1.name is enough to display the g
 
 ###### Real life example
 ```json
-"data/comexjsons": {
-  "graphs": {
-    "graph_example.json": {
-      "node0": { "name": "NGram" },
-      "node1": { "name": "Document" }
-    }
+{
+  "graph_example.json": {
+    "node0": { "name": "NGram" },
+    "node1": { "name": "Document" }
   }
 }
 ```
@@ -62,7 +52,7 @@ Having a node0.name entry and optionally a node1.name is enough to display the g
 ------------------------------------------------------
 #### Activating relatedDocs LocalDB queries
 
-The servermenu file also allows configuration of associated queries for selected node(s): **relatedDocs**
+The `project_conf.json` file also allows configuration of associated queries for selected node(s): **relatedDocs**
 
 To enable it, you need to add to your node entry the `reldbs` key with minimally a db type :
 
@@ -75,7 +65,7 @@ To enable it, you need to add to your node entry the `reldbs` key with minimally
   }
 ```
 
-The presence of this property "reldbs" makes the API usable in db.json.
+The presence of this property "reldbs" makes the API usable in the interface.
 
 ##### More relatedDocs settings
 In addition, for full configuration, the following entries can be set under node0 or node1.
@@ -115,47 +105,47 @@ Expected type is `"twitter"` and no additional conf is needed (POSS for the futu
 ```
 
 ###### Real life examples
+
 ```json
-"data/gargistex": {
-  "graphs":{
-    "model_calibration.gexf": {
-      "node0": {
-        "name": "terms",
-        "reldbs": {
-          "csv": {
-            "file": "model_calibration.csv",
-            "qcols": ["title"],
-            "template": "bib_details"
-          },
-          "twitter": {}
-        }
+{
+  "model_calibration.gexf": {
+    "node0": {
+      "name": "terms",
+      "reldbs": {
+        "csv": {
+          "file": "model_calibration.csv",
+          "qcols": ["title"],
+          "template": "bib_details"
+        },
+        "twitter": {}
       }
     }
   }
-},
-"data/test": {
-  "first" : "mini_for_csv.gexf",
-  "graphs": {
-    "mini_for_csv.gexf": {
-      "node0": {
-        "name": "terms",
-        "reldbs": {
-          "csv": {
-            "file": "mini_for_csv.csv",
-            "qcols": ["title","keywords","text"],
-            "template": "bib_details"
-          },
-          "twitter": {}
-        }
-      },
-      "node1": {
-        "name": "authors",
-        "reldbs": {
-          "csv": {
-            "file": "mini_for_csv.csv",
-            "qcols": ["author"],
-            "template": "bib_details"
-          }
+}
+```
+
+
+```json
+{
+  "mini_for_csv.gexf": {
+    "node0": {
+      "name": "terms",
+      "reldbs": {
+        "csv": {
+          "file": "mini_for_csv.csv",
+          "qcols": ["title","keywords","text"],
+          "template": "bib_details"
+        },
+        "twitter": {}
+      }
+    },
+    "node1": {
+      "name": "authors",
+      "reldbs": {
+        "csv": {
+          "file": "mini_for_csv.csv",
+          "qcols": ["author"],
+          "template": "bib_details"
         }
       }
     }

00.DOCUMENTATION/A-Introduction/project_structure.md

@@ -15,7 +15,7 @@ Après commits de la semaine 26-30 juin 2017, une structure plus facile pour les
 │   └── (graphes par sous-projets)
 ├── explorerjs.html         <= point d'entrée lancement
 ├── settings_explorerjs.js  <= config générale
-├── db.json                 <= config additionnelle par sources gexf/json
+├── server_menu.json        <= liste des sources gexf/json par projet
 ├── favicon.ico
 ├── LICENSE
 ├── README.md

00.DOCUMENTATION/C-advanced/developer_manual.md

@@ -2,7 +2,7 @@ This is a stub for a future documentation for developers.
 
 #### About settings
   - system-wide settings are in `settings_explorerjs.js`
-  - source-by-source settings (nodetypes, relatedDocs APIs) are in `db.json`
+  - source-by-source settings (nodetypes, relatedDocs APIs) are in each project dir under `data/${projectname}/project_conf.json`
 
 ## Graph input choices
 
@@ -16,7 +16,7 @@ Tina allows 3 main ways of input :
 
 The `sourcemode` value is by default the one in settings_explorerjs.js (`TW.conf.sourcemode`), unless an url argument of the same name is present.
 
-The `serverfile` option has an extended version called `servermenu`. It opens the list of files from `db.json` on the server, providing a menu to choose from it.
+The `serverfile` option has an extended version called `servermenu`. It opens the list of files from `server_menu.json` on the server, providing a menu to choose from it.
 
 The detailed implementation of these choices can be found in the function `syncRemoteGraphData()` in main.js.
 

data/test/project_conf.json

+{
+  "mini_for_csv.gexf": {
+    "node0": {
+      "name": "term",
+      "reldbs": {
+        "csv": {
+          "file": "mini_for_csv.csv",
+          "qcols": ["title","keywords","text"],
+          "template": "bib_details"
+        },
+        "twitter": {}
+      }
+    },
+    "node1": {
+      "name": "person",
+      "reldbs": {
+        "csv": {
+          "file": "mini_for_csv.csv",
+          "qcols": ["author"],
+          "template": "bib_details"
+        }
+      }
+    }
+  }
+}

db.json

-{
-  "data/ClimateChange": {
-    "graphs": {
-      "Maps_S_800.gexf": {
-        "node0": {
-          "name": "ISItermsWhitelistV2Oct_5 & ISItermsWhitelistV2Oct_5",
-          "reldbs": {
-            "CortextDB": {
-              "file": "wos_climate-change_title_2014-2015.db",
-              "qtable": "ISItermsWhitelistV2Oct_5",
-              "template": "cortext_with_link"
-            },
-            "twitter": {}
-          }
-        }
-      }
-    }
-  },
-  "data/gargistex": {
-    "first" : "shale_and_ice.gexf",
-    "graphs":{
-      "shale_and_ice.gexf": {
-        "node0": {
-          "name": "terms",
-          "reldbs": {
-            "csv": {
-              "file": "shale_and_ice.csv",
-              "qcols": ["title"],
-              "template": "bib_details"
-            },
-            "twitter": {}
-          }
-        }
-      },
-      "model_calibration.gexf": {
-        "node0": {
-          "name": "terms",
-          "reldbs": {
-            "csv": {
-              "file": "model_calibration.csv",
-              "qcols": ["title"],
-              "template": "bib_details"
-            },
-            "twitter": {}
-          }
-        }
-      }
-    }
-  },
-  "data/comexjsons": {
-    "first" : "graph_example.json",
-    "graphs": {
-      "graph_example.json": {
-        "node0": { "name": "NGram" },
-        "node1": { "name": "Document" }
-      }
-    }
-  },
-  "data/test": {
-    "first" : "mini_for_csv.gexf",
-    "graphs": {
-      "mini_for_csv.gexf": {
-        "node0": {
-          "name": "term",
-          "reldbs": {
-            "csv": {
-              "file": "mini_for_csv.csv",
-              "qcols": ["title","keywords","text"],
-              "template": "bib_details"
-            },
-            "twitter": {}
-          }
-        },
-        "node1": {
-          "name": "person",
-          "reldbs": {
-            "csv": {
-              "file": "mini_for_csv.csv",
-              "qcols": ["author"],
-              "template": "bib_details"
-            }
-          }
-        }
-      },
-      "ProgrammeDesCandidats.gexf": {
-        "node0": {
-          "name": "term",
-          "reldbs": {
-            "csv": {
-              "file": "ProgrammesCandidats2017.csv",
-              "delim": ",",
-              "qcols": ["title","abstract","node0"],
-              "template": "bib_details"
-            }
-          }
-        }
-      }
-    }
-  },
-  "data/politoscope": {
-    "title":"Politoscope",
-    "date":"2017",
-    "first" : "ProgrammeDesCandidats.enrichi.gexf",
-    "graphs": {
-      "ProgrammeDesCandidats.enrichi.gexf": {
-        "node0": { "name": "terms", "reldbs": { "twitter": {} } }
-      },
-      "ProgrammeDesCandidats.gexf": {
-        "node0": { "name": "terms", "reldbs": { "twitter": {} } }
-      }
-    }
-  },
-  "graphapi": {
-    "__comment__": "special subproject for api sourcemode",
-    "__comment__": "allows setting nodetypes and reldocs for api graph sources",
-    "__comment__": "POSS can be avoided if 2 files; filemenu.json, nodeconfs.json",
-    "graphs": {
-      "default": {
-        "node0": { "name": "NGram",   "reldbs": { "twitter": {} } },
-        "node1": { "name": "Document" }
-      }
-    }
-  }
-}

server_menu.json

+{
+  "data/ClimateChange": [
+    "Maps_S_800.gexf"
+  ],
+  "data/gargistex": [
+    "shale_and_ice.gexf",
+    "model_calibration.gexf"
+  ],
+  "data/comexjsons": [
+    "graph_example.json",
+    "graph_example2.gexf"
+  ],
+  "data/test": [
+    "mini_for_csv.gexf"
+  ],
+  "data/politoscope": [
+    "ProgrammeDesCandidats.gexf"
+  ],
+  "first_project": "data/politoscope"
+}

settings_explorerjs.js

@@ -20,9 +20,10 @@ TW.conf = (function(TW){
   TWConf.sourcemode = "servermenu"   // accepted: "api" | "serverfile" | "servermenu" | "localfile"
 
   // ...or remote bridge to default source api ajax queries
-  TWConf.sourceAPI={};
-  TWConf.sourceAPI["forNormalQuery"] = "services/api/graph";
-  TWConf.sourceAPI["forFilteredQuery"] = "services/api/graph";
+  TWConf.sourceAPI={}
+  TWConf.sourceAPI["nodetypes"] = {"node0": "NGram", "node1": "Document" }
+  TWConf.sourceAPI["forNormalQuery"] = "services/api/graph"
+  TWConf.sourceAPI["forFilteredQuery"] = "services/api/graph"
 
 
   // Related documents (topPapers) data source
@@ -37,7 +38,7 @@ TW.conf = (function(TW){
 
   // routes by corresponding type
   TWConf.relatedDocsAPIS = {
-    "twitter": "https://134.158.74.111/twitter_search",
+    "twitter": "http://127.0.0.1:5000/twitter_search",
     "CortextDB": "twbackends/phpAPI",
     "csv": "twbackends/phpAPI"
   }
@@ -195,8 +196,8 @@ TW.conf = (function(TW){
     'templates': 'twlibs/hit_templates',
     'modules':   'twmodules',
 
-    'sourceFile': "",           // server-side .gexf|.json default source
-    'sourceMenu': "db.json"     // ...or server-side gexf default source list
+    'sourceFile': "",                 // server-side gexf|json default source
+    'sourceMenu': "server_menu.json"  // ...or server-side gexf|json source list
   }
   Object.freeze(TWConf.paths)  // /!\ to prevent path modification before load
 

twbackends/phpAPI/parameters_details.php

@@ -16,9 +16,9 @@ $our_libs_root="twbackends/phpAPI";       // for our few icons and jquery-ui
 
 // 2 - paths
 $mainpath=dirname(dirname(getcwd()))."/"; // default fs path to ProjectExplorer root
-                                          // (where data dir and db.json file reside)
+                                          // (where data dir resides with the projects)
 
-$project_menu_path = "db.json";
+$project_conf_fname='project_conf.json';  // default conf to look for in project dirs
 
 // 3 - others
 $ntypes = 2;  // max node types   (node0 & node1)
@@ -37,17 +37,25 @@ $csvquote = '"';
 $memserver = 'localhost';
 $memport = 11211;
 
+// the graph file
+$gexf= str_replace('"','',$_GET["gexf"]);
+
+// reconstruct the path $projectdir/project_conf.json
+$project_dir = dirname($gexf);
 
-// CONFIGURATION PARAMS
 // --------------------
 // parse db.json project menu and create a conf by file
-$conf = read_conf($mainpath.$project_menu_path, $ntypes, $supported_dbtypes);
+$conf = read_conf(
+  $mainpath.$project_dir.'/'.$project_conf_fname,
+  $ntypes,
+  $supported_dbtypes,
+  $project_dir
+);
 
 // =======================================
 // echodump("== READ CONF ==<br>", $conf);
 // =======================================
 
-$gexf= str_replace('"','',$_GET["gexf"]);
 $ntid = $_GET["ndtype"];
 $dbtype = $_GET["dbtype"];
 $ndtype = null;

twbackends/phpAPI/tools.php

@@ -21,80 +21,71 @@ function errmsg($message, $context, $more = "") {
   (please read A-Introduction/servermenu_config.md).<br>$more</p>";
 }
 
-// reading db.json associations
+// reading project_conf.json associations
 //    source graph file <=> (db, dbtype, cols) as relatedDocs php API
 //    1) we filter db.json entries by active/inactive nodetypes
 //    2) we filter db.json entries by supported dbtypes
-function read_conf($filepath, $ntypes, $our_dbtypes) {
-  $project_menu_fh = fopen($filepath, "r");
+function read_conf($filepath, $ntypes, $our_dbtypes, $project_dir) {
+  $project_conf_fh = fopen($filepath, "r");
   $json_st = '';
-  while (!feof($project_menu_fh)) {
-    $json_st .= fgets($project_menu_fh);
+  while (!feof($project_conf_fh)) {
+    $json_st .= fgets($project_conf_fh);
   }
-  fclose($project_menu_fh);
-  $project_menu = json_decode($json_st);
+  fclose($project_conf_fh);
+  $dir_items = json_decode($json_st);
+  $conf = array();
 
-  // echodump("== db.json menu ==", $project_menu);
+  // echodump("== project_conf.json ==", $dir_items);
 
-  $conf = array();
-  foreach ($project_menu as $project_dir => $dir_items){
-    // NB access by obj property (and not array key)
-    if (! property_exists($dir_items, 'graphs')) {
-      error_log("tw/phpAPI skip error: conf file ($project_menu_path)
-                 has no 'graphs' entry for project '$project_dir' !");
-      continue;
-    }
-    foreach ($dir_items->graphs as $graph_file => $graph_conf){
-      // echodump("== $graph_file ==", $graph_conf);
+  foreach ($dir_items as $graph_file => $graph_conf){
+    // echodump("== $graph_file ==", $graph_conf);
 
-      $gpath = $project_dir.'/'.$graph_file;
+    $gpath = $project_dir.'/'.$graph_file;
 
-      // NB a graph conf can now have different settings for each nodetype
-      // node0 <=> classic type 'semantic'
-      // node1 <=> classic type 'social'
+    // NB a graph conf can now have different settings for each nodetype
+    // node0 <=> classic type 'semantic'
+    // node1 <=> classic type 'social'
 
-      // NB2 now additionnally, each nodetype can have several dbs configured !
+    // NB2 now additionnally, each nodetype can have several dbs configured !
 
-      // $conf[$gpath] = array($ntypes);
+    // $conf[$gpath] = array($ntypes);
 
-      for ($i = 0 ; $i < $ntypes ; $i++) {
-        $conf[$gpath]['node'.$i] = array();
+    for ($i = 0 ; $i < $ntypes ; $i++) {
+      $conf[$gpath]['node'.$i] = array();
 
-        // check node0, node1, etc to see if they at least have a reldb conf
-        if (property_exists($graph_conf, 'node'.$i)
-            && property_exists($graph_conf->{'node'.$i}, 'reldbs') ) {
+      // check node0, node1, etc to see if they at least have a reldb conf
+      if (property_exists($graph_conf, 'node'.$i)
+          && property_exists($graph_conf->{'node'.$i}, 'reldbs') ) {
 
-          // check for each configured db that is listed under reldbs
-          $dbinfos = $graph_conf->{'node'.$i}->reldbs;
+        // check for each configured db that is listed under reldbs
+        $dbinfos = $graph_conf->{'node'.$i}->reldbs;
 
-          foreach ($dbinfos as $dbtype => $dbconf) {
-            // filter: supported and valid conf cases
-            if (in_array($dbtype, $our_dbtypes) && $dbconf->file) {
-              // we have a file for this nodetype and dbtype: copy entire conf
-              $conf[$gpath]['node'.$i][$dbtype] = (array)$dbconf ;
+        foreach ($dbinfos as $dbtype => $dbconf) {
+          // filter: supported and valid conf cases
+          if (in_array($dbtype, $our_dbtypes) && $dbconf->file) {
+            // we have a file for this nodetype and dbtype: copy entire conf
+            $conf[$gpath]['node'.$i][$dbtype] = (array)$dbconf ;
 
-              // update files path with dirpath
-              if (array_key_exists('file', $conf[$gpath]['node'.$i][$dbtype])) {
-                $relpath = $conf[$gpath]['node'.$i][$dbtype]['file'];
-                $conf[$gpath]['node'.$i][$dbtype]['file'] = $project_dir.'/'.$relpath;
-              }
+            // update files path with dirpath
+            if (array_key_exists('file', $conf[$gpath]['node'.$i][$dbtype])) {
+              $relpath = $conf[$gpath]['node'.$i][$dbtype]['file'];
+              $conf[$gpath]['node'.$i][$dbtype]['file'] = $project_dir.'/'.$relpath;
             }
           }
-
-          // echodump("got conf", $conf[$gpath]['node'.$i]);
         }
 
-        else {
-          // empty array <=> inactive nodetype or no supported dbs
-          $conf[$gpath]['node'.$i] = array ();
-        }
-        // POSS here info on higher level may be propagated for lower ones
-        //     (ex: if dbtype is on the project level, its value should count
-        //          for each source file in the project unless overridden)
+        // echodump("got conf", $conf[$gpath]['node'.$i]);
       }
+
+      else {
+        // empty array <=> inactive nodetype or no supported dbs
+        $conf[$gpath]['node'.$i] = array ();
+      }
+      // POSS here info on higher level may be propagated for lower ones
+      //     (ex: if dbtype is on the project level, its value should count
+      //          for each source file in the project unless overridden)
     }
   }
-
   // echodump("full conf", $conf);
   return $conf;
 }

twmain/Tinaweb.js

@@ -868,7 +868,7 @@ var TinaWebJS = function ( sigmacanvas ) {
 
     // to init local, instance-related listeners (need to run at new sigma instance)
     // args: @partialGraph = a sigma instance
-    this.initSigmaListeners = function(partialGraph, initialActivetypes, initialActivereltypes, optionalConfEntry) {
+    this.initSigmaListeners = function(partialGraph, initialActivetypes, initialActivereltypes, optionalRelDocsConf) {
 
       // console.log("initSigmaListeners TW.categories / types array / reltypeskeys array: ", TW.categories, initialActivetypes, initialActivereltypes)
 
@@ -1085,13 +1085,10 @@ var TinaWebJS = function ( sigmacanvas ) {
         $("#read-opposite-neighs").readmore({maxHeight:200});
       }
 
-      // initialize reldocs tabs if declared in additionalConf
-      if (TW.conf.getRelatedDocs) {
-
-        let moreConfKey = optionalConfEntry || TW.File
-
-        resetTabs(initialActivetypes, TW.gmenuInfos[moreConfKey])
-
+      // initialize reldocs tabs if declared in optionalRelDocsConf
+      // (optionalRelDocsConf function-scope name of TW.currentRelDocsDBs)
+      if (TW.conf.getRelatedDocs && optionalRelDocsConf) {
+        resetTabs(initialActivetypes, optionalRelDocsConf)
       }
 
       // defaultColoring: an attribute name to immediately apply color with

twmain/enviroment.js

@@ -248,7 +248,7 @@ function createFilechooserEl () {
           TW.resetGraph()
 
           // run
-          mainStartGraph(theFormat, rdr.result, null, TW.instance)
+          mainStartGraph(theFormat, rdr.result, TW.instance)
 
           // NB 3rd arg null = we got no additional conf for this "unknown" file
 
@@ -1163,7 +1163,7 @@ activateRDTab = function(elTgt) {
 }
 
 
-// set up tabs for a given activetypes state and db.json entry
+// set up tabs for a given activetypes state and project_conf.json relDB entry
 function resetTabs(activetypes, dbconf) {
   let ul = document.getElementById('reldocs-tabs')
   let divs = document.getElementById('reldocs-boxes')
@@ -1178,22 +1178,20 @@ function resetTabs(activetypes, dbconf) {
     return
   }
 
-  // console.log("dbconf for this source", dbconf)
-
   // for all active nodetypes
   for (let nodetypeId in activetypes) {
     if (activetypes[nodetypeId]) {
-      let additionalConf = dbconf[nodetypeId]
-
-      if (TW.conf.debug.logSettings)
-          console.log ("additionalConf for this source", additionalConf)
-
       let possibleAPIs = []
-      if (additionalConf.reldbs) {
-        possibleAPIs = additionalConf.reldbs
 
-        // 3 vars to know which one to activate
-        let nAPIs = Object.keys(possibleAPIs).length
+      if (dbconf[nodetypeId]) {
+        if (TW.conf.debug.logSettings)
+          console.log ("additional db conf for this source", dbconf[nodetypeId])
+        possibleAPIs = dbconf[nodetypeId]
+      }
+
+      let nAPIs = Object.keys(possibleAPIs).length
+      if (nAPIs > 0) {
+        // some more vars to know which one to activate
         let iAPI = 0
         let didActiveFlag = false
 
@@ -1266,7 +1264,7 @@ function openGraph(graphPath){
     TW.resetGraph()
 
     TW.File = graphPath
-    mainStartGraph(newDataRes["format"], newDataRes["data"], TW.File, TW.instance)
+    mainStartGraph(newDataRes["format"], newDataRes["data"], TW.instance)
     writeLabel(graphPathToLabel(graphPath))
 }
 //============================= </OTHER ACTIONS > =============================//

twmain/extras_explorerjs.js

@@ -446,7 +446,7 @@ function getTopPapers(qWords, nodetypeId, chosenAPI, tgtDivId) {
     });
   }
   else {
-    let thisRelDocsConf = TW.gmenuInfos[TW.File][nodetypeId]["reldbs"][chosenAPI]
+    let thisRelDocsConf = TW.currentRelDocsDBs[nodetypeId][chosenAPI]
     // /!\ documentation and specification needed for the php use cases /!\
     let joinedQ = JSON.stringify(qWords).split('&').join('__and__');
     // cf. the php code for these url args:
@@ -511,9 +511,9 @@ function displayTopPapers(jsonHits, ndtypeId, chosenAPI, targetDiv) {
     toHtmlFun = renderTweet
   }
   else if (chosenAPI == "CortextDB" || chosenAPI == "csv") {
-    let thisRelDocsConf = TW.gmenuInfos[TW.File][ndtypeId]["reldbs"]
-    if (thisRelDocsConf && thisRelDocsConf[chosenAPI] && thisRelDocsConf[chosenAPI].template) {
-      toHtmlFun = makeRendererFromTemplate(thisRelDocsConf[chosenAPI].template)
+    let thisRelDocsConf = TW.currentRelDocsDBs[ndtypeId][chosenAPI]
+    if (thisRelDocsConf && thisRelDocsConf.template) {
+      toHtmlFun = makeRendererFromTemplate(thisRelDocsConf.template)
     }
     else {
       console.warn(`no rendering template found in ${TW.conf.paths.sourceMenu} for this source ${TW.File}...`)

twmain/main.js

@@ -3,12 +3,7 @@
 //  ======= [ main TW properties initialization ] ======== //
 
 
-TW.File = ""            // remember the currently opened file
-
-// used iff servermenu
-TW.gmenuPaths={};       // map [graphname => graphsource] for file selectors
-TW.gmenuInfos={};       // map [graphsource => { node0/1 categories
-                        //                      + relatedDocs db fields names}]
+TW.File = ""              // remember the currently opened file
 
 // a system state is the summary of tina situation
 TW.initialSystemState = {
@@ -47,16 +42,18 @@ TW.instance.init()
 TW.instance.initGUIListeners();
 TW.instance.initSearchListeners();
 
+TW.currentRelDocsDBs = []  // to make available dbconf to topPapers
+
 // show the custom name + home link of the app
 writeBrand(TW.conf.branding, TW.conf.brandingLink)
 
 // choosing the input
 // -------------------
 // type of input
-var sourcemode = isUndef(getUrlParam.sourcemode) ? TW.conf.sourcemode : getUrlParam.sourcemode
+TW.sourcemode = isUndef(getUrlParam.sourcemode) ? TW.conf.sourcemode : getUrlParam.sourcemode
 
 // if page is being run locally ==> only possible source shall be via file input
-if (window.location.protocol == 'file:' || sourcemode == 'localfile') {
+if (window.location.protocol == 'file:' || TW.sourcemode == 'localfile') {
 
   let inputDiv = document.getElementById('localInput')
   inputDiv.style.display = 'block'
@@ -77,23 +74,11 @@ if (window.location.protocol == 'file:' || sourcemode == 'localfile') {
 }
 // traditional cases: remote read from API or prepared server-side file
 else {
-  try {
-    // we'll first retrieve the menu of available sources in db.json,
-    // then get the real data in a second ajax via API or server file
-    [TW.gmenuInfos, TW.File] = readMenu(TW.conf.paths.sourceMenu)
-
-    //     NB: this menu used to be a file list for only one sourcemode
-    //         but now also contains settings for nodetypes and for
-    //         companion APIs (reldocs searches)
-    //      => we read it for all cases now
-  }
-  catch(e) {
-    console.error(`Couldn't read ${TW.conf.paths.sourceMenu}, trying to start with settings_explorer defaults.`)
-  }
-
-  // NB it will use global urlParams and TW.settings to choose the source
-  var [inFormat, inData, inConfKey, mapLabel] = syncRemoteGraphData()
-  mainStartGraph(inFormat, inData, inConfKey, TW.instance)
+  // NB it will
+  //     - use global urlParams, TW.conf and possibly server_menu.json
+  //     - choose the source and format
+  var [inFormat, inData, mapLabel] = syncRemoteGraphData()
+  mainStartGraph(inFormat, inData, TW.instance)
   writeLabel(mapLabel)
 }
 
@@ -105,18 +90,16 @@ else {
 function syncRemoteGraphData () {
   var inFormat;      // = { db|api.json , somefile.json|gexf }
   var inData;        // = {nodes: [....], edges: [....], cats:...}
-  var inConfKey;     // = source name for entry in db.json
 
   var mapLabel;      // user displayed label for this input dataset
 
   // case (1) read from remote DB via API bridge fetching
   // ex: /services/api/graph?q=filters...
-  if (sourcemode == "api") {
+  if (TW.sourcemode == "api") {
     console.log("input case: api, using TW.conf.sourceAPI")
 
     // the only API format, cf. inData
     inFormat = 'json'
-    inConfKey = 'graphapi/default'
 
     // TODO-rename: s/nodeidparam/srcparams
     var sourceinfo = getUrlParam.nodeidparam
@@ -218,7 +201,7 @@ function syncRemoteGraphData () {
   }
 
 
-  // sourcemode == "serverfile" or "servermenu"
+  // TW.sourcemode == "serverfile" or "servermenu"
   // cases            (2)       and     (3) : read a file from server
   else {
     console.log("input case: server-side file, using TW.conf.paths.sourceMenu and/or TW.File")
@@ -241,20 +224,35 @@ function syncRemoteGraphData () {
     //      --> if @file also in url, choose the db.json one matching
     //      --> otherwise, choose the "first_file" from db.json list
     // menufile case : a list of source files in ./db.json
-    if (sourcemode == 'servermenu') {
-        console.log("using entire FILEMENU TW.conf.paths.sourceMenu")
-
-        // chooser menu
-        var files_selector = '<select onchange="openGraph(this.options[this.selectedIndex].dataset.fullpath)">'
-        for (let fullPath in TW.gmenuInfos) {
-          let shortname = graphPathToLabel(fullPath)
-          let preSelected = (fullPath == TW.File)
-          files_selector += `<option ${preSelected ? "selected":""} data-fullpath="${fullPath}">`+shortname+'</option>'
+    if (TW.sourcemode == 'servermenu') {
+        console.log("== servermenu mode ==")
+        if (! linkCheck(TW.conf.paths.sourceMenu)) {
+          console.error(`servermenu mode: Couldn't read ${TW.conf.paths.sourceMenu}, referenced by TWConf.paths under sourceMenu... file is not accessible, trying to start with settings_explorer defaults.`)
         }
-        files_selector += "</select>"
-        $("#network").html(files_selector)
+        else {
+          // we'll first retrieve the menu of available sources in server_menu.json,
+          // then get the real data in a second ajax via API or server file
+          let [gmenu, firstProject] = readMenu(TW.conf.paths.sourceMenu)
+
+          // if TW.File was not set we keep the first one from readMenu
+          if (!TW.File) {
+            TW.File = firstProject + "/" + gmenu[firstProject][0]
+          }
 
-        // NB if TW.File was not in the list we keep the first one from readMenu
+          // chooser menu
+          var files_selector = '<select onchange="openGraph(this.options[this.selectedIndex].dataset.fullpath)">'
+          for (var projectPath in gmenu) {
+            for (var i in gmenu[projectPath]) {
+              let filePath = gmenu[projectPath][i]
+              let fullPath = projectPath+'/'+filePath
+              let shortname = graphPathToLabel(fullPath)
+              let preSelected = (fullPath == TW.File)
+              files_selector += `<option ${preSelected ? "selected":""} data-fullpath="${fullPath}">`+shortname+'</option>'
+            }
+          }
+          files_selector += "</select>"
+          $("#network").html(files_selector)
+        }
     }
 
     // 3) @mode is serverfile a or b (default)
@@ -271,7 +269,6 @@ function syncRemoteGraphData () {
     var finalRes = AjaxSync({ url: TW.File });
     inData = finalRes["data"]
     inFormat = finalRes["format"]
-    inConfKey = TW.File
     mapLabel = graphPathToLabel(TW.File)
 
     if (TW.conf.debug.logFetchers) {
@@ -282,7 +279,7 @@ function syncRemoteGraphData () {
     }
   }
 
-  return [inFormat, inData, inConfKey, mapLabel]
+  return [inFormat, inData, mapLabel]
 }
 
 
@@ -300,9 +297,10 @@ function syncRemoteGraphData () {
 //  args:
 //     inFormat: 'json' or 'gexf'
 //     inData: source data as str
-//     inConfKey: optional entry in db.json to declare nodetypes and reldbs
 //     twInstance: a tinaweb object (gui, methods) to bind the graph to
-function mainStartGraph(inFormat, inData, inConfKey, twInstance) {
+//
+// NB: function also uses TW.File to get the associated project_conf.json entry
+function mainStartGraph(inFormat, inData, twInstance) {
 
   // Graph-related vars
   // ------------------
@@ -325,15 +323,30 @@ function mainStartGraph(inFormat, inData, inConfKey, twInstance) {
     alert("error on data load")
   }
   else {
-      // override default categories with the ones from db.json if present
-      let additionalConf = []
-      if (TW.gmenuInfos[inConfKey]) {
-        additionalConf = TW.gmenuInfos[inConfKey]
+      let optNodeTypes = null
+      let optRelDBs = null
+      if (TW.sourcemode == "api") {
+        optNodeTypes = TW.conf.sourceAPI.nodetypes
+      }
+      else {
+        // try and retrieve associated conf
+        [optNodeTypes, optRelDBs] = readProjectConf(TW.File)
+
+        // export to global for getTopPapers function :/
+        if (optRelDBs) {
+          TW.currentRelDocsDBs = optRelDBs
+        }
+      }
+
+      if (TW.conf.debug.logSettings) {
+        console.log("READ project_conf.json nodetypes", optNodeTypes)
+        console.log("READ project_conf.json relatedDBs", optRelDBs)
       }
 
+      // override default categories with project_conf.json if present
       // parse the data
       if (TW.conf.debug.logParsers)   console.log("parsing the data")
-      let start = new ParseCustom(  inFormat , inData, additionalConf );
+      let start = new ParseCustom(  inFormat , inData, optNodeTypes);
       let catsInfos = start.scanFile();
 
       TW.categories = catsInfos.categories
@@ -473,7 +486,7 @@ function mainStartGraph(inFormat, inData, inConfKey, twInstance) {
         TW.partialGraph,
         initialActivetypes,      // to init node sliders and .class gui elements
         initialActivereltypes,   // to init edge sliders
-        inConfKey                // to init relatedDocs
+        optRelDBs                // optional conf to init relatedDocs
       )
 
       // set the initial color

twmain/methods.js

@@ -61,13 +61,12 @@ TW.pushGUIState = function( args ) {
     }
 
     // recreate tabs after type changes
-    // db.json conf entry (£TODO unify s/(?:TW.File|inConfKey)/TW.sourceId/g)
-    let inConfKey = (sourcemode != "api") ? TW.File : 'graphapi/default'
+    // project_conf.json conf entry (POSS unify s/TW.File/TW.sourceId/g)
     if (TW.conf.getRelatedDocs
         && !isUndef(args.activetypes)
-        && TW.gmenuInfos[inConfKey]) {
+        && TW.currentRelDocsDBs) {
 
-      resetTabs(newState.activetypes, TW.gmenuInfos[inConfKey])
+      resetTabs(newState.activetypes, TW.currentRelDocsDBs)
     }
 
     // 4) store it in TW.states
@@ -118,86 +117,127 @@ function graphPathToLabel(fullPath) {
     )
 }
 
-// read all sources' detailed confs
-//  -> list of source paths available
-//  -> declared nodetypes
-//  -> declared rDocs conf
-function readMenu(infofile) {
-
-  // exemple entry
-  // --------------
-  // "data/gargistex": {
-  //   "first" : "shale_and_ice.gexf",
-  //   "graphs":{
-  //     "shale_and_ice.gexf": {
-  //       "node0": {
-  //         "name": "terms",
-  //         "reldbs": {
-  //           "csv": {
-  //             "file": "shale_and_ice.csv",
-  //             "qcols": ["title"],
-  //             "template": "bib_details"
-  //           },
-  //           "twitter": {}
-  //         }
-  //       }
-  //     }
-  //   }
-  // }
-
 
+// read the server files menu list
+// (list of source paths available)
+function readMenu(infofile) {
   if (TW.conf.debug.logFetchers)  console.info(`attempting to load filemenu ${infofile}`)
+
   var preRES = AjaxSync({ url: infofile, datatype:"json" });
 
   if (preRES['OK'] && preRES.data) {
     if (TW.conf.debug.logFetchers) console.log('initial AjaxSync result preRES', preRES)
   }
 
-  // 1 - store the first one (b/c we'll loose order)
-  var first_file = "", first_dir = "" , first_path = ""
-  for( var path in preRES.data ) {
-    if (TW.conf.debug.logFetchers) console.log("db.json path", path)
-    first_file = preRES.data[path]["first"] || Object.keys(preRES.data[path]["graphs"])[0]
-    first_dir = path
-    break;
+  // we just make a clean copy, skipping invalid entries
+  let serverMenu = {}
+  let firstProject = null
+
+  // check all the paths and associated sources
+  for( var projectPath in preRES.data ) {
+    if (! preRES.data[projectPath] || ! preRES.data[projectPath].length) {
+      console.warn("sourceMenu: Skipping invalid project entry:", projectPath)
+    }
+    else {
+      if (projectPath == "first_project") {
+        firstProject = preRES.data['first_project']
+      }
+      else {
+        // test and copy if ok
+        serverMenu[projectPath] = []
+        for (var l in preRES.data[projectPath]) {
+          let sourceFile = preRES.data[projectPath][l]
+          let fileExists = linkCheck(projectPath+'/'+sourceFile)
+          if (fileExists) {
+            serverMenu[projectPath].push(sourceFile)
+          }
+        }
+        if (! serverMenu[projectPath].length) {
+          console.warn(`sourceMenu: Skipping project path ${projectPath} (none of the referenced source files are present.)`)
+          delete serverMenu[projectPath]
+        }
+      }
+    }
+  }
+  return [serverMenu, firstProject]
+}
+
+// read project_conf.json files
+function readProjectConf(aFilePath) {
+  let declaredNodetypes
+  let declaredDBConf
+
+  // we assume the filePath is of the form projectPath/sourceFile
+  let split = aFilePath.match("^(.*)/([^/]+)$")
+  if (! split) {
+    console.warn (`couldn't read associated conf for file, ${aFilePath}, will try using default nodetypes`)
   }
-  first_path = first_dir+"/"+first_file
+  else {
+    let projectPath = split[1]
+    let filePath = split[2]
 
-  // 2 - process all the paths and associated confs
-  let details = {}
+    let projectConfFile = projectPath + '/project_conf.json'
 
-  for( var path in preRES.data ) {
-      var theGraphs = preRES.data[path]["graphs"]
+    if (! linkCheck(projectConfFile)) {
+      console.warn (`no project_conf.json next to the file, ${aFilePath}, will try using default nodetypes`)
+    }
+    else {
 
-      for(var aGraph in theGraphs) {
-          let fullPath = path+"/"+aGraph
-          // ex : "RiskV2PageRank1000.gexf":data/AXA/RiskV2PageRank1000.gexf
-          // (we assume there's no duplicate basenames)
+      if (TW.conf.debug.logFetchers)  console.info(`attempting to load project conf ${projectConfFile}`)
+      var pjconfRes = AjaxSync({ url: projectConfFile, datatype:"json" });
 
-          if (TW.conf.debug.logSettings)
-            console.log("db conf entry: " + fullPath)
+      if (TW.conf.debug.logFetchers) console.log('project conf AjaxSync result pjconfRes', pjconfRes)
 
-          // for associated LocalDB php queries: CSV (or CortextDBs sql)
-          if (theGraphs[aGraph]) {
-            let gSrcEntry = theGraphs[aGraph]
-            details[fullPath] = new Array(2)
-            if (gSrcEntry.node0) {
-              details[fullPath][0] = gSrcEntry.node0
-            }
-            if (gSrcEntry.node1) {
-              details[fullPath][1] = gSrcEntry.node1
-            }
+      if (! pjconfRes['OK']
+         || ! pjconfRes.data
+         || ! pjconfRes.data[filePath] ) {
+         console.warn (`project_conf.json in ${projectPath} is not valid json or does not contain an entry for ${filePath}, will try using default nodetypes`)
+      }
+      else {
+        let confEntry = pjconfRes.data[filePath]
+        for (var ndtype in confEntry) {
+          if (! /node\d+/.test(ndtype)) {
+            console.warn (`project_conf.json in ${projectPath}, in the entry for ${filePath}, should only contain properties like 'node0', 'node1', etc.`)
           }
           else {
-            details[fullPath] = null
+            if (! confEntry[ndtype].name) {
+              console.warn (`project_conf.json in ${projectPath}, in the entry for ${filePath}.${ndtype}, should contain a 'name' property`)
+            }
+            // valid case !
+            else {
+              if (! declaredNodetypes)  declaredNodetypes = {}
+
+              // fill simple nodetypes
+              declaredNodetypes[ndtype] = confEntry[ndtype].name
+            }
+
+            // optional reldbs -----------------------
+            if (confEntry[ndtype].reldbs) {
+              if (! declaredDBConf)     declaredDBConf = {}
+
+              // it must match because we tested well-formedness above
+              let ndtypeId = ndtype.match(/^node(\d+)/)[1]
+
+              declaredDBConf[ndtypeId] = {}
+              for (var dbtype in confEntry[ndtype].reldbs) {
+                if (! TW.conf.relatedDocsAPIS[dbtype]) {
+                  console.info (`project_conf.json: ${projectPath}.${filePath}.${ndtype}: skipping unknown related docs db type **${dbtype}**. The only available db types are: ${Object.keys(TW.conf.relatedDocsAPIS)}.`)
+                }
+                else {
+                  declaredDBConf[ndtypeId][dbtype] = confEntry[ndtype].reldbs[dbtype]
+                }
+              }
+            }
+            // ----------------------------------------
+
           }
+        }
       }
+    }
   }
-
-  return [details, first_path]
+  return [declaredNodetypes, declaredDBConf]
 }
 
-
 // settings: {norender: Bool}
 function cancelSelection (fromTagCloud, settings) {
     if (TW.conf.debug.logSelections) { console.log("\t***in cancelSelection"); }

twmain/sigma.parseCustom.js

@@ -3,7 +3,7 @@
 //   (for instance loop on full gexf in scanGexf then again in dictfyGexf)
 
 // Level-01
-var ParseCustom = function ( format , data, optionalConf ) {
+var ParseCustom = function ( format , data, optionalNodeConf ) {
 
     if (format == 'gexf') {
       this.data = $.parseXML(data)
@@ -14,12 +14,10 @@ var ParseCustom = function ( format , data, optionalConf ) {
     this.format = format;
     this.nbCats = 0;
 
-    this.additionalConf = optionalConf
-
     // input = GEXFstring
     this.getGEXFCategories = function() {
         let observedCategories = scanGexf(this.data)
-        let finalCategories = sortNodeTypes(observedCategories, this.additionalConf)
+        let finalCategories = sortNodeTypes(observedCategories,optionalNodeConf)
         return finalCategories;
     }// output = {'cats':[ "cat1" , "cat2" , ...], 'rev': {cat1: 0, cat2: 1...}}
 
@@ -34,7 +32,7 @@ var ParseCustom = function ( format , data, optionalConf ) {
     // input = JSONstring
     this.getJSONCategories = function(json) {
       let observedCategories = scanJSON(this.data)
-      let finalCategories = sortNodeTypes(observedCategories, this.additionalConf)
+      let finalCategories = sortNodeTypes(observedCategories, optionalNodeConf)
       return finalCategories;
     }// output = {'cats':[ "cat1" , "cat2" , ...], 'rev': {cat1: 0, cat2: 1...}}
 
@@ -212,15 +210,20 @@ function scanGexf(gexfContent) {
 // expected content: usually a just a few cats over all nodes
 // ex: terms
 // ex: ISItermsriskV2_140 & ISItermsriskV2_140
-function sortNodeTypes(observedTypesDict, optConf) {
+// optional arg optionalNodeConf should contain keys of the form:
+//    "node0": "NGram",
+//    "node1": "Document"
+//     etc.
+// (it's read from project_conf.json)
+function sortNodeTypes(observedTypesDict, optionalNodeConf) {
   var observedTypes = Object.keys(observedTypesDict)
   observedTypes.sort(function(a,b) {return observedTypesDict[b] - observedTypesDict[a]})
 
   let nbNodeTypes = 2
   var declaredTypes = []
   for (var i = 0 ; i < nbNodeTypes ; i++ ) {
-    if (optConf[i] && optConf[i].name) {
-      declaredTypes[i] = optConf[i].name
+    if (optionalNodeConf["node"+i]) {
+      declaredTypes[i] = optionalNodeConf["node"+i]
       if (TW.conf.debug.logSettings)
         console.log("expected cat (from db.json addtional conf)", i, declaredTypes[i])
     }