fix +10 explore

botapad/utils.py

@@ -22,7 +22,7 @@ def empty_graph(gid, headers, **kwargs):
     botapad.parse_csvrows( headers, separator='auto', debug=False)
 
     graph = bot.get_igraph(weight_prop="weight")
-    graph = prepare_graph(graph)
+    graph = prepare_graph(gid, graph)
     graph['starred'] = []
     graph['queries'] = []
     graph['meta'] = {  
@@ -34,7 +34,6 @@ def empty_graph(gid, headers, **kwargs):
             'star_count': len( graph['starred'] ),
             'stats' : {}
         }
-    #graph['meta']['pedigree'] = pedigree.compute(graph)
 
     return graph
 
@@ -72,19 +71,19 @@ def merge(gid, graph, g, index=None, vid=None, **kwargs):
     for v in g.vs:
         _vid = vid(gid,v)
         if _vid not in idx:
-                uuid = "%s" % graph.vcount()
-                attrs = v.attributes()
-                attrs['uuid'] = uuid
-
-                nodetype = nodetypes[attrs['nodetype']]
-                properties = nodetype['properties']
-                for k in properties:
-                    if k not in attrs['properties']:
-                        attrs['properties'][k] = properties[k]['default']
-                
-                graph.add_vertex( **attrs )
-                idx[ _vid ] = graph.vs[graph.vcount()-1]
-                
+            uuid = "%s" % graph.vcount()
+            attrs = v.attributes()
+            attrs['uuid'] = uuid
+
+            nodetype = nodetypes[attrs['nodetype']]
+            properties = nodetype['properties']
+            for k in properties:
+                if k not in attrs['properties']:
+                    attrs['properties'][k] = properties[k]['default']
+            
+            graph.add_vertex( **attrs )
+            idx[ _vid ] = graph.vs[graph.vcount()-1]
+          
                             
     edgetypes = [ e['name'] for e in graph['edgetypes'] ]
     for k in g['edgetypes']:
@@ -151,29 +150,93 @@ def graph_stats(graph, **kwargs):
 
     
 @Composable
-def prepare_graph(graph):
+def prepare_graph(gid, graph):
 
+    if not 'starred' in graph.attributes():
+        graph['starred'] = []
     if not 'meta' in graph.attributes():
-        graph['meta'] = { 'edge_count':0,'node_count':0, }
-    if 'nodetype' not in graph.vs.attribute_names():
-        graph.vs['nodetype'] = [ "T" for e in graph.vs ]
-    if 'uuid' not in graph.vs.attribute_names():
-        graph.vs['uuid'] = range(len(graph.vs))
-    if 'properties' not in graph.vs.attribute_names():
+        graph['meta'] = {
+            'node_count': graph.vcount(),
+            'edge_count': graph.ecount(),
+            'owner': "-",
+            'star_count': len( graph['starred'] ),
+            'upvotes': 0,
+            'votes': 0
+        }
+
+    if 'properties' not in graph.attributes():    
+        graph['properties'] = {}
+    
+    v_attrs = graph.vs.attribute_names()
+    if 'nodetypes' not in graph.attributes():    
+        graph['nodetypes'] = [{
+          "_uniq_key": "_%s_T" % gid,
+          "uuid": "_%s_T" % gid,
+          "description": "T", 
+          "name": "T", 
+          'count': graph.vcount(),
+          "properties": {
+            k : {
+              "choices": None, 
+              "default": None, 
+              "encoding": "utf8", 
+              "help": "", 
+              "multi": False, 
+              "type": "Text", 
+              "uniq": False, 
+              "vtype": "unicode"
+            }  for k in v_attrs 
+          }
+        }]
+           
+    if 'nodetype' not in v_attrs:
+        graph.vs['nodetype'] = [ "_%s_T" % gid for e in graph.vs ]
+    if 'uuid' not in v_attrs:
+        if 'id' in v_attrs: 
+            graph.vs['uuid'] = [ "%s" % int(e) for e in graph.vs['id'] ]
+        else :
+            graph.vs['uuid'] = [ "%s" % e for e in range(len(graph.vs)) ]
+    
+    if 'properties' not in v_attrs:
         props = [ {  }  for i in range(len(graph.vs))]
-        attrs = graph.vs.attribute_names()
         
         for p,v  in zip(props, graph.vs):
-            for e in attrs:
-                if e not in ['nodetype', 'uuid', 'properties' ]  :
+            for e in v_attrs:
+                if e not in ( 'nodetype', 'uuid', 'properties' )  :
                     p[e] = v[e]
-            if 'label' not in attrs:
+            if 'label' not in v_attrs:
                 p['label']  = v.index
                 
         graph.vs['properties'] = props
-           
+        for k in v_attrs:
+            if k not in ( 'nodetype', 'uuid', 'properties' )  :
+                del graph.vs[k]
+    
+    
+    e_attrs = graph.es.attribute_names()
+    
+    
+    {"choices": None, "default": None, "encoding": "utf8", "help": "", "multi": False, "type": "Text", "uniq": False, "vtype": "unicode" }
+    if 'edgetypes' not in graph.attributes():    
+        graph['edgetypes'] = [{ 
+            'count': graph.ecount(),
+            'description': "E",
+            'name': "E",
+            'properties': {
+                'label': {"choices": None, "default": None, "encoding": "utf8", "help": "",
+                          "multi": False, "type": "Text", "uniq": False, "vtype": "unicode" },
+                'weight': {"choices": None, "default": None, "encoding": "utf8", "help": "",
+                          "multi": False, "type": "Numeric", "uniq": False, "vtype": "float" }
+            },
+            'type_attributes' : {},
+            'uuid' : "_%s_E" % gid,
+            '_uniq_key' : "_%s_E" % gid,
+
+    } ]
+        
+              
     if 'edgetype' not in graph.es.attribute_names():
-        graph.es['edgetype'] = [ "T" for e in graph.es ]
+        graph.es['edgetype'] = [ "_%s_E" % gid for e in graph.es ]
     if 'uuid' not in graph.es.attribute_names():
         graph.es['uuid'] = range(len(graph.es))
     if 'properties' not in graph.es.attribute_names():
@@ -181,13 +244,16 @@ def prepare_graph(graph):
         attrs = graph.es.attribute_names()
         
         for p,v  in zip(props, graph.es):
-            for e in attrs:
-                if e not in ['edgetype', 'uuid', 'properties' ]  :
+            for e in e_attrs:
+                if e not in ['edgetype', 'uuid', 'properties' ]:
                     p[e] = v[e]
-            if 'label' not in attrs:
+            if 'label' not in e_attrs:
                 p['label']  = v.index
                 
         graph.es['properties'] = props
+        for k in e_attrs:
+            if k not in ( 'edgetype', 'uuid', 'properties', 'weight' )  :
+                del graph.es[k]
 
     if 'weight' not in graph.es.attribute_names():
         graph.es['weight'] = [1. for e in graph.es ]
@@ -263,4 +329,4 @@ def igraph2dict(graph, exclude_gattrs=[], exclude_vattrs=[], exclude_eattrs=[],
 def export_graph(graph, exclude_gattrs=[], exclude_vattrs=[], exclude_eattrs=[], id_attribute=None):
     return  igraph2dict(graph, exclude_gattrs, exclude_vattrs, exclude_eattrs, id_attribute)    
 
-    
\ No newline at end of file
+    

botapadapi.py

@@ -5,6 +5,7 @@ from flask import request, jsonify
 from flask import Response, make_response
 
 import igraph
+from igraph.utils import named_temporary_file
 import pickle
 import json
 
@@ -23,7 +24,7 @@ from cello.clustering import export_clustering
 
 from pdgapi.explor import ComplexQuery, AdditiveNodes, NodeExpandQuery, layout_api, clustering_api
 
-from botapad.utils import export_graph   
+from botapad.utils import export_graph, prepare_graph   
 from botapad import Botapad, BotapadError, BotapadParseError, BotapadURLError, BotapadCsvError, BotapadPostError
 
 from botapi import BotApiError, Botagraph,  BotaIgraph, BotLoginError
@@ -40,9 +41,9 @@ def pad2pdg(gid, url, host, key, delete, debug=False):
     botapad = Botapad(bot, gid, description, delete=delete)
     return botapad.parse(url, separator='auto', debug=debug)
         
-
+AVAILABLE_FORMATS = ('pickle', 'graphml', 'graphmlz', 'gml', 'pajek')
         
-def pad2igraph(gid, url, format, delete=False, debug=False, store="/pads/"):
+def pad2igraph(gid, url, format, delete=False, store="/pads/", debug=True):
 
     print ("format", gid, url, format )
     
@@ -60,7 +61,7 @@ def pad2igraph(gid, url, format, delete=False, debug=False, store="/pads/"):
             if graph.vcount() == 0 :
                 raise BotapadParseError(url, "Botapad can't create a graph without nodes.", None )
 
-            return graph
+            return prepare_graph(gid, graph)
             
         except BotapadParseError as e :
             log = botapad.get_log()
@@ -72,38 +73,38 @@ def pad2igraph(gid, url, format, delete=False, debug=False, store="/pads/"):
 
             
         
-    elif format in ('pickle', 'graphml', 'graphmlz', 'gml', 'pajek'):
+    elif format in AVAILABLE_FORMATS:
         content = None
         if url[0:4] == 'http':
             try :
                 url = convert_url(path)
                 if format in ( 'pickle', 'picklez'):
                     raise ValueError('no pickle from HTTP : %s ' % url )
-                log( " * Downloading %s %s\n" % (url, separator))
+                print( " === Downloading %s %s\n" % (url, separator))
                 content = requests.get(url).text
             except :
                 raise BotapadURLError("Can't download %s" % url, url)
 
-        elif DEBUG : 
-            try : 
-                content = open("%s/%s.%s" % (LOCAL_PADS_STORE, url, format) , 'rb').read()
+        else : 
+            try :                 
+                print (" === reading  %s/%s.%s" % (store, url, format) )
+                content = open("%s/%s.%s" % (store, url, format) , 'r').read()
             except Exception as err :
-                raise BotapadURLError("Can't open file %s: %s" % (url, err.message ), url)
+                raise BotapadURLError("Can't open file %s: %s" % (url, err), url)
 
-        print (" === reading  %s/%s.%s" % (LOCAL_PADS_STORE, url, format) )
 
         try :
             with named_temporary_file(text=False) as tmpf: 
                 outf = open(tmpf, "wt") 
                 outf.write(content) 
-                outf.close() 
-            
+                outf.close()         
                 graph =  igraph.read(tmpf, format=format) 
   
-            return graph
+            return prepare_graph(gid, graph)
         
         except Exception as err :
-            raise BotapadError('%s : cannot read %s file at %s : %s' % ( gid, format, url, err.message ))
+            raise
+            raise BotapadError('%s : cannot read %s file at %s : %s' % ( gid, format, url, err))
 
     else :
         raise BotapadError('%s : Unsupported format %s file at %s ' % ( gid, format, url ))
@@ -146,7 +147,7 @@ def explore_engine(graphdb):
         return db_graph(graphdb, query)
         
     @Composable
-    def subgraph(query, cut=100, weighted=True, length=7, mode=ALL, add_loops=False, ):
+    def subgraph(query, cut=100, weighted=True, length=7, mode=ALL, add_loops=False, **kwargs ):
 
         graph = db_graph(graphdb, query)
 
@@ -166,6 +167,17 @@ def explore_engine(graphdb):
             
         return graph.subgraph(vs)
 
+    @Composable
+    def filtering(graph, single_filtered=False, **kwargs):
+        if single_filtered :
+            to_del = []
+            for v in graph.vs:
+                if not len(v.neighbors()):
+                    to_del.add(v.index)
+            graph.vs.delete(to_del)
+        
+        return graph
+        
     from cello.graphs.transform import VtxAttr
     
     searchs = []
@@ -211,7 +223,7 @@ def expand_prox_engine(graphdb):
     engine.scores.setup(in_name="request", out_name="scores")
 
     ## Search
-    def expand(query, length=3, cut=100, nodes=False, weightings=None):
+    def expand(query, length=3, cut=300, nodes=False, weightings=None):
 
         graph = db_graph(graphdb, query)
         gid = query.get("graph")
@@ -223,17 +235,19 @@ def expand_prox_engine(graphdb):
         pz.update( { uuids[p] : 1./len(_nodes) for p in qnodes } )
         pz.update( { uuids[p] : 1. for p in qexpand })
         
-        weightings = ["1"] if weightings == None else weightings
+        print( "expand >>> %s" % pz )
+        weightings = ["1"] if weightings in ([], None) else weightings
         wneighbors = _weights(weightings)
         
         vs = pure_prox(graph, pz, length, wneighbors)
         vs = sortcut(vs, cut)
+        vs = [ (graph.vs[v[0]]['uuid'], v[1]) for v in vs ]
         return dict(vs)
 
     scores = Optionable("scores")
     scores._func = Composable(expand)
     scores.add_option("length", Numeric( vtype=int, default=3))
-    scores.add_option("cut", Numeric( vtype=int, default=50, max=100))
+    scores.add_option("cut", Numeric( vtype=int, default=50, max=300))
     scores.add_option("nodes", Boolean( default=True))
     scores.add_option("weighting", Text(choices=[  u"0", u"1", u"weight" ], multi=True, default=u"1", help="ponderation"))
     

botapadapp.py

@@ -58,7 +58,7 @@ except:
 # delete before import
 
 # app
-print( " == Botapad %s %s ==" % ("DEBUG" if DEBUG else "", "DELETE" if DELETE else "") )
+print( " == Botapad %s %s ==" % ("DEBUG" if DEBUG else "INFO", "DELETE" if DELETE else "") )
 print( " == Running with gunicorn : %s==" % (RUN_GUNICORN) )
 print( " == engines:%s static:%s padagraph:%s==" % (ENGINES_HOST, STATIC_HOST, PADAGRAPH_HOST) )
 print( " == REDIS STORAGE : %s ==  " % REDIS_STORAGE )
@@ -375,10 +375,21 @@ from reliure.pipeline import Optionable, Composable
 
 @Composable
 def _pad2igraph(gid, url, format="csv"):
-    graph = pad2igraph(gid, url, format, delete=True,debug=DEBUG, store=LOCAL_PADS_STORE)
+    graph = pad2igraph(gid, url, format, delete=True, store=LOCAL_PADS_STORE)
+    if not 'meta' in graph.attributes() : graph['meta'] = {}
+    graph['meta']['gid'] = gid
+    graph['meta']['graph'] = gid
+    graph['properties']=  {
+      "description": "%s imported from %s [%s]" % (gid, url, format) , 
+      "image": "", 
+      "name": gid, 
+      "tags": [
+        "Botapad", format
+      ]
+    }
     graph['meta']['owner'] = None
     graph['meta']['date'] = datetime.datetime.now().strftime("%Y-%m-%d %Hh%M")
-    return graph
+    return prepare_graph(gid, graph)
 
 import traceback
     
@@ -479,7 +490,7 @@ def botimport(repo, padurl, gid, content_type):
     
                 else :
 
-                    builder = _pad2igraph | prepare_graph | compute_pedigree | graph_stats
+                    builder = _pad2igraph | compute_pedigree | graph_stats
                     
                     graph = builder( gid, padurl, reader )
                     graphdb.set_graph(gid, graph)