stable version

PhyloSpark.py

@@ -37,7 +37,7 @@ class Period:
 		# self.sc = SparkContext("local","simple app")
 		self.sc = some_sc
 		self.utls = Utils()
-		self.abs_path = "/datasets/PubMed2014/chikungunya/YYYY.txt"
+		self.abs_path = "/datasets/PubMed2014/Full_pearson-filter/YYYY.json.txt"
 		self.idx = period
 		self.minsetsize = minsetsize
 
@@ -201,7 +201,19 @@ class Phylo:
 		print("")
 		self.utls = Utils()
 		# self.years = t #list( range( t[0], t[1]+1 ) ) # combinations(self.years, 2)
-		
+		self.p = {
+			"elemfrequency": { "type": int , "value": 1 },
+			"from_": { "type": int , "value": -1 },
+			"to_": { "type": int , "value": -1 },
+			"minfpgsupp": { "type": float , "value": 0.0001 },
+			"minfsetsize": { "type": int , "value": 4 },
+			"minsetdistance": { "type": int , "value": 0 },
+			"mram": { "type": int , "value": 40 },
+			"ncores": { "type": int , "value": 20 },
+			"query": { "type": str , "value": "" }
+		}
+
+		self.WL = False
 		self.years = False
 		self.pairs = False # 
 		self.pairsD = False #lineal_comparisons( self.years )
@@ -234,7 +246,6 @@ class Phylo:
 		K = self.minK
 
 		WL = self.sc.parallelize( WL ).map( lambda x: (int(x) , 1) )
-
 		Distribution = {}
 		for y in t :
 			period_ = Period( some_sc=self.sc , period=y , numpart=self.partitions )
@@ -244,7 +255,7 @@ class Phylo:
 				"|FIs|": n_fis,
 				"|T|": n_pubs
 			}
-		
+		self.WL = WL
 		years = list(  map( int , (sorted(Distribution.keys())) )  )
 		pairs, pairsD = lineal_comparisons( years )
 
@@ -326,12 +337,14 @@ class Phylo:
 		lll = PhyloMaker(thepath , thepath , 0.0)
 
 
+		print("DEBUGGING!!!")
+		print( sorted(list( self.phylomm.keys() )) )
 		found_distances = []
 		for idx in self.phylomm:
 			if self.phylomm[idx]["count"] > 0:
 				found_distances += self.phylomm[idx]["rdd_"].filter( lambda x: x[0]>=jacc_min ).collect()
-				# for i in found_distances:
-				# 	print(i)
+				for i in found_distances:
+					print(i)
 		print( "\t",jacc_min,"-> |JACCARD|:",len(found_distances) )
 
 		timerange = [ 1982 , 2014 ]
@@ -421,7 +434,7 @@ class Phylo:
 		# print( "|V_phy|:", len(phylojson["nodes"]))
 		# print( "|E_phy|:", len(phylojson["links"]))
 		# print( "|V|:", len(graphArray["nodes"]))
-		# print( "|E|:", len(graphArray["links"]))
+		# print( "|E|:", len(graphArray["links"]) )
 
 		return graphArray
 
@@ -434,18 +447,30 @@ class Phylo:
 	def partial_compute( self , p_ ):
 
 		current_years = set(self.years)
-		t = [ p_["from_"], p_["to_"] ]
+
+		# print("")
+		# print("")
+		# print(" In Partial Compute: ")
+		# pprint.pprint( p_ )
+		# print("")
+		# print("")
+		t = [ p_["from_"]["value"], p_["to_"]["value"] ]
 		add_years = set( range( t[0], t[1]+1 ) ) 
 		dt = add_years - current_years
 		if len(dt)>0:
 			print("need to compute extra:",dt)
 			N = self.yearsD
 			K = self.minK
+			Distribution = {}
 			for y in dt:
-				period_ = Period( some_sc=self.sc , period=y )
-				period_.FPG_unit()
+				period_ = Period( some_sc=self.sc , period=y , numpart=self.partitions )
+				n_pubs , n_fis = period_.FPG_unit( self.WL )
 				N[ y ] = period_
-			self.years = sorted(list(N.keys()))
+				Distribution[ y ] = {
+					"|FIs|": n_fis,
+					"|T|": n_pubs
+				}
+			years = list(  map( int , (sorted(N.keys())) )  )
 
 			daunion = current_years.union(add_years)
 			nyears = sorted(list(daunion))
@@ -462,16 +487,27 @@ class Phylo:
 				except:
 					break
 					pass
-			self.temp_matching( newcomps)
-			self.pairs, self.pairsD = lineal_comparisons( nyears )
+			# self.temp_matching( newcomps)
+			# self.pairs, self.pairsD = lineal_comparisons( nyears )
+			# pairs, pairsD = lineal_comparisons( years )
 
-		print("")
-		print( "old jacc:", self.minjacc )
-		print( "new jacc:", p_["minjaccard"]  )
-		phylojson = self.filter_jaccard ( p_["minjaccard"] )
-		print("")
 
-		return phylojson
+
+			pairs, pairsD = lineal_comparisons( nyears )
+
+			return { "diff_time": { "Distribution": Distribution , "years":nyears , "pairs":pairs , "pairsD":pairsD } }
+
+
+
+
+
+		# print("")
+		# print( "old jacc:", self.minjacc )
+		# print( "new jacc:", p_["minjaccard"]  )
+		# phylojson = self.filter_jaccard ( p_["minjaccard"] )
+		# print("")
+
+		return None
 
 
 

phylum_srv.py

@@ -99,6 +99,10 @@ def close_contexts():
 @app.route("/test" , methods=("GET", "POST",))
 def test_post():
 	pprint.pprint( request )
+	query = "void"
+	GG = False
+	stats = False
+	records = { "Count": 0 }
 	if request.method == "POST":
 		p_ = {}
 		print( "did a POST" )
@@ -108,7 +112,7 @@ def test_post():
 
 
 		t_i = time.time() ##
-		print( "downloading..." ) ##
+		# print( "downloading..." ) ##
 
 		query = urllib.parse.unquote(p_["query"])
 		query_size = p_["c"]
@@ -145,19 +149,31 @@ def test_post():
 
 		sID = p_["scontext"]
 		if sID not in I:
-			periods_ = range( int(p_["from_"]) , int(p_["to_"]) )
+
+			periods_ = range( int(p_["from_"]) , int(p_["to_"])+1 )
 			# periods_ = range( 2008 , 2011 )
 			minjaccard = float(p_["minsetdistance"])
 			I[ sID ] = Phylo( t=periods_ , minJ=float(p_["minsetdistance"]) , memm=p_["mram"]+"g" , ncores=p_["ncores"])
+
+
+			for k in p_:
+				if "scontext"!=k:
+					try:
+						# print("\t\t",k)
+						the_type = I[ sID ].p[k]["type"]
+						I[ sID ].p[k]["value"] = the_type(p_[k])
+					except:
+						# print("\t\t\t",k,"\t","error!!!")
+						pass
+			print("\nYour \"",sID,"\" instance has been CREATED.")
+			pprint.pprint( I[ sID ].p )
+			print( "" )
+			print( "" )
+
 			# executes a fp-growth per year
 			stats , years , pairs , pairsD = I[ sID ].FPG_chain( t=periods_ , WL=records["IdList"] )
 			I[ sID ].years , I[ sID ].pairs , I[ sID ].pairsD = years , pairs , pairsD
-			# print( "YEARS:",years )
-			# print("")
-			# print( "Pairs:" , pairs )
-			# print("")
-			# print( "PairsD:" , pairsD )
-			# print("")
+			
 			# # pairs of years to be multiplied
 			I[ sID ].temp_matching(  thepairs = pairs )
 			GG = I[ sID ].filter_jaccard ( jacc_min=minjaccard )
@@ -173,8 +189,43 @@ def test_post():
 			f.close()
 
 		else: 
-			GG = I[ sID ].partial_compute( p_ )
-			print("\nYour \"",sID,"\" instance has been modified.")
+			print("\nYour \"",sID,"\" instance has been MODIFIED.")
+			print( "\t" , "before:" )
+			pprint.pprint( I[ sID ].p )
+			print( "" )
+			params_ = {}
+			for k in p_:
+				if "scontext"!=k:
+					try:
+						# print("\t\t",k)
+						the_type = I[ sID ].p[k]["type"]
+						I[ sID ].p[k]["value"] = the_type(p_[k])
+					except:
+						# print("\t\t\t",k,"\t","error!!!")
+						pass
+			print( "" )
+			print( "\t" , "after:" )
+			pprint.pprint( I[ sID ].p )
+			print( "" )
+			
+			diff_ress = I[ sID ].partial_compute( I[ sID ].p )
+			if "diff_time" in diff_ress:
+				# I[ sID ].stats = diff_ress["stats"]
+				I[ sID ].years = diff_ress["diff_time"]["years"]
+				I[ sID ].pairs = diff_ress["diff_time"]["pairs"]
+				I[ sID ].pairsD = diff_ress["diff_time"]["pairsD"]
+
+				I[ sID ].temp_matching(  thepairs = diff_ress["diff_time"]["pairs"] )
+				GG = I[ sID ].filter_jaccard ( jacc_min=I[ sID ].p["minsetdistance"]["value"] )
+
+				Ya = str(I[ sID ].p["from_"]["value"])
+				Yb = str(I[ sID ].p["to_"]["value"])
+				GG_v = str( len( GG["nodes"] ) )
+				GG_e = str( len( GG["links"] ) )
+				query_file = query.replace(" ","_")+"__"+Ya+"-"+Yb+"__"+GG_v+"x"+GG_e
+				f = open( "static/Phylo/data/"+query_file+".json","w")
+				f.write( json.dumps( GG,indent=1 ) )
+				f.close()
 
 
 

static/phylomain.js

@@ -29,7 +29,7 @@ function makeMiniGraph(className , w , h , b , rawdata, isStacked) {
           labelsKMB:false,
           labelsKMG2:false,
           labelsUTC:false,
-          drawAxis:false, //axis legends
+          drawAxis:false, // axis legends
           legend: "follow",
 
         }
@@ -104,7 +104,7 @@ function doGraph(className, the_title, labels, donnes, isStacked) {
 
 /* Open when someone clicks on the span element */
 function openNav() {
-    console.clear()
+    // console.clear()
     document.getElementById("myNav").style.width = "100%";
 }
 
@@ -126,12 +126,15 @@ function dict_diff(obj1, obj2) {
 }
 
 
-var K = {}
-
+var K_i2s = {}
+// var K_s2i = {}
+var loader_ = '<img width=20 src="/static/Phylo/libs/img2/loading-bar.gif"></img>'
 var G = {
     "params_t0" : {},
 }
 
+var POST_ = false
+
 // "scontext"
 // "from_"
 // "to_"
@@ -164,20 +167,34 @@ function send_params( D ) {
         url: "/test",
         data: D ,
         dataType: 'json',
-        success : function(data){ 
+        success : function(data) {
+            // console.clear()
+            console.log( "EEEEHHH " )
             console.log( "EEEEHHH " )
             console.log( data )
             var toQuery = []
+
             if( data["phylojson"]["links"].length>0 ) {
+
+              POST_ = data
+
+              $("#pubmed_fetch-results").html( "<i>Perfect</i>" )
+
               for(var i in data["phylojson"]["nodes"]) {
                 var noeud = data["phylojson"]["nodes"][i]
-                if( noeud.type=="mesh_term" ) {
+                // // here noeud.label is Integer
+                if( noeud.type=="mesh_term" && !(noeud.label in K_i2s)  ) {
                   toQuery.push( noeud.label )
                 }
               }
-              get_ngrams( toQuery )
+
+              if(toQuery.length>0)
+                get_ngrams( toQuery )
+              
               MainFunction_new( data["phylojson"] )
               closeNav()
+            } else {
+              $("#pubmed_fetch-results").html( "<i>No results</i>" )
             }
         },
         error: function(jqxhr,textStatus,errorThrown) {
@@ -185,29 +202,29 @@ function send_params( D ) {
             console.log(textStatus);
             console.log(errorThrown);
             console.log('PROBLEM: ajax in #send_params").bind(click)');
-
         }
     });
     return false;
 }
 
 $("#pubmed_fetch").bind('click', function() {
-    
-  console.log( "hola mundo" )
-
-	var URL = "<URL>"
+      
+    console.log( "hola mundo" )
 
-	var params = { 
-		"a":"some", 
-		"b": "parameter" 
-	}
+    var URL = "<URL>"
 
+    var params = { 
+    	"a":"some", 
+    	"b": "parameter" 
+    }
 
     var count = $("#pubmed_scan-results").data( "count" )
     var query = $("#pubmedquery").val()
     if(query=="" || count==0)
         return false
 
+    $("#pubmed_fetch-results").html(loader_)
+    
     if ($.isEmptyObject(G["params_t0"])) {
 
         console.log( "params_t0   empty()" )
@@ -220,8 +237,9 @@ $("#pubmed_fetch").bind('click', function() {
         var params_diff = dict_diff( params_t1 , G["params_t0"] )
 
         // spark context has changed ->  change everything
-        if("scontext" in params_diff)
-            return send_params( params_t1 )
+        if("scontext" in params_diff) {
+          return send_params( params_t1 )
+        }
 
         if ( ! $.isEmptyObject(params_diff) ) {
             var params_cp = $.extend({}, params_diff);
@@ -231,7 +249,7 @@ $("#pubmed_fetch").bind('click', function() {
                 console.log( "there is a difference" )
                 console.log( params_diff )
                 params_diff["scontext"] = params_t1["scontext"]
-                // G["params_t0"] = params_t1
+                G["params_t0"] = params_t1
                 return send_params( params_diff )
             }
         }
@@ -295,8 +313,9 @@ $("#pubmed_scan").bind('click', function() {
     var query = $("#pubmedquery").val() 
     console.log( query )
     if(query!="") {
-        G["params_t0"] = {}
-        pubmed_scan( query )
+      $("#pubmed_scan-results").html(loader_)
+      G["params_t0"] = {}
+      pubmed_scan( query )
     }
     return false;
 });
@@ -311,15 +330,15 @@ function get_ngrams( query ) {
         data: DD ,
         success : function(data) { 
             for(var i in data){
-                K[i] = data[i]
+                K_i2s[i] = data[i]
             }
 
             for(var i in dicts.nodes) {
               if( dicts.nodes[i].type=="mesh_term" ) {
                 console.log ( dicts.nodes[i]  )
                 var ID = dicts.nodes[i].id
-                dicts.nodes[i].label = K[ID]
-                Nodes[ ID ].label = K[ID]
+                dicts.nodes[i].label = K_i2s[ID]
+                Nodes[ ID ].label = K_i2s[ID]
               }
             }
             partialGraph.refresh()