last things

Twitter/julian.py

@@ -24,10 +24,13 @@ class Hola:
 		partial_match_d = {}
 
 		for w in elems:
-			if "*" in w:
-				partial_match.append( w.replace("*","%") )
-			else:
-				exact_match.append( w )
+			# w = "%"+w+"%"
+			partial_match.append( "%"+w+"%" )
+		# for w in elems:
+		# 	if "*" in w:
+		# 		partial_match.append( w.replace("*","%") )
+		# 	else:
+		# 		exact_match.append( w )
 
 		conditions = []
 
@@ -36,10 +39,10 @@ class Hola:
 			concats = "\" OR "+what+" like \""
 			partial_ += concats.join( partial_match ) +'"'
 			conditions.append( partial_ )
-			if merge:
-				for w in partial_match:
-					wc = w[:]
-					partial_match_d[wc.replace("%","").lower()] = w.replace("%","*")
+			# if merge:
+			# 	for w in partial_match:
+			# 		wc = w[:]
+			# 		partial_match_d[wc.replace("%","").lower()] = w.replace("%","*")
 
 		if len(exact_match)>0:
 			exact_ = what+"=\""
@@ -55,10 +58,10 @@ class Hola:
 
 	def querier( self , what=False , elems = [] , time = [] , merge=False):
 		
-		table = "tweet_"+what
+		table = what #"tweet_"+what
 		query = "SELECT * FROM "+table
 
-		conditions , partial_match_d = self.make_conditions( what , elems , merge)
+		conditions , partial_match_d = self.make_conditions( "text" , elems , merge)
 		query = query+conditions
 		print(query)
 
@@ -70,6 +73,17 @@ class Hola:
 		rows = cursor.fetchall()
 		P = 0
 		T = {}
+
+
+
+		# RH = {}
+		# for r in rows:
+		# 	# print( dict(r) )
+		
+
+
+
+
 		for r in rows:
 			# print( dict(r) )
 			ID = str(r["id"]) 
@@ -79,14 +93,19 @@ class Hola:
 					"hashtags": {},
 					"date": ""
 				}
-			T[ID]["c"] += 1
-			lower_hash = r["hashtag"].lower()
-			if merge:
-				for w in partial_match_d:
-					if w in lower_hash:
-						lower_hash = partial_match_d[w]
-			T[ID]["hashtags"][ lower_hash ] = True
-			T[ID]["date"] = r["day"]
+
+
+			# lower_hash = r["hashtag"].lower()
+			# if merge:
+			# 	for w in partial_match_d:
+			# 		if w in lower_hash:
+			# 			lower_hash = partial_match_d[w]
+
+			for q in elems:
+				if q.lower() in r["text"].lower():
+					T[ID]["c"] += 1
+					T[ID]["hashtags"][ q.lower() ] = True
+					T[ID]["date"] = r["created_at"]
 
 		conn.close()
 		# print( query )
@@ -135,10 +154,34 @@ class Hola:
 		T = RES["tweets"]
 		T_meta = self.get_metadata( data=list(T.keys()) )
 		Priority = RES["tweets_priority"]
+		# print("\n")
+		# print(Priority)
+		# print("\n")
 		data = []
 
+
+
+		Ngrams = {}
+		Days_x_Tweets = {}
+		for ID in T:
+			tweet = T[ID]
+			dateiso = dateutil.parser.parse( tweet["date"] )
+			# jour = dateiso.strftime('%Y-%m-%dT%H:%M:%S')
+			jour = dateiso.strftime('%Y-%m-%d')
+			if jour not in Days_x_Tweets:
+				Days_x_Tweets[jour] = {
+					"tweets":[]
+				}
+			Days_x_Tweets[jour]["tweets"].append( ID )
+			
+			for h in tweet["hashtags"] :
+				Ngrams[h] = True
+		Ngrams = Ngrams.keys()
+
+
 		for i in sorted(Priority.keys() , reverse=True):
-			if i>1:
+			# print(i)
+			if i>1 and len(Ngrams)>1:
 				print(i ,":",len(Priority[i]),"tweets" )
 				for t in Priority[i]:
 					info = {}
@@ -156,25 +199,28 @@ class Hola:
 					info["FV"] = T_meta[t]["fv"]
 					info["RT"] = T_meta[t]["rt"]
 					data.append(info)
-
-		Ngrams = {}
-		Days_x_Tweets = {}
-		for ID in T:
-			tweet = T[ID]
-			dateiso = dateutil.parser.parse( tweet["date"] )
-			# jour = dateiso.strftime('%Y-%m-%dT%H:%M:%S')
-			jour = dateiso.strftime('%Y-%m-%d')
-			if jour not in Days_x_Tweets:
-				Days_x_Tweets[jour] = {
-					"tweets":[]
-				}
-			Days_x_Tweets[jour]["tweets"].append( ID )
-			
-			for h in tweet["hashtags"] :
-				Ngrams[h] = True
+			if len(Ngrams)==1:
+				for t in Priority[i]:
+					# print( t )
+					# print( "  FV:",T_meta[t]["fv"] ," | ", "RT:",T_meta[t]["rt"] )
+					# print("")
+					info = {}
+					info["id"] = t
+					info["created_at"] = T_meta[t]["created_at"]
+					info["text"] = T_meta[t]["text"]
+					info["score"] = i
+					info["author"] = {
+						"screen_name":T_meta[t]["user"],
+						"lang": "es",
+						"name": T_meta[t]["text"],
+						"profile_image_url": "http://img.com"
+					}
+					info["query"] = sorted(list(T[t]["hashtags"].keys()))
+					info["FV"] = T_meta[t]["fv"]
+					info["RT"] = T_meta[t]["rt"]
+					data.append(info)
 
 
-		Ngrams = Ngrams.keys()
 
 		DatesIndex = sorted(Days_x_Tweets.keys())
 		for jour in DatesIndex:
@@ -209,6 +255,10 @@ class Hola:
 				tweet.append(score)
 			finaldata.append(tweet)
 
+		# print("\n\ndataaaaa:")
+		# print(data)
+		# print(Ngrams)
+		# print("\n")
 		FinalArray = {
 			"data":finaldata,
 			"labels":Ngrams,
@@ -260,7 +310,8 @@ class Hola:
 						Term_User[t][u] = 0
 					Term_User[t][u] += 1
 
-		del User_Term["SectorSalud"]
+		if "SectorSalud" in User_Term:
+			del User_Term["SectorSalud"]
 		for t in Term_User:
 			if "SectorSalud" in Term_User[t]:
 				del Term_User[t]["SectorSalud"]

bogota.sqlite

-/var/www/DBs/SectorSalud/bogota.sqlite
+/home/ubuntu/bogota/bogotaa.sqlite
\ No newline at end of file

colomb.py

@@ -59,7 +59,7 @@ def dygraph_julian_json():
 
     from Twitter.julian import Hola
     inst = Hola()
-    RES = inst.querier( "hashtag" , original_query ,  ["2013-03-12","2016-03-12"] , merge)
+    RES = inst.querier( "tweet" , original_query ,  ["2013-03-12","2017-07-12"] , merge)
     data_final = inst.formatData( RES )
     return json.dumps( data_final )
 
@@ -249,12 +249,12 @@ def ajax_response(status, msg):
 
 #app.wsgi_app = ProxyFix(app.wsgi_app)
 
-#if __name__ == '__main__':
-#    #app.run(host='0.0.0.0')
-#    from flask_reverse_proxy import FlaskReverseProxied
-#    proxied = FlaskReverseProxied()
-#    proxied.init_app(app)
-#    app.run()
+if __name__ == '__main__':
+     #app.run(host='0.0.0.0')
+    from flask_reverse_proxy import FlaskReverseProxied
+    proxied = FlaskReverseProxied()
+    proxied.init_app(app)
+    app.run()
 
-from reverseproxy import ReverseProxied
-app.wsgi_app = ReverseProxied(app.wsgi_app)
+#from reverseproxy import ReverseProxied
+#app.wsgi_app = ReverseProxied(app.wsgi_app)
\ No newline at end of file

templates/index.html

@@ -35,50 +35,30 @@
 <ul>
 <li>
 	<a href='dygraph_julian?motscles=["hospital"]' target=_blank>["hospital"]</a>
-	<it> : query simple, busqueda exacta</it>
+	<it> : simple query</it>
 </li>
 <li>
         <a href='dygraph_julian?motscles=["hospital","vacuna"]' target=_blank>["hospital","vacuna"]</a>
-        <it> : query multiple, busqueda exacta</it>
+        <it> : multiple query</it>
 </li>
 
 
 <li>
-	<a href='dygraph_julian?motscles=["*hospital"]' target=_blank>["*hospital"]</a>
-	<it> : query expandida, busqueda con asterisco hacia la derecha de la palabra</it>
-</li>
-
-<li>
-        <a href='dygraph_julian?motscles=["hospital*"]' target=_blank>["hospital*"]</a>
-	 <it> : query expandida, busqueda con asterisco hacia la izquierda de la palabra</it>
-</li>
-
-<li>
-        <a href='dygraph_julian?motscles=["*hospital*"]' target=_blank>["*hospital*"]</a>
-         <it> : query expandida, busqueda con asterisco en ambos extremos</it>
-</li>
-
-<li>
-        <a href='dygraph_julian?motscles=["*hospital*"]&merge=["yes"]' target=_blank>["*hospital*"]&merge=["yes"]</a>
-         <it> : query expandida, asterisco en ambos extremos pero ahora con MERGE <b>agrupas</b> el resultado en solo una curva.</it>
-</li>
-
-<li>
-        <a href='dygraph_julian?motscles=["*bola*","*zika*","*chiku*"]&merge=["yes"]' target=_blank>["*bola*","*zika*","*chiku*"]&merge=["yes"]</a>
-         <it> : query expandida multiple, asterisco en ambos extremos. Una curva por hashtag con asterisco</it>
+        <a href='dygraph_julian?motscles=["ebola","zika","chiku"]' target=_blank>["ebola","zika","chiku"]</a>
+         <it> : multiple query </it>
 </li>
 
 <li>
 .
 </li>
 
-<li>
+<!-- <li>
         <a href='dygraph_julian?motscles=["*"]' target=_blank>Socio-semantic of Source</a>
          <it> : See ego-centered network of SectorSalud</it>
-</li>
+ </li>-->
 
 <li>
-        <a href='dygraph_julian?motscles=["*"]' target=_blank>Socio-semantic of Target</a>
+        <a href='http://134.158.74.254/phylum/explorerjs.html?file=data/bogota/users_hashtags.json' target=_blank>Socio-semantic of Target</a>
          <it> : See network related to how the people receives the difussion of SectorSalud</it>
 </li>