[FEATURE] dynamic query for pubmed: OK

analysis/functions.py

@@ -269,7 +269,6 @@ from analysis.tfidf import tfidf
 
 def do_tfidf(corpus, reset=True):
     print("doing tfidf")
-    print("\t",corpus.type)
     with transaction.atomic():
         if reset==True:
             NodeNodeNgram.objects.filter(nodex=corpus).delete()

gargantext_web/urls.py

@@ -67,6 +67,7 @@ urlpatterns = patterns('',
     url(r'^nodeinfo/(\d+)$', views.nodeinfo),
     url(r'^tests/mvc$', views.tests_mvc),
     url(r'^tests/mvc-listdocuments$', views.tests_mvc_listdocuments),
+
     url(r'^tests/pubmedquery$', pubmedscrapper.getGlobalStats),
     url(r'^tests/project/(\d+)/pubmedquery/go$', pubmedscrapper.doTheQuery)
 

gargantext_web/views.py

@@ -212,6 +212,8 @@ def project(request, project_id):
     cooclists       = ""#.children.filter(type=type_cooclist)
     
     for corpus in corpora:
+        # print("corpus", corpus.pk , corpus.name , corpus.type_id)
+
         docs_count =  corpus.children.count()
         docs_total += docs_count
         
@@ -219,10 +221,17 @@ def project(request, project_id):
         corpus_view['id']         = corpus.pk
         corpus_view['name']       = corpus.name
         corpus_view['count']      = corpus.children.count()
-        
-        for node_resource in Node_Resource.objects.filter(node=corpus):
-            donut_part[node_resource.resource.type] += docs_count
-            list_corpora[node_resource.resource.type.name].append(corpus_view)
+
+        #just get first element of the corpora and get his type.
+        corpus_type = Node_Resource.objects.filter(node=corpus)[0].resource.type
+        list_corpora[corpus_type].append(corpus_view)
+
+        ## For avoiding to list repeated elements, like when u use the dynamic query (per each xml, 1)
+        # for node_resource in Node_Resource.objects.filter(node=corpus):
+        #     print( "node_resource.id:",node_resource.id , node_resource.resource.file )
+        #     donut_part[node_resource.resource.type] += docs_count
+        #     list_corpora[node_resource.resource.type.name].append(corpus_view)
+            # print(node_resource.resource.type.name)
     list_corpora = dict(list_corpora)
 
     if docs_total == 0 or docs_total is None:
@@ -235,8 +244,6 @@ def project(request, project_id):
 
 
     if request.method == 'POST':
-        print("original file:")
-        print(request.FILES)
 
         form = CustomForm(request.POST, request.FILES)
         if form.is_valid():
@@ -249,9 +256,6 @@ def project(request, project_id):
             print("-------------")
             print(name,"|",resource_type,"|",thefile)
             print("-------------")
-            
-            print("new file:")
-            print(thefile)
 
             try:
                 parent      = Node.objects.get(id=project_id)
@@ -280,8 +284,6 @@ def project(request, project_id):
 
                 corpus.save()
 
-                print(request.user, resource_type , thefile )
-
                 corpus.add_resource(
                         user=request.user,
                         type=resource_type,
@@ -324,80 +326,6 @@ def project(request, project_id):
                 })
     else:
         form = CustomForm()
-
-
-    # if request.method == 'POST':
-    #     #form = CorpusForm(request.POST, request.FILES)
-    #     #print(str(request.POST))
-    #     name        = str(request.POST['name'])
-    #     try:
-    #         resource_type = ResourceType.objects.get(id=str(request.POST['type']))
-    #     except Exception as error:
-    #         print(error)
-    #         resource_type = None
-        
-    #     try:
-    #         file = request.FILES['file']
-    #     except Exception as error:
-    #         print(error)
-    #         file = None
-
-    #     #if name != "" and resource_type is not None and file is not None:
-    #     try:
-    #         parent      = Node.objects.get(id=project_id)
-    #         node_type   = NodeType.objects.get(name='Corpus')
-            
-    #         if resource_type.name == "europress_french":
-    #             language    = Language.objects.get(iso2='fr')
-    #         elif resource_type.name == "europress_english":
-    #             language    = Language.objects.get(iso2='en')
-            
-    #         try:
-    #             corpus = Node(
-    #                     user=request.user,
-    #                     parent=parent,
-    #                     type=node_type,
-    #                     language=language,
-    #                     name=name,
-    #                     )
-    #         except:
-    #             corpus = Node(
-    #                     user=request.user,
-    #                     parent=parent,
-    #                     type=node_type,
-    #                     name=name,
-    #                     )
-
-    #         corpus.save()
-
-    #         print(request.user, resource_type , file )
-    #         print(corpus.language)
-    #         corpus.add_resource(
-    #                 user=request.user,
-    #                 type=resource_type,
-    #                 file=file
-    #                 )
-
-    #         try:
-    #             #corpus.parse_and_extract_ngrams()
-    #             #corpus.parse_and_extract_ngrams.apply_async((), countdown=3)
-    #             if DEBUG is True:
-    #                 corpus.workflow()
-    #             else:
-    #                 corpus.workflow.apply_async((), countdown=3)
-
-    #         except Exception as error:
-    #             print(error)
-
-    #         return HttpResponseRedirect('/project/' + str(project_id))
-    #     except Exception as error:
-    #         print('ee', error)
-    #         form = CorpusForm(request=request)
-    #         formResource = ResourceForm()
-
-    # else:
-    #     form = CorpusForm(request=request)
-    #     formResource = ResourceForm()
        
     return render(request, 'project.html', {
             'form'          : form,

node/models.py

@@ -236,12 +236,17 @@ class Node(CTENode):
 
     @current_app.task(filter=task_method)
     def workflow(self, keys=None, ngramsextractorscache=None, ngramscaches=None, verbose=False):
-        print("In workflow() START")
+        print("In workflow()    parse_resources()")
         self.parse_resources()
+        print("In workflow()    / parse_resources()")
+        print("In workflow()    extract_ngrams()")
         type_document   = NodeType.objects.get(name='Document')
         self.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
+        print("In workflow()    / extract_ngrams()")
+        print("In workflow()    do_tfidf()")
         from analysis.functions import do_tfidf
         do_tfidf(self)
+        print("In workflow()    / do_tfidf()")
         print("In workflow() END")
 
 class Node_Metadata(models.Model):

scrap_pubmed/MedlineFetcherDavid2015.py

@@ -56,6 +56,7 @@ class MedlineFetcher:
         # webEnv = doc.xpathEval('eSearchResult/WebEnv/text()')[0]
         # print count, queryKey, webEnv
         values = { "count": int(str(count)), "queryKey": queryKey , "webEnv":webEnv }
+        print(values)
         return values
 
 
@@ -126,28 +127,27 @@ class MedlineFetcher:
             # medlineEfetchRAW(str(year) + '[dp] '+query , retmax=300)
             pubmedquery = str(year) + '[dp] '+query
             globalresults = self.medlineEsearch(pubmedquery)
-            N+=globalresults["count"]
-            querymetadata = { 
-                "string": pubmedquery , 
-                "count": globalresults["count"] , 
-                "queryKey":globalresults["queryKey"] , 
-                "webEnv":globalresults["webEnv"] , 
-                "retmax":0 
-            }
-            thequeries.append ( querymetadata )
+            if globalresults["count"]>0:
+                N+=globalresults["count"]
+                querymetadata = { 
+                    "string": pubmedquery , 
+                    "count": globalresults["count"] , 
+                    "queryKey":globalresults["queryKey"] , 
+                    "webEnv":globalresults["webEnv"] , 
+                    "retmax":0 
+                }
+                thequeries.append ( querymetadata )
 
         print("Total Number:", N,"publications")
         print("And i want just:",globalLimit,"publications")
         print("---------------------------------------\n")
 
-        for query in thequeries:
+        for i,query in enumerate(thequeries):
             k = query["count"]
             percentage = k/float(N)
             retmax_forthisyear = int(round(globalLimit*percentage))
             query["retmax"] = retmax_forthisyear
-            # self.medlineEfetchRAW( query )
 
-        print ('Done !')
         return thequeries
 
 

scrap_pubmed/views.py

@@ -12,6 +12,11 @@ from gargantext_web.api import JsonHttpResponse
 from urllib.request import urlopen, urlretrieve
 import json
 
+from gargantext_web.settings import MEDIA_ROOT
+from datetime import datetime
+from django.core.files import File
+from gargantext_web.settings import DEBUG
+
 from node.models import Language, ResourceType, Resource, \
         Node, NodeType, Node_Resource, Project, Corpus, \
         Ngram, Node_Ngram, NodeNgramNgram, NodeNodeNgram
@@ -24,7 +29,7 @@ def getGlobalStats(request ):
 	if request.method == "POST":
 		query = request.POST["query"]
 		instancia = MedlineFetcher()
-		alist = instancia.serialFetcher( 5, query , 200 )
+		alist = instancia.serialFetcher( 5, query , 100 )
 
 	data = alist
 	return JsonHttpResponse(data)
@@ -43,8 +48,6 @@ def doTheQuery(request , project_id):
 		instancia = MedlineFetcher()
 		thequeries = json.loads(query)
 
-		print("------------------")
-
 		urlreqs = []
 		for yearquery in thequeries:
 			urlreqs.append( instancia.medlineEfetchRAW( yearquery ) )
@@ -58,71 +61,49 @@ def doTheQuery(request , project_id):
 		"""
 
 		thefile = "how we do this here?"
-		resource_type = ResourceType()
-		resource_type.name = name
-
-		try:
-			parent      = Node.objects.get(id=project_id)
-			node_type   = NodeType.objects.get(name='Corpus')
-			type_id = NodeType.objects.get(name='Document').id
-			user_id = User.objects.get( username=request.user ).id
-
-			corpus = Node(
-				user=request.user,
-				parent=parent,
-				type=node_type,
-				name=name,
-			)
-
-			corpus.save()
-			
-			parser = PubmedFileParser()
-			metadata_list = []
-			for url in urlreqs:
-				data = urlopen(url)
-				metadata_list += parser.parse( data.read() )
-				# corpus.add_resource( user=request.user, type=resource_type, file=data.read() )
-				break
+		resource_type = ResourceType.objects.get(name="pubmed" )
 
+		parent      = Node.objects.get(id=project_id)
+		node_type   = NodeType.objects.get(name='Corpus')
+		type_id = NodeType.objects.get(name='Document').id
+		user_id = User.objects.get( username=request.user ).id
 
-			from parsing.Caches import LanguagesCache
-			langages_cache = LanguagesCache()
-			for i, metadata_values in enumerate(metadata_list):
-				name = metadata_values.get('title', '')[:200]
-				language = langages_cache[metadata_values['language_iso2']] if 'language_iso2' in metadata_values else None,
-				if isinstance(language, tuple):
-					language = language[0]
+		corpus = Node(
+			user=request.user,
+			parent=parent,
+			type=node_type,
+			name=name,
+		)
 
-				Node(
-					user_id  = user_id,
-					type_id  = type_id,
-					name     = name,
-					parent   = parent,
-					language_id = language.id if language else None,
-					metadata = metadata_values
-				).save()
+		corpus.save()
 
-			parent.children.all().make_metadata_filterable()
-
-			type_document   = NodeType.objects.get(name='Document')
-			print("printing here 01")
-			parent.children.filter(type_id=type_document.pk).extract_ngrams(keys=['title',])
-			print("printing here 02")
+		try:
+			for url in urlreqs:
+				print(url)
+				data = urlopen(url)
+				xmlname = MEDIA_ROOT + '/corpora/%s/%s.xml' % (request.user, str(datetime.now().microsecond))
+				f = open(xmlname, 'w')
+				myfile = File(f)
+				myfile.write( data.read().decode('utf-8') )
+				myfile.close()
+				f.close()
+				corpus.add_resource( user=request.user, type=resource_type, file=xmlname )
 
-			print("now we've to apply do_tfidf...")
+			try:
+				if DEBUG is True:
+					corpus.workflow()
+				else:
+					corpus.workflow.apply_async((), countdown=3)
 
+				return JsonHttpResponse(["workflow","finished"])
 
-			# thetitles = parent.children.filter(type_id=type_document.pk)
-			# print(Node.objects.filter(parent=parent))
-			# from analysis.functions import do_tfidf
-			# do_tfidf(corpus)
+			except Exception as error:
+				print(error)
 
-			print("ca va?")
+			return JsonHttpResponse(["workflow","finished","outside the try-except"])
 
 		except Exception as error:
 			print("lele",error)
 
-
-
 	data = alist
 	return JsonHttpResponse(data)
\ No newline at end of file

templates/project.html

@@ -213,6 +213,7 @@
 		      success: function(data) {
 				console.log("in doTheQuery()")
 		        console.log(data)
+		        location.reload();
 		      },
 		        error: function(result) {
 		            console.log("in doTheQuery(). Data not found");
@@ -241,11 +242,14 @@
 
 	            thequeries = data
 	            var N=0,k=0;
-	            for(var i in thequeries) N += thequeries[i].count
 
-	            if(N>0) {
+	            for(var i in thequeries) N += thequeries[i].count
+	            if( N>0) {
 	            	$("#results").html("Result: "+N+" publications in the last 5 years")
 	            	$('#id_thebutton').prop('disabled', false);
+	            } else {
+	            	$("#results").html("No results!.")
+	            	$('#id_thebutton').prop('disabled', true);
 	            }
 
 		      },