[UPDATE] explorer view, good version, do not merge yet

parsing/corpustools.py

@@ -33,6 +33,7 @@ class Parsers(defaultdict):
 
     _parsers = {
         'pubmed'            : PubmedFileParser,
+        'istex'             : ISText,
         'isi'               : IsiFileParser,
         'ris'               : RisFileParser,
         'europress'         : EuropressFileParser,

scrap_pubmed/views.py

@@ -187,7 +187,31 @@ def testISTEX(request , project_id):
 	print(request.method)
 	alist = ["bar","foo"]
 
+	# SQLAlchemy session
+	session = Session()
 
+	# do we have a valid project id?
+	try:
+		project_id = int(project_id)
+	except ValueError:
+		raise Http404()
+
+	# do we have a valid project?
+	project = (session
+		.query(Node)
+		.filter(Node.id == project_id)
+		.filter(Node.type_id == cache.NodeType['Project'].id)
+	).first()
+
+	if project is None:
+		raise Http404()
+
+	# do we have a valid user?
+	user = request.user
+	if not user.is_authenticated():
+		return redirect('/login/?next=%s' % request.path)
+	if project.user_id != user.id:
+		return HttpResponseForbidden()
 
 	if request.method == "POST":
 		# print(alist)
@@ -196,63 +220,73 @@ def testISTEX(request , project_id):
 		N = 60
 		if "query" in request.POST: query = request.POST["query"]
 		if "string" in request.POST: query_string = request.POST["string"].replace(" ","+")
-		# if "N" in request.POST: N = request.POST["N"]
+		if "N" in request.POST: N = int(request.POST["N"])
 		print(query_string , query , N)
 
 
-		# urlreqs = []
-		# pagesize = 50
-		# tasks = MedlineFetcher()
-		# chunks = list(tasks.chunks(range(N), pagesize))
-		# for k in chunks:
-		# 	if (k[0]+pagesize)>N: pagesize = N-k[0]
-		# 	urlreqs.append("http://api.istex.fr/document/?q="+query_string+"&output=*&"+"from="+str(k[0])+"&size="+str(pagesize))
-		# print(urlreqs)
+		urlreqs = []
+		pagesize = 500
+		tasks = MedlineFetcher()
+		chunks = list(tasks.chunks(range(N), pagesize))
+		for k in chunks:
+			if (k[0]+pagesize)>N: pagesize = N-k[0]
+			urlreqs.append("http://api.istex.fr/document/?q="+query_string+"&output=*&"+"from="+str(k[0])+"&size="+str(pagesize))
+		print(urlreqs)
 
 		# urlreqs = ["http://localhost/374255" , "http://localhost/374278" ]
 		# print(urlreqs)
 
-		# resource_type = ResourceType.objects.get(name="istext" )
-
-		# parent      = Node.objects.get(id=project_id)
-		# node_type   = NodeType.objects.get(name='Corpus')
-		# type_id = NodeType.objects.get(name='Document').id
-		# user_id = User.objects.get( username=request.user ).id
-
-		# corpus = Node(
-		# 	user=request.user,
-		# 	parent=parent,
-		# 	type=node_type,
-		# 	name=query,
-		# )
-
-		# corpus.save()
-
-		# # configuring your queue with the event
-		# for i in range(8):
-		# 	t = threading.Thread(target=tasks.worker2) #thing to do
-		# 	t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
-		# 	t.start()
-		# for url in urlreqs:
-		# 	filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.now().microsecond))
-		# 	tasks.q.put( [url , filename]) #put a task in th queue
-		# tasks.q.join() # wait until everything is finished
-		# for filename in tasks.firstResults:
-		# 	corpus.add_resource( user=request.user, type=resource_type, file=filename )
-
-
-		# corpus.save()
-		# print("DEBUG:",DEBUG)
-		# # do the WorkFlow
-		# try:
-		# 	if DEBUG is True:
-		# 		corpus.workflow()
-		# 	else:
-		# 		corpus.workflow.apply_async((), countdown=3)
-
-		# 	return JsonHttpResponse(["workflow","finished"])
-		# except Exception as error:
-		# 	print(error)
+		resourcetype = cache.ResourceType["istex"]
+		print(resourcetype)
+		# corpus node instanciation as a Django model
+		corpus = Node(
+			name = query,
+			user_id = request.user.id,
+			parent_id = project_id,
+			type_id = cache.NodeType['Corpus'].id,
+			language_id = None,
+		)
+		session.add(corpus)
+		session.commit()
+
+		tasks = MedlineFetcher()
+		for i in range(8):
+			t = threading.Thread(target=tasks.worker2) #thing to do
+			t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
+			t.start()
+		for url in urlreqs:
+			filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.datetime.now().isoformat()))
+			tasks.q.put( [url , filename]) #put a task in th queue
+		tasks.q.join() # wait until everything is finished
+
+		dwnldsOK = 0
+		for filename in tasks.firstResults:
+			if filename!=False:
+				print(filename)
+				# add the uploaded resource to the corpus
+				add_resource(corpus,
+					user_id = request.user.id,
+					type_id = resourcetype.id,
+					file = filename,
+				)
+				dwnldsOK+=1
+
+		if dwnldsOK == 0: return JsonHttpResponse(["fail"])
+
+		try:
+			def apply_workflow(corpus):
+				parse_resources(corpus)
+				extract_ngrams(corpus, ['title'])
+				compute_tfidf(corpus)
+			if DEBUG:
+				apply_workflow(corpus)
+			else:
+				thread = threading.Thread(target=apply_workflow, args=(corpus, ), daemon=True)
+				thread.start()
+		except Exception as error:
+			print('WORKFLOW ERROR')
+			print(error)
+		return HttpResponseRedirect('/project/' + str(project_id))
 
 	data = [query_string,query,N]
 	return JsonHttpResponse(data)

templates/project.html

@@ -276,10 +276,15 @@
 		function bringDaNoise() {
 			var theresults = $("#theresults").html()
 			if( theresults && theresults.search("No results")==-1 ) {
+				var origQuery = $("#id_name").val()
 				console.log("we've in dynamic mode")
 				$("#simpleloader").html('<img width="30px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img>')
 				$("#submit_thing").prop('onclick',null);
-				doTheQuery();
+				var theType = $("#id_type option:selected").html();
+				if(theType=="pubmed") doTheQuery();
+				if(theType=="istex") {
+					testISTEX(origQuery.replace(" ","+"),1000)
+				}
 			}
 			else {
 				console.log("we dont have nothing inside results div")
@@ -301,6 +306,8 @@
 
 		function getGlobalResults(value){
 			console.log("in getGlobalResults()")
+			console.log("value:")
+			console.log(value)
 			// AJAX to django
 			var pubmedquery = $("#id_name").val()
 			var Npubs = $("#id_N").val();
@@ -311,6 +318,8 @@
 			$("#"+value.id).prop('onclick',null);
 
 			var theType = $("#id_type option:selected").html();
+			console.log("theType:")
+			console.log(theType)
 
 			if(theType=="pubmed") {
 			    $.ajax({
@@ -348,7 +357,7 @@
 			    });	
 			}
 
-			if(theType=="istext") {
+			if(theType=="istex") {
 				console.log(window.location.origin+"tests/istextquery")
 			    $.ajax({
 				  // contentType: "application/json",
@@ -424,7 +433,7 @@
 		//CSS events for changing the Select element
 		function CustomForSelect( selected ) {
 			// show Radio-Inputs and trigger FileOrNotFile>@upload-file events
-			if(selected=="pubmed" || selected=="istext") {
+			if(selected=="pubmed" || selected=="istex") {
 			// if(selected=="pubmed") {
 				console.log("show the button for: "+selected)
 				$("#pubmedcrawl").css("visibility", "visible"); 
@@ -475,7 +484,7 @@
 			var origQuery = query
 
 
-			var pubmedifiedQuery = { query : query , string: query }
+			var pubmedifiedQuery = { query : query , string: query , N:Npubs}
 			// console.log(pubmedifiedQuery)
 
 			var projectid = window.location.href.split("project")[1].replace(/\//g, '')//replace all the slashes
@@ -491,7 +500,7 @@
 		      success: function(data) {
 				console.log("ajax_success: in testISTEX()")
 		        console.log(data)
-		        // location.reload();
+		        location.reload();
 		      },
 		        error: function(result) {
 		            console.log("in testISTEX(). Data not found");