Merge branch 'samuel' into merge

annotations/urls.py

@@ -6,4 +6,5 @@ urlpatterns = patterns('',
     url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view
     url(r'^corpus/(?P<corpus_id>[0-9]+)/document/(?P<doc_id>[0-9]+)$', views.NgramList.as_view()), # the list associated with an ngram
     url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.NgramEdit.as_view()), #
+    url(r'^lists/(?P<list_id>[0-9]+)/multiple?$', views.deleteMultiple ), #
 )

annotations/views.py

@@ -16,6 +16,8 @@ from rest_framework.authentication import SessionAuthentication, BasicAuthentica
 from node.models import Node
 from gargantext_web.db import *
 from ngram.lists import listIds, listNgramIds, ngramList
+from gargantext_web.api import JsonHttpResponse
+import json
 
 
 @login_required
@@ -109,6 +111,26 @@ class NgramEdit(APIView):
         session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).delete()
         return Response(None, 204)
 
+def deleteMultiple(request, list_id):
+    results = ["hola","mundo"]
+
+    user = request.user
+    if not user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+
+    if request.POST:
+        todel_ids = json.loads(request.POST['to_delete'])
+        for ngram_id in todel_ids:
+            # add the ngram to the list if not already done
+            node_ngram = session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).first()
+            if node_ngram is None:
+                node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
+                session.add(node_ngram)
+                session.commit()
+
+    return JsonHttpResponse(results)
+
+
 
 class Document(APIView):
     """

gargantext_web/urls.py

@@ -85,7 +85,10 @@ urlpatterns = patterns('',
     url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
     url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
     url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', samtest.test_ngrams),
-    url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams)
+    url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams),
+    url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', samtest.test_journals),
+    url(r'^project/(\d+)/corpus/(\d+)/journals$', samtest.get_journals),
+    url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
 )
 
 

node/admin.py

@@ -98,8 +98,8 @@ from django import forms
 
 from django.utils.translation import ugettext_lazy as _
 class CustomForm(forms.Form):
-    name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
     type = ModelChoiceField( ResourceType.objects.all() , widget=forms.Select(attrs={'onchange':'CustomForSelect( $("option:selected", this).text() );'}) )
+    name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
     file = forms.FileField()
 
     # Description: clean_file()

scrappers/scrap_pubmed/views.py

@@ -269,11 +269,10 @@ def testISTEX(request , project_id):
 					type_id = resourcetype.id,
 					file = filename,
 				)
-				dwnldsOK+=1			
-
-		# print(urlreqs)
-
-
+				dwnldsOK+=1	
+		if dwnldsOK == 0: return JsonHttpResponse(["fail"])		
+		###########################
+		###########################
 		try:
 			if not DEBUG:
 				apply_workflow.apply_async((corpus.id,),)
@@ -287,50 +286,6 @@ def testISTEX(request , project_id):
 		return HttpResponseRedirect('/project/' + str(project_id))
 
 
-
-
-		# resource_type = ResourceType.objects.get(name="istext" )
-
-		# parent      = Node.objects.get(id=project_id)
-		# node_type   = NodeType.objects.get(name='Corpus')
-		# type_id = NodeType.objects.get(name='Document').id
-		# user_id = User.objects.get( username=request.user ).id
-
-		# corpus = Node(
-		# 	user=request.user,
-		# 	parent=parent,
-		# 	type=node_type,
-		# 	name=query,
-		# )
-
-		# corpus.save()
-
-		# # configuring your queue with the event
-		# for i in range(8):
-		# 	t = threading.Thread(target=tasks.worker2) #thing to do
-		# 	t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
-		# 	t.start()
-		# for url in urlreqs:
-		# 	filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.now().microsecond))
-		# 	tasks.q.put( [url , filename]) #put a task in th queue
-		# tasks.q.join() # wait until everything is finished
-		# for filename in tasks.firstResults:
-		# 	corpus.add_resource( user=request.user, type=resource_type, file=filename )
-
-
-		# corpus.save()
-		# print("DEBUG:",DEBUG)
-		# # do the WorkFlow
-		# try:
-		# 	if DEBUG is True:
-		# 		corpus.workflow()
-		# 	else:
-		# 		corpus.workflow.apply_async((), countdown=3)
-
-		# 	return JsonHttpResponse(["workflow","finished"])
-		# except Exception as error:
-		# 	print(error)
-
 	data = [query_string,query,N]
 	return JsonHttpResponse(data)
 

static/js/dyna_chart_and_table.js → static/js/Docs_dyna_chart_and_table.js

@@ -119,11 +119,15 @@ function Final_UpdateTable( action ) {
 var current_docs = {}
 var BIS_dict = {}
 
-var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\//);
-var projectid = path[1]
-var corpusid  = path[2]
-
-var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
+var url_elems = window.location.href.split("/")
+var url_mainIDs = {}
+for(var i=0; i<url_elems.length; i++) {
+  // if the this element is a number:
+  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
+    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
+  }
+}
+var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
 // $.ajax({
 //   url: theurl,
 //   success: function(data) {
@@ -204,6 +208,38 @@ function transformContent(rec_id , header , content) {
   } else return content;
 }
 
+
+$("#move2trash")
+.click(function(){
+
+    var ids2trash = []
+    for(var i in Garbage) {
+      ids2trash.push(AjaxRecords[i].id);
+    }
+
+    console.log("ids to the trash:")
+    console.log(ids2trash)
+
+    $.ajax({
+      url: "/tests/move2trash/",
+      data: "nodeids="+JSON.stringify(ids2trash),
+      type: 'POST',
+      beforeSend: function(xhr) {
+        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+      },
+      success: function(data) {
+        console.log("in #move2trash")
+        console.log(data)
+        location.reload();
+      },
+        error: function(result) {
+            console.log("Data not found in #move2trash");
+            console.log(result)
+        }
+    });
+})
+.hide();
+
 //generic enough
 function ulWriter(rowIndex, record, columns, cellWriter) {
   // pr("\tulWriter: "+record.id)
@@ -224,9 +260,30 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
   return '<tr>' + tr + '</tr>';
 }
 
+
+
+// var div__filter_for_search = ''
+//   div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple  data-max-options="1">';
+//   div__filter_for_search += '  <optgroup label="All" data-max-options="1" selected>';
+//   div__filter_for_search += '    <option>Title</option>';
+//   div__filter_for_search += '    <option>Date</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '  <optgroup label="Category" data-max-options="1">';
+//   div__filter_for_search += '    <option>Title</option>';
+//   div__filter_for_search += '    <option>Date</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '  <optgroup label="Duplicates" data-max-options="1">';
+//   div__filter_for_search += '    <option>by DOI</option>';
+//   div__filter_for_search += '    <option>by Title</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '</select>';
+
+// $("#supmofos").html(div__filter_for_search)
+
+
 // (3) Get records and hyperdata for paginator
   $.ajax({
-    url: '/tests/paginator/corpus/'+corpusid,
+    url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
     success: function(data){
       console.log(data)
 
@@ -235,7 +292,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
         var orig_id = parseInt(data.records[i].id)
         var arr_id = parseInt(i)
         RecDict[orig_id] = arr_id;
-        data.records[i]["name"] = '<a target="_blank" href="/project/'+projectid+'/corpus/'+ corpusid + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
+        data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
         data.records[i]["del"] = false
 
         var date = data.records[i]["date"];  
@@ -247,36 +304,6 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
 
       // $("#move2trash").prop('disabled', true);
 
-      $("#move2trash")
-      .click(function(){
-
-          var ids2trash = []
-          for(var i in Garbage) {
-            ids2trash.push(AjaxRecords[i].id);
-          }
-
-          console.log("ids to the trash:")
-          console.log(ids2trash)
-
-          $.ajax({
-            url: "/tests/move2trash/",
-            data: "nodeids="+JSON.stringify(ids2trash),
-            type: 'POST',
-            beforeSend: function(xhr) {
-              xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-            },
-            success: function(data) {
-              console.log("in #move2trash")
-              console.log(data)
-              location.reload();
-            },
-              error: function(result) {
-                  console.log("Data not found in #move2trash");
-                  console.log(result)
-              }
-          });
-      })
-      .hide();
 
 
       var t0 = AjaxRecords[0].date.split("-").map(Number)
@@ -429,6 +456,13 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
       $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
       $(".dynatable-record-count").insertAfter(".imadiv")
       $(".dynatable-pagination-links").insertAfter(".imadiv")
-      // console.log(RecDict)
+      
+      var the_content = $("#supmofos").html();
+      $(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
+      $("#supmofos").remove()
+      // .insertAfter("#dynatable-query-search-my-ajax-table")
+
+
     }
   });
+

static/js/Journals_dyna_chart_and_table.js

+
+function pr(msg) {
+    console.log(msg)
+}
+
+function getCookie(name) {
+    var cookieValue = null;
+    if (document.cookie && document.cookie != '') {
+        var cookies = document.cookie.split(';');
+        for (var i = 0; i < cookies.length; i++) {
+            var cookie = jQuery.trim(cookies[i]);
+            // Does this cookie string begin with the name we want?
+            if (cookie.substring(0, name.length + 1) == (name + '=')) {
+                cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                break;
+            }
+        }
+    }
+    return cookieValue;
+}
+
+
+var latest,oldest;
+
+var TheBuffer = false
+
+var PossibleActions = []
+
+var action1 = {
+  "id":"to_delete",
+  "name": "Delete",
+  "color":"red"
+}
+var action2 = {
+  "id":"to_keep",
+  "name": "Keep",
+  "color":"green"
+}
+
+var action3 = {
+  "id":"to_group",
+  "name": "Group",
+  "color":"blue"
+}
+
+PossibleActions.push(action1)
+PossibleActions.push(action2)
+PossibleActions.push(action3)
+
+var FlagsBuffer = {}
+for(var i in PossibleActions) {
+  FlagsBuffer[PossibleActions[i].id] = {}
+}
+
+
+var MyTable;
+var RecDict={};
+var AjaxRecords = []
+
+//      D3.js: Interactive timerange variables.
+var LineChart = dc.lineChart("#monthly-move-chart"); 
+var volumeChart = dc.barChart("#monthly-volume-chart");
+
+
+
+function Push2Buffer( NewVal ) {
+    if ( TheBuffer == false) {
+        if( ! NewVal ) {
+            var limits = [ oldest , latest ];
+            NewVal = limits;
+        }
+        console.log( " - - - - - - " )
+        console.log( "\tchanging to:" )
+        console.log( NewVal )
+        TheBuffer = NewVal;
+        Final_UpdateTable( "changerange" )
+        console.log( "- - - - - - -\n" )
+        return 1;
+    }
+
+    if ( TheBuffer != false ) {
+        var past = TheBuffer[0]+"_"+TheBuffer[1]
+
+        if( ! NewVal ) {
+            var limits = [ oldest , latest ];
+            NewVal = limits;
+        }
+        var now = NewVal[0]+"_"+NewVal[1]
+        
+        if ( past != now ) {
+            console.log( " - - - - - - " )
+            console.log( "\tchanging to:" )
+            console.log( NewVal )
+            TheBuffer = NewVal;
+            Final_UpdateTable( "changerange" )
+            console.log( "- - - - - - -\n" )
+        }
+        return 1;
+    }
+}
+
+function Final_UpdateTable( action ) {
+    // (1) Identifying if the button is collapsed:   
+    var isCollapsed=false;
+    var accordiontext = $("#collapseOne").attr("class")
+    if(accordiontext.indexOf("collapse in")>-1) 
+        isCollapsed=true;
+
+
+    var UpdateTable = false
+    if ( (action == "click" && !isCollapsed) || (action=="changerange" && isCollapsed) ) {
+        UpdateTable = true;
+        $("#corpusdisplayer").html("Close Folder")
+    } else $("#corpusdisplayer").html("Open Folder")
+
+    pr("update table??: "+UpdateTable)
+
+    if ( ! UpdateTable ) return false; //stop whatever you wanted to do.
+
+
+
+    var TimeRange = AjaxRecords;
+
+    var dataini = (TheBuffer[0])?TheBuffer[0]:oldest;
+    var datafin = (TheBuffer[1])?TheBuffer[1]:latest;
+    pr("show me the pubs of the selected period")
+    pr("\tfrom ["+dataini+"] to ["+datafin+"]")
+    pr("\tfrom ["+oldest+"] to ["+latest+"]")
+
+    TimeRange = []
+    for (var i in AjaxRecords) {
+        if(AjaxRecords[i].score>=dataini && AjaxRecords[i].score<=datafin){
+            // pr( AjaxRecords[i].date+" : "+AjaxRecords[i].id )
+            TimeRange.push(AjaxRecords[i])
+        }
+    }
+    
+    MyTable = $('#my-ajax-table').dynatable({
+        dataset: {
+            records: TimeRange
+        },
+        features: {
+            pushState: false,
+            // sort: false
+        },
+        writers: {
+          _rowWriter: ulWriter
+          // _cellWriter: customCellWriter
+        }
+    });
+    MyTable.data('dynatable').settings.dataset.originalRecords = []
+    MyTable.data('dynatable').settings.dataset.originalRecords = TimeRange;
+    
+    MyTable.data('dynatable').paginationPage.set(1);
+    MyTable.data('dynatable').process();
+}
+
+//          STEP 01:
+//      Get all the duplicates using the Django-Garg API
+var current_docs = {}
+var BIS_dict = {}
+
+var url_elems = window.location.href.split("/")
+var url_mainIDs = {}
+for(var i=0; i<url_elems.length; i++) {
+  // if the this element is a number:
+  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
+    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
+  }
+}
+var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
+// $.ajax({
+//   url: theurl,
+//   success: function(data) {
+//     bisarray = data.data
+//     for(var i in bisarray) {
+//         untitlebis = bisarray[i].values
+//         BIS_dict[untitlebis[0]] = [bisarray[i].count , 0];// [ total amount , removed ]
+//     }
+//     pr(BIS_dict)
+//     if(Object.keys(BIS_dict).length>0) $("#delAll").css("visibility", "visible"); $("#delAll").show();
+//   }
+// });
+
+
+
+
+
+function getRecord(rec_id) {
+  return MyTable.data('dynatable').settings.dataset.originalRecords[rec_id];
+  // return AjaxRecords[rec_id]
+}
+
+function getRecords() {
+  return MyTable.data('dynatable').settings.dataset.originalRecords;
+}
+
+function transformContent2(rec_id) {
+  // pr("\t\ttransformContent2: "+rec_id)
+  var elem = AjaxRecords[rec_id];
+  var result = {}
+  // console.log("\t\t\telement flag : "+elem["flag"])
+  if (elem["flag"]) {
+    result["id"] = elem["id"]
+    result["score"] = '<div class="'+elem["flag"]+'"><i>'+elem["score"]+'</div>'
+    result["name"] = '<div class="'+elem["flag"]+'"><i>'+elem["name"]+'</div>'
+    result["flag"] = '<input id='+rec_id+' onclick="overRide(this)" type="checkbox" checked/>'
+  } else {
+    result["id"] = elem["id"]
+    result["score"] = elem["score"]
+    result["name"] = elem["name"]
+    result["flag"] = '<input id='+rec_id+' onclick="overRide(this)" type="checkbox"/>'
+  }
+
+  return result;
+}
+
+function overRide(elem) {
+  var id = elem.id
+  var current_flag = $("input[type='radio'][name='radios']:checked").val()
+  var this_newflag = (current_flag==AjaxRecords[id]["flag"])?false:current_flag
+
+  console.log("striking: "+id+" | this-elem_flag: "+AjaxRecords[id]["flag"]+" | current_flag: "+current_flag)
+  console.log("\t so the new flag is: "+this_newflag)
+  AjaxRecords[id]["flag"] = Mark_NGram ( id , AjaxRecords[id]["flag"] , this_newflag );
+
+  MyTable.data('dynatable').dom.update();
+
+}
+
+function transformContent(rec_id , header , content) {
+  if(header=="flag") {
+    // pr("\t\ttransformContent1: "+rec_id)
+    if(content==true) return '<input id='+rec_id+' onclick="overRide(this)" type="checkbox" checked/>'
+    if(content==false) return '<input id='+rec_id+' onclick="overRide(this)" type="checkbox"/>'
+  } else return content;
+}
+
+
+function Mark_NGram( ngram_id , old_flag , new_flag ) {
+  if(new_flag){
+    for(var f in FlagsBuffer) {
+      if( new_flag==f )
+        FlagsBuffer[f][ngram_id] = true;
+      else 
+        delete FlagsBuffer[f][ngram_id];
+    }
+  } else {
+    delete FlagsBuffer[ old_flag ][ngram_id];
+  }
+  return new_flag;
+}
+
+
+//generic enough
+function ulWriter(rowIndex, record, columns, cellWriter) {
+  // pr("\tulWriter: "+record.id)
+  var tr = '';
+  var cp_rec = {}
+  if(!MyTable) {
+    for(var i in record) {
+      cp_rec[i] = transformContent(RecDict[record.id], i , record[i])
+    }
+  } else {
+    // pr("\t\tbfr transf2: rec_id="+record.id+" | arg="+RecDict[record.id])
+    cp_rec = transformContent2(RecDict[record.id])
+  }
+  // grab the record's attribute for each column
+  // console.log("\tin ulWriter:")
+  // console.log(record)
+  for (var i = 0, len = columns.length; i < len; i++) {
+    tr += cellWriter(columns[i], cp_rec);
+  }
+  var data_id = RecDict[record.id]
+  return '<tr data-stuff='+data_id+'>' + tr + '</tr>';
+}
+
+
+
+
+
+function Main_test( data , initial) {
+
+
+    var DistributionDict = {}
+    for(var i in DistributionDict)
+        delete DistributionDict[i];
+    delete DistributionDict;
+    DistributionDict = {}
+
+    AjaxRecords = []
+
+    var FirstScore = initial;
+
+    var arrayd3 = []
+
+    //  div_table += "\t"+"\t"+"\t"+'<input type="checkbox" id="multiple_selection" onclick="SelectAll(this);" /> Select'+"\n"
+    $("#div-table").html("")
+    $("#div-table").empty();
+    var div_table = '<p align="right">'+"\n"
+      div_table += '<table id="my-ajax-table" class="table table-bordered table-hover">'+"\n"
+      div_table += "\t"+'<thead>'+"\n"
+      div_table += "\t"+"\t"+'<th data-dynatable-column="name">Title</th>'+"\n"
+      div_table += "\t"+"\t"+'<th data-dynatable-column="score" data-dynatable-sorts="score">No. Pubs</th>'+"\n"
+      // div_table += "\t"+"\t"+'<th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>'+"\n"
+      div_table += "\t"+"\t"+'</th>'+"\n"
+      div_table += "\t"+'</thead>'+"\n"
+      div_table += "\t"+'<tbody>'+"\n"
+      div_table += "\t"+'</tbody>'+"\n"
+      div_table += '</table>'+"\n"
+      div_table += '</p>';
+    $("#div-table").html(div_table)
+
+
+    // $("#stats").html(div_stats)
+
+
+    var ID = 0
+    for(var i in data) {
+      console.log(i)
+    
+      // var le_ngram = data.ngrams[i]
+
+      var orig_id = ID
+      var arr_id = parseInt(ID)
+      RecDict[orig_id] = arr_id;
+
+      var url_title = encodeURIComponent(i)//.replace(" ","+")
+      // url_title = i.replace(" ","+")
+      var node_info = {
+        "id" : ID,
+        "name" : '<a target=_blank href="http://google.com/search?q='+url_title+'">'+i+'</a>',
+        "score": data[i],
+      }
+      AjaxRecords.push(node_info)
+
+      if ( ! DistributionDict[node_info.score] ) DistributionDict[node_info.score] = 0;
+      DistributionDict[node_info.score]++;
+
+      ID++;
+    }
+
+    // console.log("The Distribution!:")
+    // console.log(Distribution)
+    var DistributionList = []
+    var min_occ=99999,max_occ=-1,min_frec=99999,max_frec=-1;
+    for(var i in DistributionDict) {
+      var info = {
+        "x_occ":Number(i),
+        "y_frec":DistributionDict[i]
+      }
+      DistributionList.push(info)
+      if(info.x_occ > max_occ) max_occ = info.x_occ
+      if(info.x_occ < min_occ) min_occ = info.x_occ
+      if(info.y_frec > max_frec) max_frec = info.y_frec
+      if(info.y_frec < min_frec) min_frec = info.y_frec
+    }
+
+    oldest = Number(min_occ);
+    latest = Number(max_occ);
+
+
+    var ndx = false;
+    ndx = crossfilter();
+    ndx.add(DistributionList);
+
+    // x_occs  = ndx.dimension(dc.pluck('x_occ'));
+    var x_occs = ndx.dimension(function (d) {
+        return d.x_occ;
+    });
+
+    var y_frecs = x_occs.group().reduceSum(function (d) {
+        return d.y_frec;
+    });
+
+    console.log("scores: [ "+min_occ+" , "+max_occ+" ] ")
+    console.log("frecs: [ "+min_frec+" , "+max_frec+" ] ")
+
+
+    LineChart
+      .width(800)
+      .height(150)
+      .margins({top: 10, right: 50, bottom: 25, left: 40})
+      .group(y_frecs)
+      .dimension(x_occs)
+      .transitionDuration(500)
+      .x(d3.scale.linear().domain([min_occ,max_occ+min_occ]))
+      // .y(d3.scale.log().domain([min_frec/2,max_frec*2]))
+      .renderArea(true)
+      // .valueAccessor(function (d) {
+      //     return d.value;
+      // })
+      
+      // .stack(y_frecs, function (d) {
+      //     return d.value;
+      // })
+      // .ordinalColors(d3.scale.category10())
+      .elasticY(true)
+      // .round(dc.round.floor)
+      .renderHorizontalGridLines(true)
+      .renderVerticalGridLines(true)
+      // .colors('red')
+      // .interpolate("monotone")
+      // .renderDataPoints({radius: 2, fillOpacity: 0.8, strokeOpacity: 0.8})
+      .brushOn(false)
+      .title(function (d) {
+                  var value = d.value.avg ? d.value.avg : d.value;
+                  if (isNaN(value)) value = 0;
+                  return value+" ngrams with "+FirstScore+"="+Number(d.key);
+              })
+      .xAxis();
+    LineChart.yAxis().ticks(5)
+    LineChart.render()
+
+
+    volumeChart.width(800)
+            .height(100)
+            .margins({top: 30, right: 50, bottom: 20, left: 40})
+            .dimension(x_occs)
+            .group(y_frecs)
+            .centerBar(true)
+            .gap(5)
+            .x(d3.scale.linear().domain([min_occ/2,max_occ+min_occ]))
+            .y(d3.scale.sqrt().domain([min_frec/2,max_frec+min_frec]))
+            // .elasticY(true)
+            // // .round(d3.time.month.round)
+            // // .xUnits(d3.time.months)
+            .renderlet(function (chart) {
+                chart.select("g.y").style("display", "none");
+                LineChart.filter(chart.filter());
+                console.log("lalaal moveChart.focus(chartfilt);")
+            })
+            .on("filtered", function (chart) {
+                dc.events.trigger(function () {
+                    var chartfilt = chart.filter()
+                    // tricky part: identifying when the moveChart changes.
+                    if(chartfilt) {
+                        Push2Buffer ( chart.filter() )
+                    } else {
+                        if(TheBuffer) {
+                            Push2Buffer ( false )
+                        }
+                    }
+                    LineChart.focus(chartfilt);
+                });
+            })
+            .xAxis()
+      volumeChart.yAxis().ticks(5)
+      volumeChart.render()
+
+    LineChart.filterAll();
+    volumeChart.filterAll();
+    dc.redrawAll();
+
+    MyTable = []
+    MyTable = $('#my-ajax-table').dynatable({
+                dataset: {
+                  records: AjaxRecords
+                },
+                features: {
+                  pushState: false,
+                  // sort: false //i need to fix the sorting function... the current one just sucks
+                },
+                writers: {
+                  _rowWriter: ulWriter
+                  // _cellWriter: customCellWriter
+                }
+              });
+
+    // MyTable.data('dynatable').settings.dataset.records = []
+    // MyTable.data('dynatable').settings.dataset.originalRecords = []
+    // MyTable.data('dynatable').settings.dataset.originalRecords = AjaxRecords;
+    
+    MyTable.data('dynatable').paginationPage.set(1);
+    // MyTable.data('dynatable').process();
+    // MyTable.data('dynatable').sorts.clear();
+    MyTable.data('dynatable').process();
+
+    // // // $("#score_column_id").children()[0].text = FirstScore
+    // // // // MyTable.data('dynatable').process();
+
+    if ( $(".imadiv").length>0 ) return 1;
+    $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
+    $(".dynatable-record-count").insertAfter(".imadiv")
+    $(".dynatable-pagination-links").insertAfter(".imadiv")
+
+
+
+    return "OK"
+}
+
+console.log(window.location.href+"/journals.json")
+$.ajax({
+  url: window.location.href+"/journals.json",
+  success: function(data){
+
+
+    console.log(data)
+    // // Initializing the Charts and Table
+    var result = Main_test( data , "FirstScore" )
+    console.log( result )
+
+
+
+
+  }
+});
\ No newline at end of file

static/js/test_dyna_chart_and_table.js → static/js/NGrams_dyna_chart_and_table.js

@@ -159,8 +159,16 @@ function Final_UpdateTable( action ) {
 //      Get all the duplicates using the Django-Garg API
 var current_docs = {}
 var BIS_dict = {}
-var corpusid = window.location.href.split("corpus")[1].replace(/\//g, '')//replace all the slashes
-var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
+
+var url_elems = window.location.href.split("/")
+var url_mainIDs = {}
+for(var i=0; i<url_elems.length; i++) {
+  // if the this element is a number:
+  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
+    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
+  }
+}
+var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
 // $.ajax({
 //   url: theurl,
 //   success: function(data) {
@@ -364,14 +372,46 @@ $("#Clean_All").click(function(){
 $("#Save_All").click(function(){
 
   var sum__selected_elems = 0;
+  var poubelle = []
   for(var i in FlagsBuffer)
+    if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
     sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
+  for(var i in poubelle)
+    delete FlagsBuffer[poubelle[i]];
 
   if ( sum__selected_elems>0 ) {
     console.log("")
     console.log("Do the ajax conexion with API and send this array to be processed:")
+    for(var i in FlagsBuffer) {
+      var real_ids = []
+      for (var j in FlagsBuffer[i])
+        real_ids.push( AjaxRecords[j].id );
+
+      FlagsBuffer[i] = real_ids
+    
+    }
     console.log(FlagsBuffer)
-    console.log("")
+    var list_id = $("#list_id").val()
+    // '/annotations/lists/'+list_id+'/ngrams/108642'
+
+    console.log(window.location.origin+'/annotations/lists/'+list_id+"/multiple")
+    console.log(real_ids)
+      $.ajax({
+        method: "POST",
+        url: window.location.origin+'/annotations/lists/'+list_id+"/multiple",
+        data: "to_delete="+JSON.stringify(real_ids),
+        beforeSend: function(xhr) {
+          xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+        },
+        success: function(data){
+              console.log(data)
+        },
+        error: function(result) {
+            console.log("Data not found in #Save_All");
+            console.log(result)
+        }
+      });
+    // console.log("")
   }
 
 });

templates/corpus.html

@@ -5,6 +5,8 @@
 {% load staticfiles %}
 
 <link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
+
 <link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
 <link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
 
@@ -132,7 +134,6 @@ th a {
 </div>
 </div>
 
-
 <div class="container">
 		<div class="row">
 				<div class="col-md-4">
@@ -178,12 +179,29 @@ th a {
         </div>
 </div>
 
+
+<div id="supmofos">
+  
+
+  <select data-width="100px" dir="ltr" class="selectpicker">
+    <option selected>All</option>
+    <option>Title</option>
+    <option>Date</option>
+    <optgroup label="Duplicates">
+      <option>by DOI</option>
+      <option>by Title</option>
+    </optgroup>
+  </select>
+
+</div>
+
 <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
 <script src="{% static "js/charts/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
 <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
 
 <!-- custom-lib for dynatable.js and dc.js -->
-<script type="text/javascript" src="{% static "js/dyna_chart_and_table.js" %}"></script>
+<script type="text/javascript" src="{% static "js/Docs_dyna_chart_and_table.js" %}"></script>
 
 
 

templates/project.html

@@ -264,7 +264,7 @@
 		        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
 		      },
 		      success: function(data) {
-				console.log("in doTheQuery()")
+				console.log("in doTheQuery() Ajax.Success:")
 		        console.log(data)
 		        location.reload();
 		      },
@@ -487,7 +487,6 @@
 
 
 			var pubmedifiedQuery = { query : query , string: query }
-			// console.log(pubmedifiedQuery)
 
 			var projectid = window.location.href.split("project")[1].replace(/\//g, '')//replace all the slashes
 

templates/tests/journals.html

+
+{% extends "menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+
+<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
+
+<link rel="stylesheet" type="text/css" href="{% static "css/dc.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery.dynatable.css"%}"/>
+
+
+<script type="text/javascript" src="{% static "js/charts/d3.js"%}"></script>
+<script type="text/javascript" src="{% static "js/charts/crossfilter.js"%}"></script>
+<!-- <script type="text/javascript" src="http://localhost/i7/js_examples/DC_linear/dc.js"></script> -->
+<script type="text/javascript" src="{% static "js/charts/dc.js"%}"></script>
+
+
+<style>
+.no-transition {
+  -webkit-transition: height 0.1s;
+  -moz-transition: height 0.1s;
+  -ms-transition: height 0.1s;
+  -o-transition: height 0.1s;
+  transition: height 0.1s;
+}
+th { color: #fff; }
+
+th a {
+  color: #fff;
+  font-weight: normal;
+  font-style: italic;
+  font-size: 0.9em;
+}
+
+/*tr:hover {
+  cursor: pointer;
+  color: #000;
+  font-weight: bold;
+}*/
+
+.to_delete {
+  color:red; 
+  opacity: 0.8;
+}
+
+.to_group {
+  color:blue;
+}
+
+
+.to_keep {
+  color:green;
+}
+
+.dynatable-record-count {
+  font-size: 0.7em;
+}
+.dynatable-pagination-links {
+  font-size: 0.7em;  
+}
+input[type=radio] {
+    display:none;
+}
+ 
+input[type=radio] + label {
+    display:inline-block;
+    margin:-2px;
+    padding: 4px 12px;
+    margin-bottom: 0;
+    font-size: 14px;
+    line-height: 20px;
+    color: #333;
+    text-align: center;
+    text-shadow: 0 1px 1px rgba(255,255,255,0.75);
+    vertical-align: middle;
+    cursor: pointer;
+    background-color: #f5f5f5;
+    background-image: -moz-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));
+    background-image: -webkit-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -o-linear-gradient(top,#fff,#e6e6e6);
+    background-image: linear-gradient(to bottom,#fff,#e6e6e6);
+    background-repeat: repeat-x;
+    border: 1px solid #ccc;
+    border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+    border-color: rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);
+    border-bottom-color: #b3b3b3;
+    filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);
+    filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
+    -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+}
+ 
+input[type=radio]:checked + label {
+       background-image: none;
+    outline: 0;
+    -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+        background-color:#e0e0e0;
+}
+
+</style>
+
+{% endblock %}
+
+
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+        <div class="jumbotron">
+                        {% if project %}
+                            <h1>{{ project.name }}, {{ corpus.name }}
+                            </h1>
+                            {% endif %}
+
+                        {% if corpus %}
+                        <p>
+                        Created on {{ corpus.date }} 
+                        </p>
+                        <div id="stats"></div>
+                        {% endif %}
+
+                            <!--                        <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
+                        <a class="btn btn-primary btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save as</a>
+                        <a class="btn btn-primary btn-lg" role="button" href="/delete/{{ corpus.id }}">Delete</a></p>
+
+                        {% if number == 0 %}
+                        <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add documents</a></p>
+                        {% endif %}
+    
+        </div>
+</div>
+
+                <div class="container">
+                        <div class="container">
+
+                            <div class="row">
+                                    
+                                    <center>
+                                    <div id="ScoresBox"></div> <br>(Blue bars: all,  Green line: zoom)
+                                            <a class="reset" href="javascript:volumeChart.filterAll();dc.redrawAll();"
+                                                    style="display: none;">reset</a>
+                                            <div class="clearfix"></div>
+                                    </center>
+                                    <div id="monthly-move-chart"></div>
+                            </div>
+
+                            <div class="row">
+                                    <div id="monthly-volume-chart"></div>
+                                    <p class="muted pull-left" style="margin-right: 15px;">Select a time range to zoom in</p>
+                            </div>
+                    </div>
+                        <p align="center">
+                        <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a></p>
+
+
+<div class="container">
+        <div class="jumbotron">
+
+<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+  <div class="panel panel-body">
+    <div class="panel-heading">
+      <h4 class="panel-title">
+        <a data-toggle="collapse" data-parent="#accordion" href="#collapseOne">
+           <p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg" style="width:200px; margin:0 auto; display:block;">Open Folder</h2></p>
+        </a>
+      </h4>
+    </div>
+    <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
+      <div class="panel-body">
+                  <div id="div-table"></div>
+                <!-- <p align="right">
+                    <table id="my-ajax-table" class="table table-bordered">
+                      <thead>
+                        <th data-dynatable-column="name">Title</th>
+                        <th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>
+                        <th data-dynatable-column="del" data-dynatable-no-sort="true">
+                            <input type="checkbox" id="multiple_selection" onclick="SelectAll(this);" /> Select
+                        </th>
+                      </thead>
+                      <tbody>
+                      </tbody>
+                    </table>
+                </p> -->
+
+
+      </div>
+    </div>
+  </div>
+</div>
+</div>
+</div>
+
+
+<div class="container">
+		<div class="row">
+				<div class="col-md-4">
+						<div class="jumbotron">
+						<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
+						<ol>
+								<li>Count</li> <!-- read, compute -->
+								<li>Filter</li> <!-- count, compute -->
+								<li>Compare</li> <!-- select, cut -->
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+					</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+										<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3>
+						<ol>
+								<li>Sort</li>
+								<li>Group</li>
+								<li>Cluster</li>
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+						</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+								{% if processing > 0 %}
+                <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
+                {% else %}
+                                <h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% endif %}
+                        <ol>
+                                <li>Visualize</li>
+                                <li>Explore</li>
+                                <li>Read</li>
+                        </ol>
+                        <h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+                        </div>
+                </div>
+        </div>
+</div>
+
+
+
+
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
+
+<!-- custom-lib for dynatable.js and dc.js -->
+<script type="text/javascript" src="{% static "js/Journals_dyna_chart_and_table.js" %}"></script>
+
+
+
+
+{% endblock %}

templates/tests/ngrams.html

@@ -281,7 +281,7 @@ input[type=radio]:checked + label {
 <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
 
 <!-- custom-lib for dynatable.js and dc.js -->
-<script type="text/javascript" src="{% static "js/test_dyna_chart_and_table.js" %}"></script>
+<script type="text/javascript" src="{% static "js/NGrams_dyna_chart_and_table.js" %}"></script>
 
 
 

tests/ngramstable/views.py

@@ -50,6 +50,46 @@ from gargantext_web import about
 from gargantext_web.api import JsonHttpResponse
 
 
+from ngram.lists import listIds, listNgramIds, ngramList , doList
+
+
+def test_page(request , project_id , corpus_id):
+
+    if not request.user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+    
+    try:
+        offset = int(project_id)
+        offset = int(corpus_id)
+    except ValueError:
+        raise Http404()
+
+    t = get_template('tests/test_select-boostrap.html')
+    
+    user = cache.User[request.user.username].id
+    date = datetime.datetime.now()
+    project = cache.Node[int(project_id)]
+    corpus  = cache.Node[int(corpus_id)]
+    type_doc_id = cache.NodeType['Document'].id
+    number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+    try:
+        processing = corpus.hyperdata['Processing']
+    except Exception as error:
+        print(error)
+        processing = 0
+
+    html = t.render(Context({
+            'debug': settings.DEBUG,
+            'user': user,
+            'date': date,
+            'project': project,
+            'corpus' : corpus,
+            'processing' : processing,
+            'number' : number,
+            }))
+
+    return HttpResponse(html)
+
 def get_ngrams(request , project_id , corpus_id ):
     if not request.user.is_authenticated():
         return redirect('/login/?next=%s' % request.path)
@@ -68,6 +108,80 @@ def get_ngrams(request , project_id , corpus_id ):
     corpus  = cache.Node[int(corpus_id)]
     type_doc_id = cache.NodeType['Document'].id
     number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+
+    lists = dict()
+    for list_type in ['MiamList', 'StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+
+    try:
+        processing = corpus.hyperdata['Processing']
+    except Exception as error:
+        print(error)
+        processing = 0
+
+    html = t.render(Context({
+            'debug': settings.DEBUG,
+            'user': user,
+            'date': date,
+            'project': project,
+            'corpus' : corpus,
+            'processing' : processing,
+            'number' : number,
+            'list_id': list_id[0][0],
+            }))
+
+    return HttpResponse(html)
+
+
+
+def test_test(request , corpus_id , doc_id):
+    """Get All for a doc id"""
+    corpus_id = int(corpus_id)
+    doc_id = int(doc_id)
+    lists = dict()
+    for list_type in ['StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+    print(list_id[0][0])
+
+
+    # # # ngrams of list_id of corpus_id:
+    # commeca = "StopList"
+    doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
+
+    to_del = {}
+    for n in doc_ngram_list:
+        to_del[ n[0] ] = True
+
+    print( to_del.keys() )
+
+    results = [ "hola" , "mundo" ]
+    return JsonHttpResponse(results)
+
+
+def get_journals(request , project_id , corpus_id ):
+
+    if not request.user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+    
+    try:
+        offset = int(project_id)
+        offset = int(corpus_id)
+    except ValueError:
+        raise Http404()
+
+    t = get_template('tests/journals.html')
+    
+    user = cache.User[request.user.username].id
+    date = datetime.datetime.now()
+    project = cache.Node[int(project_id)]
+    corpus  = cache.Node[int(corpus_id)]
+    type_doc_id = cache.NodeType['Document'].id
+    number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+
     try:
         processing = corpus.hyperdata['Processing']
     except Exception as error:
@@ -86,6 +200,21 @@ def get_ngrams(request , project_id , corpus_id ):
 
     return HttpResponse(html)
 
+def test_journals(request , project_id, corpus_id ):
+    results = ["hola" , "mundo"]
+
+    JournalsDict = {}
+
+    user_id = request.user.id
+    document_type_id = cache.NodeType['Document'].id
+    documents  = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
+    for doc in documents:
+        if "journal" in doc.hyperdata:
+            journal = doc.hyperdata["journal"]
+            if journal not in JournalsDict:
+                JournalsDict [journal] = 0
+            JournalsDict[journal] += 1
+    return JsonHttpResponse(JournalsDict)
 
 def test_ngrams(request , project_id, corpus_id ):
     results = ["hola" , "mundo"]
@@ -94,6 +223,17 @@ def test_ngrams(request , project_id, corpus_id ):
     whitelist_type_id = cache.NodeType['WhiteList'].id
     document_type_id = cache.NodeType['Document'].id
 
+    corpus_id = int(corpus_id)
+    lists = dict()
+    for list_type in ['StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+    doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
+    StopList = {}
+    for n in doc_ngram_list:
+        StopList[ n[0] ] = True
+
     # # 13099  clinical benefits
     # # 7492   recent data
     # # 14279  brain development
@@ -145,15 +285,16 @@ def test_ngrams(request , project_id, corpus_id ):
     for doc in documents:
         NgramOccs = session.query(Node_Ngram).filter( Node_Ngram.node_id==doc.id).all()
         for ngram in NgramOccs:
-            if ngram.ngram_id not in Ngrams_Scores:
-                Ngrams_Scores[ngram.ngram_id] = {} 
-                Ngrams_Scores[ngram.ngram_id]["scores"] = {
-                    "occ_sum": 0.0,
-                    "occ_uniq": 0.0,
-                    "tfidf_sum": 0.0
-                }
-            Ngrams_Scores[ngram.ngram_id]["scores"]["occ_sum"]+=ngram.weight
-            Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]+=1
+            if ngram.ngram_id not in StopList:
+                if ngram.ngram_id not in Ngrams_Scores:
+                    Ngrams_Scores[ngram.ngram_id] = {} 
+                    Ngrams_Scores[ngram.ngram_id]["scores"] = {
+                        "occ_sum": 0.0,
+                        "occ_uniq": 0.0,
+                        "tfidf_sum": 0.0
+                    }
+                Ngrams_Scores[ngram.ngram_id]["scores"]["occ_sum"]+=ngram.weight
+                Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]+=1
             # print("\t" , ngram.ngram_id , "\t" , ngram.weight )
     ## Getting the Effective nro de OCCS / >##
 
@@ -177,9 +318,10 @@ def test_ngrams(request , project_id, corpus_id ):
     Sum = 0
     NgramTFIDF = session.query(NodeNodeNgram).filter( NodeNodeNgram.nodex_id==corpus_id ).all()
     for ngram in NgramTFIDF:
-        Ngrams_Scores[ngram.ngram_id]["scores"]["tfidf_sum"] += ngram.score
-        Sum += Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]
-        # print( "docid:", ngram.nodey_id , ngram.ngram_id  , ngram.score)
+        if ngram.ngram_id not in StopList:
+            Ngrams_Scores[ngram.ngram_id]["scores"]["tfidf_sum"] += ngram.score
+            Sum += Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]
+            # print( "docid:", ngram.nodey_id , ngram.ngram_id  , ngram.score)
 
 
     # import pprint
@@ -210,13 +352,14 @@ def test_ngrams(request , project_id, corpus_id ):
     query = session.query(Ngram).filter(Ngram.id.in_( ngrams_ids ))
     ngrams_data = query.all()
     for ngram in ngrams_data:
-    	occ_uniq = Ngrams_Scores[ngram.id]["scores"]["occ_uniq"]
-    	if occ_uniq > occs_threshold:
-	        Ngrams_Scores[ngram.id]["name"] = ngram.terms
-	        Ngrams_Scores[ngram.id]["id"] = ngram.id
-	        Ngrams_Scores[ngram.id]["scores"]["tfidf"] = Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"] / occ_uniq
-	        del Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"]
-	        Metrics["ngrams"].append( Ngrams_Scores[ngram.id] )
+        if ngram.id not in StopList:
+            occ_uniq = Ngrams_Scores[ngram.id]["scores"]["occ_uniq"]
+            if occ_uniq > occs_threshold:
+                Ngrams_Scores[ngram.id]["name"] = ngram.terms
+                Ngrams_Scores[ngram.id]["id"] = ngram.id
+                Ngrams_Scores[ngram.id]["scores"]["tfidf"] = Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"] / occ_uniq
+                del Ngrams_Scores[ngram.id]["scores"]["tfidf_sum"]
+                Metrics["ngrams"].append( Ngrams_Scores[ngram.id] )