Merge branch 'samuel' into merge

annotations/urls.py

@@ -6,4 +6,5 @@ urlpatterns = patterns('',
     url(r'^document/(?P<doc_id>[0-9]+)$', views.Document.as_view()), # document view
     url(r'^corpus/(?P<corpus_id>[0-9]+)/document/(?P<doc_id>[0-9]+)$', views.NgramList.as_view()), # the list associated with an ngram
     url(r'^lists/(?P<list_id>[0-9]+)/ngrams(?:/(?P<ngram_id>[0-9]+))?$', views.NgramEdit.as_view()), #
+    url(r'^lists/(?P<list_id>[0-9]+)/multiple?$', views.deleteMultiple ), #
 )

annotations/views.py

@@ -16,6 +16,8 @@ from rest_framework.authentication import SessionAuthentication, BasicAuthentica
 from node.models import Node
 from gargantext_web.db import *
 from ngram.lists import listIds, listNgramIds, ngramList
+from gargantext_web.api import JsonHttpResponse
+import json
 
 
 @login_required
@@ -109,6 +111,26 @@ class NgramEdit(APIView):
         session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).delete()
         return Response(None, 204)
 
+def deleteMultiple(request, list_id):
+    results = ["hola","mundo"]
+
+    user = request.user
+    if not user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+
+    if request.POST:
+        todel_ids = json.loads(request.POST['to_delete'])
+        for ngram_id in todel_ids:
+            # add the ngram to the list if not already done
+            node_ngram = session.query(Node_Ngram).filter(Node_Ngram.node_id==list_id).filter(Node_Ngram.ngram_id==ngram_id).first()
+            if node_ngram is None:
+                node_ngram = Node_Ngram(node_id=list_id, ngram_id=ngram_id, weight=1.0)
+                session.add(node_ngram)
+                session.commit()
+
+    return JsonHttpResponse(results)
+
+
 
 class Document(APIView):
     """

gargantext_web/urls.py

@@ -85,7 +85,10 @@ urlpatterns = patterns('',
     url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
     url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
     url(r'^project/(\d+)/corpus/(\d+)/ngrams/ngrams.json$', samtest.test_ngrams),
-    url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams)
+    url(r'^project/(\d+)/corpus/(\d+)/ngrams$', samtest.get_ngrams),
+    url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', samtest.test_journals),
+    url(r'^project/(\d+)/corpus/(\d+)/journals$', samtest.get_journals),
+    url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
 )
 
 

node/admin.py

@@ -98,8 +98,8 @@ from django import forms
 
 from django.utils.translation import ugettext_lazy as _
 class CustomForm(forms.Form):
-    name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
     type = ModelChoiceField( ResourceType.objects.all() , widget=forms.Select(attrs={'onchange':'CustomForSelect( $("option:selected", this).text() );'}) )
+    name = forms.CharField( label='Name', max_length=199 , widget=forms.TextInput(attrs={ 'required': 'true' }))
     file = forms.FileField()
 
     # Description: clean_file()

scrappers/scrap_pubmed/views.py

@@ -270,10 +270,9 @@ def testISTEX(request , project_id):
 					file = filename,
 				)
 				dwnldsOK+=1	
-
-		# print(urlreqs)
-
-
+		if dwnldsOK == 0: return JsonHttpResponse(["fail"])		
+		###########################
+		###########################
 		try:
 			if not DEBUG:
 				apply_workflow.apply_async((corpus.id,),)
@@ -287,50 +286,6 @@ def testISTEX(request , project_id):
 		return HttpResponseRedirect('/project/' + str(project_id))
 
 
-
-
-		# resource_type = ResourceType.objects.get(name="istext" )
-
-		# parent      = Node.objects.get(id=project_id)
-		# node_type   = NodeType.objects.get(name='Corpus')
-		# type_id = NodeType.objects.get(name='Document').id
-		# user_id = User.objects.get( username=request.user ).id
-
-		# corpus = Node(
-		# 	user=request.user,
-		# 	parent=parent,
-		# 	type=node_type,
-		# 	name=query,
-		# )
-
-		# corpus.save()
-
-		# # configuring your queue with the event
-		# for i in range(8):
-		# 	t = threading.Thread(target=tasks.worker2) #thing to do
-		# 	t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
-		# 	t.start()
-		# for url in urlreqs:
-		# 	filename = MEDIA_ROOT + '/corpora/%s/%s' % (request.user, str(datetime.now().microsecond))
-		# 	tasks.q.put( [url , filename]) #put a task in th queue
-		# tasks.q.join() # wait until everything is finished
-		# for filename in tasks.firstResults:
-		# 	corpus.add_resource( user=request.user, type=resource_type, file=filename )
-
-
-		# corpus.save()
-		# print("DEBUG:",DEBUG)
-		# # do the WorkFlow
-		# try:
-		# 	if DEBUG is True:
-		# 		corpus.workflow()
-		# 	else:
-		# 		corpus.workflow.apply_async((), countdown=3)
-
-		# 	return JsonHttpResponse(["workflow","finished"])
-		# except Exception as error:
-		# 	print(error)
-
 	data = [query_string,query,N]
 	return JsonHttpResponse(data)
 

static/js/dyna_chart_and_table.js → static/js/Docs_dyna_chart_and_table.js

@@ -119,11 +119,15 @@ function Final_UpdateTable( action ) {
 var current_docs = {}
 var BIS_dict = {}
 
-var path = window.location.pathname.match(/\/project\/(.*)\/corpus\/(.*)\//);
-var projectid = path[1]
-var corpusid  = path[2]
-
-var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
+var url_elems = window.location.href.split("/")
+var url_mainIDs = {}
+for(var i=0; i<url_elems.length; i++) {
+  // if the this element is a number:
+  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
+    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
+  }
+}
+var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
 // $.ajax({
 //   url: theurl,
 //   success: function(data) {
@@ -204,6 +208,38 @@ function transformContent(rec_id , header , content) {
   } else return content;
 }
 
+
+$("#move2trash")
+.click(function(){
+
+    var ids2trash = []
+    for(var i in Garbage) {
+      ids2trash.push(AjaxRecords[i].id);
+    }
+
+    console.log("ids to the trash:")
+    console.log(ids2trash)
+
+    $.ajax({
+      url: "/tests/move2trash/",
+      data: "nodeids="+JSON.stringify(ids2trash),
+      type: 'POST',
+      beforeSend: function(xhr) {
+        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+      },
+      success: function(data) {
+        console.log("in #move2trash")
+        console.log(data)
+        location.reload();
+      },
+        error: function(result) {
+            console.log("Data not found in #move2trash");
+            console.log(result)
+        }
+    });
+})
+.hide();
+
 //generic enough
 function ulWriter(rowIndex, record, columns, cellWriter) {
   // pr("\tulWriter: "+record.id)
@@ -224,9 +260,30 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
   return '<tr>' + tr + '</tr>';
 }
 
+
+
+// var div__filter_for_search = ''
+//   div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple  data-max-options="1">';
+//   div__filter_for_search += '  <optgroup label="All" data-max-options="1" selected>';
+//   div__filter_for_search += '    <option>Title</option>';
+//   div__filter_for_search += '    <option>Date</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '  <optgroup label="Category" data-max-options="1">';
+//   div__filter_for_search += '    <option>Title</option>';
+//   div__filter_for_search += '    <option>Date</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '  <optgroup label="Duplicates" data-max-options="1">';
+//   div__filter_for_search += '    <option>by DOI</option>';
+//   div__filter_for_search += '    <option>by Title</option>';
+//   div__filter_for_search += '  </optgroup>';
+//   div__filter_for_search += '</select>';
+
+// $("#supmofos").html(div__filter_for_search)
+
+
 // (3) Get records and hyperdata for paginator
   $.ajax({
-    url: '/tests/paginator/corpus/'+corpusid,
+    url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
     success: function(data){
       console.log(data)
 
@@ -235,7 +292,7 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
         var orig_id = parseInt(data.records[i].id)
         var arr_id = parseInt(i)
         RecDict[orig_id] = arr_id;
-        data.records[i]["name"] = '<a target="_blank" href="/project/'+projectid+'/corpus/'+ corpusid + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
+        data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
         data.records[i]["del"] = false
 
         var date = data.records[i]["date"];  
@@ -247,36 +304,6 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
 
       // $("#move2trash").prop('disabled', true);
 
-      $("#move2trash")
-      .click(function(){
-
-          var ids2trash = []
-          for(var i in Garbage) {
-            ids2trash.push(AjaxRecords[i].id);
-          }
-
-          console.log("ids to the trash:")
-          console.log(ids2trash)
-
-          $.ajax({
-            url: "/tests/move2trash/",
-            data: "nodeids="+JSON.stringify(ids2trash),
-            type: 'POST',
-            beforeSend: function(xhr) {
-              xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-            },
-            success: function(data) {
-              console.log("in #move2trash")
-              console.log(data)
-              location.reload();
-            },
-              error: function(result) {
-                  console.log("Data not found in #move2trash");
-                  console.log(result)
-              }
-          });
-      })
-      .hide();
 
 
       var t0 = AjaxRecords[0].date.split("-").map(Number)
@@ -429,6 +456,13 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
       $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
       $(".dynatable-record-count").insertAfter(".imadiv")
       $(".dynatable-pagination-links").insertAfter(".imadiv")
-      // console.log(RecDict)
+      
+      var the_content = $("#supmofos").html();
+      $(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
+      $("#supmofos").remove()
+      // .insertAfter("#dynatable-query-search-my-ajax-table")
+
+
     }
   });
+

static/js/test_dyna_chart_and_table.js → static/js/NGrams_dyna_chart_and_table.js

@@ -159,8 +159,16 @@ function Final_UpdateTable( action ) {
 //      Get all the duplicates using the Django-Garg API
 var current_docs = {}
 var BIS_dict = {}
-var corpusid = window.location.href.split("corpus")[1].replace(/\//g, '')//replace all the slashes
-var theurl = "/api/nodes/"+corpusid+"/children/duplicates?keys=title&limit=9999"
+
+var url_elems = window.location.href.split("/")
+var url_mainIDs = {}
+for(var i=0; i<url_elems.length; i++) {
+  // if the this element is a number:
+  if(url_elems[i]!="" && !isNaN(Number(url_elems[i]))) {
+    url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
+  }
+}
+var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
 // $.ajax({
 //   url: theurl,
 //   success: function(data) {
@@ -364,14 +372,46 @@ $("#Clean_All").click(function(){
 $("#Save_All").click(function(){
 
   var sum__selected_elems = 0;
+  var poubelle = []
   for(var i in FlagsBuffer)
+    if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
     sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
+  for(var i in poubelle)
+    delete FlagsBuffer[poubelle[i]];
 
   if ( sum__selected_elems>0 ) {
     console.log("")
     console.log("Do the ajax conexion with API and send this array to be processed:")
+    for(var i in FlagsBuffer) {
+      var real_ids = []
+      for (var j in FlagsBuffer[i])
+        real_ids.push( AjaxRecords[j].id );
+
+      FlagsBuffer[i] = real_ids
+    
+    }
     console.log(FlagsBuffer)
-    console.log("")
+    var list_id = $("#list_id").val()
+    // '/annotations/lists/'+list_id+'/ngrams/108642'
+
+    console.log(window.location.origin+'/annotations/lists/'+list_id+"/multiple")
+    console.log(real_ids)
+      $.ajax({
+        method: "POST",
+        url: window.location.origin+'/annotations/lists/'+list_id+"/multiple",
+        data: "to_delete="+JSON.stringify(real_ids),
+        beforeSend: function(xhr) {
+          xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
+        },
+        success: function(data){
+              console.log(data)
+        },
+        error: function(result) {
+            console.log("Data not found in #Save_All");
+            console.log(result)
+        }
+      });
+    // console.log("")
   }
 
 });

templates/corpus.html

@@ -5,6 +5,8 @@
 {% load staticfiles %}
 
 <link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
+
 <link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
 <link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
 
@@ -132,7 +134,6 @@ th a {
 </div>
 </div>
 
-
 <div class="container">
 		<div class="row">
 				<div class="col-md-4">
@@ -178,12 +179,29 @@ th a {
         </div>
 </div>
 
+
+<div id="supmofos">
+  
+
+  <select data-width="100px" dir="ltr" class="selectpicker">
+    <option selected>All</option>
+    <option>Title</option>
+    <option>Date</option>
+    <optgroup label="Duplicates">
+      <option>by DOI</option>
+      <option>by Title</option>
+    </optgroup>
+  </select>
+
+</div>
+
 <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
 <script src="{% static "js/charts/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
 <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
 
 <!-- custom-lib for dynatable.js and dc.js -->
-<script type="text/javascript" src="{% static "js/dyna_chart_and_table.js" %}"></script>
+<script type="text/javascript" src="{% static "js/Docs_dyna_chart_and_table.js" %}"></script>
 
 
 

templates/project.html

@@ -264,7 +264,7 @@
 		        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
 		      },
 		      success: function(data) {
-				console.log("in doTheQuery()")
+				console.log("in doTheQuery() Ajax.Success:")
 		        console.log(data)
 		        location.reload();
 		      },
@@ -487,7 +487,6 @@
 
 
 			var pubmedifiedQuery = { query : query , string: query }
-			// console.log(pubmedifiedQuery)
 
 			var projectid = window.location.href.split("project")[1].replace(/\//g, '')//replace all the slashes
 

templates/tests/journals.html

+
+{% extends "menu.html" %}
+
+{% block css %}
+{% load staticfiles %}
+
+<link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
+
+<link rel="stylesheet" type="text/css" href="{% static "css/dc.css"%}"/>
+<link rel="stylesheet" type="text/css" href="{% static "css/jquery.dynatable.css"%}"/>
+
+
+<script type="text/javascript" src="{% static "js/charts/d3.js"%}"></script>
+<script type="text/javascript" src="{% static "js/charts/crossfilter.js"%}"></script>
+<!-- <script type="text/javascript" src="http://localhost/i7/js_examples/DC_linear/dc.js"></script> -->
+<script type="text/javascript" src="{% static "js/charts/dc.js"%}"></script>
+
+
+<style>
+.no-transition {
+  -webkit-transition: height 0.1s;
+  -moz-transition: height 0.1s;
+  -ms-transition: height 0.1s;
+  -o-transition: height 0.1s;
+  transition: height 0.1s;
+}
+th { color: #fff; }
+
+th a {
+  color: #fff;
+  font-weight: normal;
+  font-style: italic;
+  font-size: 0.9em;
+}
+
+/*tr:hover {
+  cursor: pointer;
+  color: #000;
+  font-weight: bold;
+}*/
+
+.to_delete {
+  color:red; 
+  opacity: 0.8;
+}
+
+.to_group {
+  color:blue;
+}
+
+
+.to_keep {
+  color:green;
+}
+
+.dynatable-record-count {
+  font-size: 0.7em;
+}
+.dynatable-pagination-links {
+  font-size: 0.7em;  
+}
+input[type=radio] {
+    display:none;
+}
+ 
+input[type=radio] + label {
+    display:inline-block;
+    margin:-2px;
+    padding: 4px 12px;
+    margin-bottom: 0;
+    font-size: 14px;
+    line-height: 20px;
+    color: #333;
+    text-align: center;
+    text-shadow: 0 1px 1px rgba(255,255,255,0.75);
+    vertical-align: middle;
+    cursor: pointer;
+    background-color: #f5f5f5;
+    background-image: -moz-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));
+    background-image: -webkit-linear-gradient(top,#fff,#e6e6e6);
+    background-image: -o-linear-gradient(top,#fff,#e6e6e6);
+    background-image: linear-gradient(to bottom,#fff,#e6e6e6);
+    background-repeat: repeat-x;
+    border: 1px solid #ccc;
+    border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+    border-color: rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);
+    border-bottom-color: #b3b3b3;
+    filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);
+    filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
+    -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);
+}
+ 
+input[type=radio]:checked + label {
+       background-image: none;
+    outline: 0;
+    -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+    box-shadow: inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);
+        background-color:#e0e0e0;
+}
+
+</style>
+
+{% endblock %}
+
+
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+        <div class="jumbotron">
+                        {% if project %}
+                            <h1>{{ project.name }}, {{ corpus.name }}
+                            </h1>
+                            {% endif %}
+
+                        {% if corpus %}
+                        <p>
+                        Created on {{ corpus.date }} 
+                        </p>
+                        <div id="stats"></div>
+                        {% endif %}
+
+                            <!--                        <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add file</a> -->
+                        <a class="btn btn-primary btn-lg" role="button" href="/project/{{project.id}}/corpus/{{ corpus.id }}/corpus.csv">Save as</a>
+                        <a class="btn btn-primary btn-lg" role="button" href="/delete/{{ corpus.id }}">Delete</a></p>
+
+                        {% if number == 0 %}
+                        <a class="btn btn-primary btn-lg" role="button" href="/admin/documents/corpus/{{ corpus.id }}/">Add documents</a></p>
+                        {% endif %}
+    
+        </div>
+</div>
+
+                <div class="container">
+                        <div class="container">
+
+                            <div class="row">
+                                    
+                                    <center>
+                                    <div id="ScoresBox"></div> <br>(Blue bars: all,  Green line: zoom)
+                                            <a class="reset" href="javascript:volumeChart.filterAll();dc.redrawAll();"
+                                                    style="display: none;">reset</a>
+                                            <div class="clearfix"></div>
+                                    </center>
+                                    <div id="monthly-move-chart"></div>
+                            </div>
+
+                            <div class="row">
+                                    <div id="monthly-volume-chart"></div>
+                                    <p class="muted pull-left" style="margin-right: 15px;">Select a time range to zoom in</p>
+                            </div>
+                    </div>
+                        <p align="center">
+                        <a class="btn btn-xs btn-default" role="button" href="/chart/corpus/{{ corpus.id }}/data.csv">Save</a></p>
+
+
+<div class="container">
+        <div class="jumbotron">
+
+<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+  <div class="panel panel-body">
+    <div class="panel-heading">
+      <h4 class="panel-title">
+        <a data-toggle="collapse" data-parent="#accordion" href="#collapseOne">
+           <p id="corpusdisplayer" onclick='Final_UpdateTable("click")' class="btn btn-primary btn-lg" style="width:200px; margin:0 auto; display:block;">Open Folder</h2></p>
+        </a>
+      </h4>
+    </div>
+    <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
+      <div class="panel-body">
+                  <div id="div-table"></div>
+                <!-- <p align="right">
+                    <table id="my-ajax-table" class="table table-bordered">
+                      <thead>
+                        <th data-dynatable-column="name">Title</th>
+                        <th id="score_column_id" data-dynatable-sorts="score" data-dynatable-column="score">Score</th>
+                        <th data-dynatable-column="del" data-dynatable-no-sort="true">
+                            <input type="checkbox" id="multiple_selection" onclick="SelectAll(this);" /> Select
+                        </th>
+                      </thead>
+                      <tbody>
+                      </tbody>
+                    </table>
+                </p> -->
+
+
+      </div>
+    </div>
+  </div>
+</div>
+</div>
+</div>
+
+
+<div class="container">
+		<div class="row">
+				<div class="col-md-4">
+						<div class="jumbotron">
+						<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
+						<ol>
+								<li>Count</li> <!-- read, compute -->
+								<li>Filter</li> <!-- count, compute -->
+								<li>Compare</li> <!-- select, cut -->
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+					</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+										<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3>
+						<ol>
+								<li>Sort</li>
+								<li>Group</li>
+								<li>Cluster</li>
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+						</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+								{% if processing > 0 %}
+                <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
+                {% else %}
+                                <h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% endif %}
+                        <ol>
+                                <li>Visualize</li>
+                                <li>Explore</li>
+                                <li>Read</li>
+                        </ol>
+                        <h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+                        </div>
+                </div>
+        </div>
+</div>
+
+
+
+
+<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script src="{% static "js/charts/bootstrap.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
+
+<!-- custom-lib for dynatable.js and dc.js -->
+<script type="text/javascript" src="{% static "js/Journals_dyna_chart_and_table.js" %}"></script>
+
+
+
+
+{% endblock %}

templates/tests/ngrams.html

@@ -281,7 +281,7 @@ input[type=radio]:checked + label {
 <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
 
 <!-- custom-lib for dynatable.js and dc.js -->
-<script type="text/javascript" src="{% static "js/test_dyna_chart_and_table.js" %}"></script>
+<script type="text/javascript" src="{% static "js/NGrams_dyna_chart_and_table.js" %}"></script>
 
 
 

tests/ngramstable/views.py

@@ -50,6 +50,46 @@ from gargantext_web import about
 from gargantext_web.api import JsonHttpResponse
 
 
+from ngram.lists import listIds, listNgramIds, ngramList , doList
+
+
+def test_page(request , project_id , corpus_id):
+
+    if not request.user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+    
+    try:
+        offset = int(project_id)
+        offset = int(corpus_id)
+    except ValueError:
+        raise Http404()
+
+    t = get_template('tests/test_select-boostrap.html')
+    
+    user = cache.User[request.user.username].id
+    date = datetime.datetime.now()
+    project = cache.Node[int(project_id)]
+    corpus  = cache.Node[int(corpus_id)]
+    type_doc_id = cache.NodeType['Document'].id
+    number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+    try:
+        processing = corpus.hyperdata['Processing']
+    except Exception as error:
+        print(error)
+        processing = 0
+
+    html = t.render(Context({
+            'debug': settings.DEBUG,
+            'user': user,
+            'date': date,
+            'project': project,
+            'corpus' : corpus,
+            'processing' : processing,
+            'number' : number,
+            }))
+
+    return HttpResponse(html)
+
 def get_ngrams(request , project_id , corpus_id ):
     if not request.user.is_authenticated():
         return redirect('/login/?next=%s' % request.path)
@@ -68,6 +108,80 @@ def get_ngrams(request , project_id , corpus_id ):
     corpus  = cache.Node[int(corpus_id)]
     type_doc_id = cache.NodeType['Document'].id
     number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+
+    lists = dict()
+    for list_type in ['MiamList', 'StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+
+    try:
+        processing = corpus.hyperdata['Processing']
+    except Exception as error:
+        print(error)
+        processing = 0
+
+    html = t.render(Context({
+            'debug': settings.DEBUG,
+            'user': user,
+            'date': date,
+            'project': project,
+            'corpus' : corpus,
+            'processing' : processing,
+            'number' : number,
+            'list_id': list_id[0][0],
+            }))
+
+    return HttpResponse(html)
+
+
+
+def test_test(request , corpus_id , doc_id):
+    """Get All for a doc id"""
+    corpus_id = int(corpus_id)
+    doc_id = int(doc_id)
+    lists = dict()
+    for list_type in ['StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+    print(list_id[0][0])
+
+
+    # # # ngrams of list_id of corpus_id:
+    # commeca = "StopList"
+    doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
+
+    to_del = {}
+    for n in doc_ngram_list:
+        to_del[ n[0] ] = True
+
+    print( to_del.keys() )
+
+    results = [ "hola" , "mundo" ]
+    return JsonHttpResponse(results)
+
+
+def get_journals(request , project_id , corpus_id ):
+
+    if not request.user.is_authenticated():
+        return redirect('/login/?next=%s' % request.path)
+    
+    try:
+        offset = int(project_id)
+        offset = int(corpus_id)
+    except ValueError:
+        raise Http404()
+
+    t = get_template('tests/journals.html')
+    
+    user = cache.User[request.user.username].id
+    date = datetime.datetime.now()
+    project = cache.Node[int(project_id)]
+    corpus  = cache.Node[int(corpus_id)]
+    type_doc_id = cache.NodeType['Document'].id
+    number = session.query(func.count(Node.id)).filter(Node.parent_id==corpus_id, Node.type_id==type_doc_id).all()[0][0]
+
     try:
         processing = corpus.hyperdata['Processing']
     except Exception as error:
@@ -86,6 +200,21 @@ def get_ngrams(request , project_id , corpus_id ):
 
     return HttpResponse(html)
 
+def test_journals(request , project_id, corpus_id ):
+    results = ["hola" , "mundo"]
+
+    JournalsDict = {}
+
+    user_id = request.user.id
+    document_type_id = cache.NodeType['Document'].id
+    documents  = session.query(Node).filter(Node.user_id == user_id , Node.parent_id==corpus_id , Node.type_id == document_type_id ).all()
+    for doc in documents:
+        if "journal" in doc.hyperdata:
+            journal = doc.hyperdata["journal"]
+            if journal not in JournalsDict:
+                JournalsDict [journal] = 0
+            JournalsDict[journal] += 1
+    return JsonHttpResponse(JournalsDict)
 
 def test_ngrams(request , project_id, corpus_id ):
     results = ["hola" , "mundo"]
@@ -94,6 +223,17 @@ def test_ngrams(request , project_id, corpus_id ):
     whitelist_type_id = cache.NodeType['WhiteList'].id
     document_type_id = cache.NodeType['Document'].id
 
+    corpus_id = int(corpus_id)
+    lists = dict()
+    for list_type in ['StopList']:
+        list_id = list()
+        list_id = listIds(user_id=request.user.id, corpus_id=int(corpus_id), typeList=list_type)
+        lists["%s" % list_id[0][0]] = list_type
+    doc_ngram_list = listNgramIds(corpus_id=corpus_id, list_id=list_id[0][0], doc_id=list_id[0][0], user_id=request.user.id)
+    StopList = {}
+    for n in doc_ngram_list:
+        StopList[ n[0] ] = True
+
     # # 13099  clinical benefits
     # # 7492   recent data
     # # 14279  brain development
@@ -145,6 +285,7 @@ def test_ngrams(request , project_id, corpus_id ):
     for doc in documents:
         NgramOccs = session.query(Node_Ngram).filter( Node_Ngram.node_id==doc.id).all()
         for ngram in NgramOccs:
+            if ngram.ngram_id not in StopList:
                 if ngram.ngram_id not in Ngrams_Scores:
                     Ngrams_Scores[ngram.ngram_id] = {} 
                     Ngrams_Scores[ngram.ngram_id]["scores"] = {
@@ -177,6 +318,7 @@ def test_ngrams(request , project_id, corpus_id ):
     Sum = 0
     NgramTFIDF = session.query(NodeNodeNgram).filter( NodeNodeNgram.nodex_id==corpus_id ).all()
     for ngram in NgramTFIDF:
+        if ngram.ngram_id not in StopList:
             Ngrams_Scores[ngram.ngram_id]["scores"]["tfidf_sum"] += ngram.score
             Sum += Ngrams_Scores[ngram.ngram_id]["scores"]["occ_uniq"]
             # print( "docid:", ngram.nodey_id , ngram.ngram_id  , ngram.score)
@@ -210,6 +352,7 @@ def test_ngrams(request , project_id, corpus_id ):
     query = session.query(Ngram).filter(Ngram.id.in_( ngrams_ids ))
     ngrams_data = query.all()
     for ngram in ngrams_data:
+        if ngram.id not in StopList:
             occ_uniq = Ngrams_Scores[ngram.id]["scores"]["occ_uniq"]
             if occ_uniq > occs_threshold:
                 Ngrams_Scores[ngram.id]["name"] = ngram.terms