Merge branch 'samuel' into merge

gargantext_web/urls.py

@@ -54,9 +54,9 @@ urlpatterns = patterns('',
     url(r'^project/(\d+)/corpus/(\d+)/journals/journals.json$', corpus_views.test_journals),
     url(r'^project/(\d+)/corpus/(\d+)/journals', corpus_views.get_journals),
 
-    # Terms view
-    url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', corpus_views.test_ngrams),
-    url(r'^project/(\d+)/corpus/(\d+)/terms', corpus_views.get_ngrams),
+    # # Terms view
+    # url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', corpus_views.test_ngrams),
+    # url(r'^project/(\d+)/corpus/(\d+)/terms', corpus_views.get_ngrams),
 
     # Update corpus
     url(r'^project/(\d+)/corpus/(\d+)/(\w+)/update$', views.update),
@@ -108,7 +108,9 @@ urlpatterns = patterns('',
     url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX),
     url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
     url(r'^tests/move2trash/$' , views.move_to_trash_multiple ),
-    url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test)
+    url(r'^corpus/(\d+)/document/(\d+)/testpage$', samtest.test_test),
+    url(r'^project/(\d+)/corpus/(\d+)/terms/ngrams.json$', samtest.test_ngrams),
+    url(r'^project/(\d+)/corpus/(\d+)/terms', samtest.get_ngrams)
 )
 
 

parsing/FileParsers/ISTex.py

@@ -19,7 +19,7 @@ class ISTex(FileParser):
             "source"           : 'corpusName',
             "title"             : 'title',
             "genre"             : "genre",
-            # "language_iso3"     : 'MedlineCitation/Article/Language',
+            "language_iso3"     : 'language',
             "doi"               : 'doi',
             "host"              : 'host',
             "publication_date"  : 'pubdate',
@@ -34,11 +34,13 @@ class ISTex(FileParser):
                 try:
                     # print(path," ==> ",len(json_doc[path]))
                     hyperdata[key] = json_doc[path]
-                except: pass
+                except:
+                    pass
 
-            # print("|",hyperdata["publication_date"])
+            # print("|",hyperdata["language_iso3"])
 
-            if "doi" in hyperdata: hyperdata["doi"] = hyperdata["doi"][0]
+            if "doi" in hyperdata: 
+                hyperdata["doi"] = hyperdata["doi"][0]
             
             keywords = []
             if "keywords" in hyperdata:
@@ -75,6 +77,11 @@ class ISTex(FileParser):
             if "genre" in hyperdata:
                 if len(hyperdata["genre"])==0:
                     hyperdata.pop("genre")
+            if "language_iso3" in hyperdata:
+                if len(hyperdata["language_iso3"])>0:
+                    hyperdata["language_iso3"] = hyperdata["language_iso3"][0]
+                else:
+                    hyperdata["language_iso3"] = "eng"
 
             RealDate = hyperdata["publication_date"]
             if "publication_date" in hyperdata: hyperdata.pop("publication_date")

scrappers/scrap_pubmed/views.py

@@ -66,7 +66,7 @@ def getGlobalStatsISTEXT(request ):
 		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
 		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
 		query_string = query.replace(" ","+")
-		url = "http://api.istex.fr/document/?q="+query_string
+		url = "http://api.istex.fr/document/?q="+query_string+"&output=id,title,abstract,pubdate,corpusName,authors,language"
 
 		tasks = MedlineFetcher()
 

static/js/Docs_dyna_chart_and_table.js

@@ -127,19 +127,9 @@ for(var i=0; i<url_elems.length; i++) {
     url_mainIDs[url_elems[i-1]] = Number(url_elems[i]);
   }
 }
-var theurl = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
-// $.ajax({
-//   url: theurl,
-//   success: function(data) {
-//     bisarray = data.data
-//     for(var i in bisarray) {
-//         untitlebis = bisarray[i].values
-//         BIS_dict[untitlebis[0]] = [bisarray[i].count , 0];// [ total amount , removed ]
-//     }
-//     pr(BIS_dict)
-//     if(Object.keys(BIS_dict).length>0) $("#delAll").css("visibility", "visible"); $("#delAll").show();
-//   }
-// });
+
+
+
 
 
 
@@ -168,7 +158,7 @@ function getRecords() {
 function transformContent2(rec_id) {
   // pr("\t\ttransformContent2: "+rec_id)
   var elem = AjaxRecords[rec_id];
-  // pr("\t\t\t"+elem.date)
+  // pr("\t"+elem.title)
   var result = {}
   if (elem["del"]) {
     result["id"] = elem["id"]
@@ -261,46 +251,39 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
 }
 
 
+function Main_test( Data , SearchFilter ) {
 
-// var div__filter_for_search = ''
-//   div__filter_for_search += '<select data-width="100px" class="selectpicker" multiple  data-max-options="1">';
-//   div__filter_for_search += '  <optgroup label="All" data-max-options="1" selected>';
-//   div__filter_for_search += '    <option>Title</option>';
-//   div__filter_for_search += '    <option>Date</option>';
-//   div__filter_for_search += '  </optgroup>';
-//   div__filter_for_search += '  <optgroup label="Category" data-max-options="1">';
-//   div__filter_for_search += '    <option>Title</option>';
-//   div__filter_for_search += '    <option>Date</option>';
-//   div__filter_for_search += '  </optgroup>';
-//   div__filter_for_search += '  <optgroup label="Duplicates" data-max-options="1">';
-//   div__filter_for_search += '    <option>by DOI</option>';
-//   div__filter_for_search += '    <option>by Title</option>';
-//   div__filter_for_search += '  </optgroup>';
-//   div__filter_for_search += '</select>';
+  var DistributionDict = {}
+  for(var i in DistributionDict)
+    delete DistributionDict[i];
+  delete DistributionDict;
+  DistributionDict = {}
 
-// $("#supmofos").html(div__filter_for_search)
 
-
-// (3) Get records and hyperdata for paginator
-  $.ajax({
-    url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
-    success: function(data){
-      console.log(data)
+  $("#div-table").html("")
+  $("#div-table").empty();
+  var div_table = '<p align="right">'+"\n"
+    div_table += '<table id="my-ajax-table" class="table table-bordered">'+"\n"
+    div_table += "\t"+'<thead>'+"\n"
+    div_table += "\t"+"\t"+'<th width="100px;" data-dynatable-column="date">Date</th>'+"\n"
+    div_table += "\t"+"\t"+'<th data-dynatable-column="name">Title</th>'+"\n"
+    div_table += "\t"+"\t"+'<th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>'+"\n"
+    div_table += "\t"+"\t"+'</th>'+"\n"
+    div_table += "\t"+'</thead>'+"\n"
+    div_table += "\t"+'<tbody>'+"\n"
+    div_table += "\t"+'</tbody>'+"\n"
+    div_table += '</table>'+"\n"
+    div_table += '</p>';
+  $("#div-table").html(div_table)
 
   var justdates = {}
-      for(var i in data.records) {
-        var orig_id = parseInt(data.records[i].id)
-        var arr_id = parseInt(i)
-        RecDict[orig_id] = arr_id;
-        data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
-        data.records[i]["del"] = false
-
-        var date = data.records[i]["date"];  
+  for(var i in Data) {
+    var date = Data[i]["date"];  
     if ( ! justdates[date] ) justdates[date] = 0;
     justdates[date]++;
-        // console.log(data.records[i]["date"]+"  :  originalRecords["+arr_id+"] <- "+orig_id+" | "+data.records[i]["name"])
+    // console.log(Data[i]["date"]+"  :  originalRecords["+arr_id+"] <- "+orig_id+" | "+Data[i]["name"])
   }
-      AjaxRecords = data.records; // backup!!
+
 
   // $("#move2trash").prop('disabled', true);
 
@@ -316,8 +299,8 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
 
 
   var arrayd3 = []
-      for(var e in data.records) {
-          var date = data.records[e]["date"]; 
+  for(var e in Data) {
+      var date = Data[e]["date"]; 
       if(justdates[date]!=false) {
           var info = {}
           info.date = date
@@ -438,13 +421,14 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
 
   dc.renderAll();
 
+  MyTable = []
   MyTable = $('#my-ajax-table').dynatable({
               dataset: {
-                    records: data.records
+                records: Data
               },
               features: {
                 pushState: false,
-                    sort: false //i need to fix the sorting function... the current one just sucks
+                // sort: false //i need to fix the sorting function... the current one just sucks
               },
               writers: {
                 _rowWriter: ulWriter
@@ -452,17 +436,87 @@ function ulWriter(rowIndex, record, columns, cellWriter) {
               }
             });
 
+  MyTable.data('dynatable').paginationPage.set(1);
+  MyTable.data('dynatable').process();
+
   if ( $(".imadiv").length>0 ) return 1;
   $('<br><br><div class="imadiv"></div>').insertAfter(".dynatable-per-page")
   $(".dynatable-record-count").insertAfter(".imadiv")
   $(".dynatable-pagination-links").insertAfter(".imadiv")
   
-      var the_content = $("#supmofos").html();
+  $("#filter_search").html( $("#filter_search").html().replace('selected="selected"') );
+  $("#"+SearchFilter).attr( "selected" , "selected" )
+
+  var the_content = $("#filter_search").html();
   $(""+the_content).insertAfter("#dynatable-query-search-my-ajax-table")
-      $("#supmofos").remove()
   // .insertAfter("#dynatable-query-search-my-ajax-table")
   
+  return "OK"
+}
+
+
+
+function SearchFilters( elem ) {
+  var MODE = elem.value;
+
+  if( MODE == "filter_all") {
+    var result = Main_test(AjaxRecords , MODE)
+    console.log( result )
+  }
+
+  if( MODE == "filter_dupl-titles") {
+
+    var getDupl_API = "/api/nodes/"+url_mainIDs["corpus"]+"/children/duplicates?keys=title&limit=9999"
+    $.ajax({
+      url: getDupl_API,
+      success: function(data) {
+        bisarray = data.data
+        for(var i in bisarray) {
+            titlebis = bisarray[i].values
+            BIS_dict[titlebis[0]] = true;
+        }
+        var Duplicates = []
+        for(var r in AjaxRecords) {
+          if ( BIS_dict[AjaxRecords[r].title] )
+            Duplicates.push( AjaxRecords[r] )
+        }
+        var result = Main_test(Duplicates , MODE)
+        console.log( result )
 
+        MyTable.data('dynatable').sorts.clear();
+        MyTable.data('dynatable').sorts.add('title', 1) // 1=ASCENDING,
+        MyTable.data('dynatable').process();
       }
     });
 
+  }
+
+}
+
+
+
+
+// FIRST portion of code to be EXECUTED:
+// (3) Get records and hyperdata for paginator
+$.ajax({
+  url: '/tests/paginator/corpus/'+url_mainIDs["corpus"],
+  success: function(data){
+
+    for(var i in data.records) {
+      var orig_id = parseInt(data.records[i].id)
+      var arr_id = parseInt(i)
+      RecDict[orig_id] = arr_id;
+      data.records[i]["title"] = data.records[i]["name"];
+      data.records[i]["name"] = '<a target="_blank" href="/project/'+url_mainIDs["project"]+'/corpus/'+ url_mainIDs["corpus"] + '/document/'+orig_id+'">'+data.records[i]["name"]+'</a>'
+      data.records[i]["del"] = false
+    }
+    AjaxRecords = data.records; // backup-ing in global variable!
+
+    var result = Main_test(data.records , "filter_all")
+    console.log( result )
+  },
+});
+
+
+
+

static/js/NGrams_dyna_chart_and_table.js

@@ -370,15 +370,18 @@ $("#Clean_All").click(function(){
 });
 
 $("#Save_All").click(function(){
-
+  console.log("click in save all 01")
   var sum__selected_elems = 0;
   var poubelle = []
-  for(var i in FlagsBuffer)
-    if (Object.keys(FlagsBuffer[i]).length==0) poubelle.push(i)
+  for(var i in FlagsBuffer) {
+    if (Object.keys(FlagsBuffer[i]).length==0) 
+      poubelle.push(i)
     sum__selected_elems += Object.keys(FlagsBuffer[i]).length;
+  }
+  console.log("click in save all 02")
   for(var i in poubelle)
     delete FlagsBuffer[poubelle[i]];
-
+  console.log("click in save all 03, sum:"+sum__selected_elems)
   if ( sum__selected_elems>0 ) {
     console.log("")
     console.log("Do the ajax conexion with API and send this array to be processed:")
@@ -498,11 +501,6 @@ function Main_test( data , initial) {
     oldest = Number(min_occ);
     latest = Number(max_occ);
 
-
-
-
-
-
     var ndx = false;
     ndx = crossfilter();
     ndx.add(DistributionList);

templates/corpus/documents.html

@@ -4,7 +4,7 @@
 {% load staticfiles %}
 
 <link rel="stylesheet" type="text/css" href="{% static "css/bootstrap.css" %}">
-<link rel="stylesheet" type="text/css" href="{% static "js/bootstrap/bootstrap-select.min.css" %}">
+<!-- here goes import stylesheet js/bootstrap/bootstrap-multiselect.css, mais ca marche pas-->
 
 <link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
 <link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
@@ -72,6 +72,7 @@ th a {
 
 
 
+<input type="hidden" id="list_id" value="{{ list_id }}"></input>
 
 <div class="container">
         <div class="jumbotron">
@@ -87,18 +88,7 @@ th a {
     </div>
     <div id="collapseOne" class="panel-collapse collapse no-transition" role="tabpanel">
       <div class="panel-body">
-                <p align="right">
-                    <table id="my-ajax-table" class="table table-bordered">
-                      <thead>
-                        <!-- <th data-dynatable-column="id">ID</th> -->
-                        <th width="100px;" data-dynatable-column="date">Date</th>
-                        <th data-dynatable-column="name">Title</th>
-                        <th data-dynatable-column="del" data-dynatable-no-sort="true">Trash</th>
-                      </thead>
-                      <tbody>
-                      </tbody>
-                    </table>
-                </p>
+                  <div id="div-table"></div>
                 <p align="right">
                     <button id="move2trash" class="btn btn-primary btn-lg" >Trash It!</button>
                 </p>
@@ -111,24 +101,70 @@ th a {
 </div>
 </div>
 
-<div id="supmofos">
+<div class="container">
+		<div class="row">
+				<div class="col-md-4">
+						<div class="jumbotron">
+						<h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/chart">Advanced charts</a></h3>
+						<ol>
+								<li>Count</li> <!-- read, compute -->
+								<li>Filter</li> <!-- count, compute -->
+								<li>Compare</li> <!-- select, cut -->
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+					</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+                            <!-- <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}/matrix">Matrix</a></h3> -->
+								<h3>Matrix (soon)</h3>
+						<ol>
+								<li>Sort</li>
+								<li>Group</li>
+								<li>Cluster</li>
+						</ol>
+						<h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+						</div>
+				</div>
+
+				<div class="col-md-4">
+						<div class="jumbotron">
+								{% if processing > 0 %}
+                <h3> <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Graph (later)</h3>
+                {% else %}
+                                <h3><a href="/project/{{project.id}}/corpus/{{ corpus.id }}/explorer">Graph</a></h3>
+                {% endif %}
+                        <ol>
+                                <li>Visualize</li>
+                                <li>Explore</li>
+                                <li>Read</li>
+                        </ol>
+                        <h4><a href="/project/{{project.id}}/corpus/{{corpus.id}}/">Back to corpus</a></h3>
+                        </div>
+                </div>
+        </div>
+</div>
 
 
-  <select data-width="100px" dir="ltr" class="selectpicker">
-    <option selected>All</option>
-    <option>Title</option>
-    <option>Date</option>
-    <optgroup label="Duplicates">
-      <option>by DOI</option>
-      <option>by Title</option>
-    </optgroup>
-  </select>
+<div id="filter_search" style="visibility:hidden">
+<select id="example-single-optgroups" onchange="SearchFilters(this);">
+    <!-- <optgroup label=""> -->
+        <option id="filter_all" value="filter_all">All</option>
+        <!-- <option id="filter_title" value="filter_title">Title</option> -->
+        <!-- <option id="filter_date" value="filter_date">Date</option> -->
+    <!-- </optgroup> -->
+    <!-- <optgroup label="Duplicates"> -->
+        <!-- <option value="filter_doi">By DOI</option> -->
+        <option id="filter_dupl-titles" value="filter_dupl-titles">Duplicates by Title</option>
+    <!-- </optgroup> -->
+</select>
 
 </div>
 
 <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
 <script src="{% static "js/charts/bootstrap.min.js" %}"></script>
-<script type="text/javascript" src="{% static "js/bootstrap/bootstrap-select.min.js" %}"></script>
+<!-- here goes import script js/bootstrap/bootstrap-multiselect.js, mais ca marche pas-->
 <script type="text/javascript" src="{% static "js/jquery/jquery.dynatable.js" %}"></script>
 
 <!-- custom-lib for dynatable.js and dc.js -->

templates/corpus/journals.html

@@ -110,6 +110,8 @@ input[type=radio]:checked + label {
 {% endblock %}
 
 
+<input type="hidden" id="list_id" value="{{ list_id }}"></input>
+
 {% block content %}
                 <div class="container">
                         <div class="container">

templates/corpus/terms.html

@@ -114,6 +114,8 @@ input[type=radio]:checked + label {
 
 {% block content %}
 
+<input type="hidden" id="list_id" value="{{ list_id }}"></input>
+
                 <div class="container">
                         <div class="container">
 

tests/ngramstable/views.py

@@ -100,7 +100,7 @@ def get_ngrams(request , project_id , corpus_id ):
     except ValueError:
         raise Http404()
 
-    t = get_template('tests/ngrams.html')
+    t = get_template('corpus/terms.html')
     
     user = cache.User[request.user.username].id
     date = datetime.datetime.now()
@@ -173,7 +173,7 @@ def get_journals(request , project_id , corpus_id ):
     except ValueError:
         raise Http404()
 
-    t = get_template('tests/journals.html')
+    t = get_template('corpus/journals.html')
     
     user = cache.User[request.user.username].id
     date = datetime.datetime.now()