Merge branch 'refactoring' into refactoring-alex

TODO.md

+# API
+
+Be more careful about authorizations.
+
+# Constants
+
+Remove "magic numbers" (such as 4096, etc.) from the code and put them in
+`constants.py`.
+
 # Projects
 
 ## Overview of all projects

gargantext/models/nodes.py

@@ -89,14 +89,30 @@ class Node(Base):
             {'type': type, 'path':path, 'url':url, 'extracted': False}
         ))
 
-    def status(self, action=None, progress=None):
-        if 'status' not in self.hyperdata:
-            self['status'] = MutableDict(
-                {'action': action, 'progress': progress}
-            )
-        else:
-            if action is not None:
-                self['status']['action'] = action
-            if progress is not None:
-                self['status']['progress'] = progress
-        return self['status']
+    def status(self, action=None, progress=None, complete=False):
+        """Get the status of the given action
+        """
+        # if the hyperdata do not have data about status
+        if 'statuses' not in self.hyperdata:
+            self['statuses'] = MutableList()
+        # if no action name is given, return the last appended status
+        if action is None:
+            for status in self['statuses']:
+                if not status['complete']:
+                    return status
+            if len(self['statuses']):
+                return self['statuses'][-1]
+            return None
+        # retrieve the status concerning by the given action name
+        for status in self['statuses']:
+            if status['action'] == action:
+                if progress is not None:
+                    status['progress'] = progress
+                if complete:
+                    status['complete'] = complete
+                return status
+        # if no status has been found for the action, append a new one
+        self['statuses'].append(MutableDict(
+            {'action': action, 'progress': progress, 'complete': complete}
+        ))
+        return self['statuses'][-1]

gargantext/util/toolchain/ngrams_extraction.py

@@ -44,7 +44,7 @@ def extract_ngrams(corpus, rule='{<JJ.*>*<NN.*>+<JJ.*>*}', keys=('title', 'abstr
     resource_type_index = corpus.resources()[0]['type']
     resource_type = RESOURCETYPES[resource_type_index]
     default_language_iso2 = resource_type['default_language']
-    for document in corpus.children('DOCUMENT'):
+    for documents_count, document in enumerate(corpus.children('DOCUMENT')):
         # get ngrams extractor for the current document
         language_iso2 = document.hyperdata.get('language_iso2', default_language_iso2)
         try:
@@ -68,5 +68,12 @@ def extract_ngrams(corpus, rule='{<JJ.*>*<NN.*>+<JJ.*>*}', keys=('title', 'abstr
             _integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor)
             nodes_ngrams_count.clear()
             ngrams_data.clear()
+        if documents_count % 1024 == 0:
+            corpus.status('ngrams_extraction', progress=documents_count+1)
+            corpus.save_hyperdata()
+            session.commit()
     # integrate ngrams and nodes-ngrams
     _integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor)
+    corpus.status('ngrams_extraction', progress=documents_count+1, complete=True)
+    corpus.save_hyperdata()
+    session.commit()

gargantext/util/toolchain/parsing.py

@@ -21,12 +21,13 @@ def parse(corpus):
             )
             session.add(document)
             if documents_count % 64 == 0:
-                corpus.status(action='parsing', progress=documents_count)
+                corpus.status('parsing', progress=documents_count)
                 corpus.save_hyperdata()
+                session.commit()
             documents_count += 1
         # update info about the resource
         resource['extracted'] = True
-        corpus.save_hyperdata()
     # commit all changes
-    corpus.status(action='parsing', progress=documents_count)
+    corpus.status('parsing', progress=documents_count, complete=True)
+    corpus.save_hyperdata()
     session.commit()

gargantext/views/api/nodes.py

@@ -63,6 +63,9 @@ class NodeListResource(APIView):
             'records': [dict(zip(parameters['fields'], node)) for node in query]
         })
 
+    def post(self, request):
+        pass
+
     def delete(self, request):
         """Removes the list of nodes corresponding to the query.
         WARNING! THIS IS TOTALLY UNTESTED!!!!!
@@ -76,6 +79,7 @@ class NodeListResource(APIView):
             'count': count,
         }, 200)
 
+
 class NodeResource(APIView):
 
     def _query(self, request, node_id):
@@ -88,7 +92,6 @@ class NodeResource(APIView):
         return user, node
 
     def get(self, request, node_id):
-        from sqlalchemy import delete
         user, node = self._query(request, node_id)
         return JsonHttpResponse({
             'id': node.id,
@@ -98,6 +101,7 @@ class NodeResource(APIView):
         })
 
     def delete(self, request, node_id):
+        from sqlalchemy import delete
         user, node = self._query(request, node_id)
         result = session.execute(
             delete(Node).where(Node.id == node_id)

gargantext/views/pages/corpora.py

@@ -35,7 +35,7 @@ def corpus(request, project_id, corpus_id):
             'date': datetime.now(),
             'project': project,
             'corpus': corpus,
-            # 'processing': processing,
+            # 'processing': corpus['extracted'],
             # 'number': number,
             'view': 'documents'
         },

gargantext/views/pages/projects.py

@@ -102,10 +102,20 @@ def project(request, project_id):
     sourcename2corpora = defaultdict(list)
     for corpus in corpora:
         # we only consider the first resource of the corpus to determine its type
-        corpus.count = corpus.children('DOCUMENT').count()
         resource = corpus.resources()[0]
-        resource_type = RESOURCETYPES[resource['type']]
-        sourcename2corpora[resource_type['name']].append(corpus)
+        resource_type_name = RESOURCETYPES[resource['type']]['name']
+        # add some data for the viewer
+        corpus.count = corpus.children('DOCUMENT').count()
+        status = corpus.status()
+        if status is not None and not status['complete']:
+            corpus.status_message = '(in progress: %s, %d complete)' % (
+                status['action'].replace('_', ' '),
+                status['progress'],
+            )
+        else:
+            corpus.status_message = ''
+        # add
+        sourcename2corpora[resource_type_name].append(corpus)
     # source & their respective counts
     total_documentscount = 0
     sourcename2documentscount = defaultdict(int)

static/js/gargantext/garganrest.js

+// check if an object is an array
+if (typeof Array.isArray != 'function') {
+    Array.isArray = function(stuff) {
+        return Object.prototype.toString.call(stuff) == '[object Array]';
+    };
+}
+
+
+// CSRF token management for Django
+var getCookie = function(name) {
+    var cookieValue = null;
+    if (document.cookie && document.cookie != '') {
+        var cookies = document.cookie.split(';');
+        for (var i = 0; i < cookies.length; i++) {
+            var cookie = jQuery.trim(cookies[i]);
+            // Does this cookie string begin with the name we want?
+            if (cookie.substring(0, name.length + 1) == (name + '=')) {
+                cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                break;
+            }
+        }
+    }
+    return cookieValue;
+}
+var csrftoken = getCookie('csrftoken');
+$.ajaxSetup({
+    beforeSend: function(xhr, settings) {
+        xhr.setRequestHeader("X-CSRFToken", csrftoken);
+    }
+});
+
+
+// Resource class
+var Resource = function(url_path) {
+
+    // retrieve one or many items
+    this.get = function(criteria, callback) {
+        var url = url_path;
+        switch (typeof criteria) {
+            // get the list, according to the criteria passed as parameters
+            case 'object':
+                var url_parameters = '';
+                $.each(criteria, function(key, value) {
+                    if (Array.isArray(value)) {
+                        $.each(value, function(i, item) {
+                            url_parameters += url_parameters.length ? '&' : '?';
+                            url_parameters += encodeURIComponent(key) + '[]=' + encodeURIComponent(item);
+                        });
+                    } else {
+                        url_parameters += url_parameters.length ? '&' : '?';
+                        url_parameters += encodeURIComponent(key) + '=' + encodeURIComponent(value);
+                    }
+                });
+                url += url_parameters;
+                break;
+            // get the list, without paramters
+            case 'function':
+                callback = criteria;
+                break;
+            case 'number':
+            case 'string':
+                url += '/' + criteria;
+                break;
+        }
+        $.ajax({
+            url: url,
+            type: 'GET',
+            success: callback
+        });
+    };
+    // change an item
+    this.change = this.update = function(item, callback) {
+        $.ajax({
+            url: url_path + '/' + item.id,
+            type: 'PATCH',
+            success: callback
+        });
+    };
+    // remove an item
+    this.delete = this.remove = function(id, callback) {
+        if (id.id != undefined) {
+            id = id.id;
+        }
+        $.ajax({
+            url: url_path + '/' + id,
+            type: 'DELETE',
+            success: callback
+        });
+    };
+    // add an item
+    this.add = this.append = function(value, callback) {
+        $.ajax({
+            url: url_path + '/' + id,
+            type: 'POST',
+            success: callback
+        });
+    };
+};
+
+var GarganRest = function(base_path, path_list) {
+    var that = this;
+    $.each(path_list, function(p, path){
+        that[path] = new Resource(base_path + path);
+    });
+};
+
+garganrest = new GarganRest('/api/', ['nodes']);
+
+
+// var log = function(result){console.log(result);};
+// garganrest.nodes.get(log);
+// garganrest.nodes.get(167, log);
+// garganrest.nodes.delete(167, log);
+// garganrest.nodes.get({
+//     pagination_offset: 0,
+//     pagination_limit: 10,
+//     type: ['DOCUMENT'],
+//     parent_id: 2,
+// }, log);

templates/pages/projects/project.html

@@ -4,6 +4,7 @@
 {% load staticfiles %}
 <link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
 <script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
+<script type="text/javascript" src="{% static "js/gargantext/garganrest.js" %}"></script>
 <link rel="stylesheet" href="http://code.jquery.com/ui/1.11.2/themes/smoothness/jquery-ui.css">
 
 <script type="text/javascript" src="{% static "js/morris.min.js" %}"></script>
@@ -75,22 +76,27 @@
             {{ key }}
             <ul>
                 {% for corpus in corpora %}
-                <li>
+                <li id="corpus_{{corpus.id}}">
                     {% ifequal corpus.processing 1 %}
                     {{corpus.name}}:
                     <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}">
                     Processing, drink a cup of tea, and refresh the page :)
                     {% else %}
-                    <a href="/projects/{{project.id}}/corpora/{{corpus.id}}">  {{corpus.name}} , {{ corpus.count }} documents</a>
+                    <a href="/projects/{{project.id}}/corpora/{{corpus.id}}">
+                        {{corpus.name}}, {{ corpus.count }} documents {{ corpus.status_message }}
+                    </a>
                     {% endifequal %}
                     <button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom"
-                    data-content='
+                    data-content="
                     <ul>
-                        <li> Rename </li>
-                        <li> Add new documents </li>
-                        <li><a href="/delete/{{corpus.id}}">Delete</a></li>
+                        <li>Rename</li>
+                        <li>Add new documents</li>
+                        <li onclick=&quot;
+                            garganrest.nodes.delete({{corpus.id}}, function(){$('#corpus_'+{{corpus.id}}).remove()});
+                            $(this).parent().parent().remove();
+                        &quot;><a href=&quot;#&quot;>Delete</a></li>
                     </ul>
-                    '>Manage</button>
+                    ">Manage</button>
                 </li>
                 {% endfor %}
             </ul>
@@ -104,7 +110,9 @@
         <h3>
             <a href="/projects/{{project.id}}/corpora/{{corpus.id}}">{{corpus.name}}</a>
         </h3>
-        <h4>{{ corpus.count }} Documents </h4>
+        <h4>
+            {{ corpus.count }} documents
+        </h4>
         <h5>Activity:</h5>
         <div class="chart" data-percent="73">73%</div>
     </div>