Merge branch 'testing' into stable-merge

CHANGELOG.md

-
-## Version 3.0.6.2
+## TODO
 * Guided Tour
-
-## Version 3.0.6.1
 * Sources form highlighting crawlers
+
+## Version 3.0.6.2
+* SQL Table change. Update:
+    * psql gargandb
+    * drop table contacts
+    * ./manage.py dbmigrate.py
+* Init accounts with ISC-PIF partner group
 * Link to Licence
 
 ## Version 3.0.6

gargantext/models/nodes.py

@@ -6,7 +6,6 @@ from datetime import datetime
 
 from .users import User
 
-
 __all__ = ['Node', 'NodeNode']
 
 class NodeType(TypeDecorator):

gargantext/models/users.py

@@ -3,11 +3,9 @@ from gargantext.util.db import *
 
 from datetime import datetime
 
-__all__ = ['User']
-
+__all__ = ['User', 'Contact']
 
 class User(Base):
-
     # The properties below are a reflection of Django's auth module's models.
     __tablename__ = models.User._meta.db_table
     id           = Column(Integer, primary_key=True)
@@ -48,6 +46,7 @@ class User(Base):
         return query
 
     def contacts_nodes(self, typename=None):
+        from .nodes import Node
         for contact in self.contacts():
             contact_nodes = (session
                 .query(Node)
@@ -61,15 +60,16 @@ class User(Base):
         """check if a given node is owned by the user"""
         return (node.user_id == self.id) or \
                 node.id in (contact.id for contact in self.contacts())
+    
     def get_params(self, username=None):
         print(self.__dict__.items())
         return self.hyperdata
 
 class Contact(Base):
     __tablename__ = 'contacts'
-    id = Column(Integer, primary_key=True)
-    user1_id = Column(Integer, primary_key=True)
-    user2_id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    user1_id = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
+    user2_id = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
     is_blocked = Column(Boolean(), default=False)
     date_creation = Column(DateTime(timezone=False))
 

gargantext/util/db.py

@@ -2,7 +2,9 @@ from gargantext           import settings
 from gargantext.util.json import json_dumps
 
 
+########################################################################
 # get engine, session, etc.
+########################################################################
 from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import delete
@@ -25,21 +27,24 @@ Base = declarative_base()
 session = scoped_session(sessionmaker(bind=engine))
 
 
+########################################################################
 # tools to build models
-
+########################################################################
 from sqlalchemy.types import *
 from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint
 from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 Double = DOUBLE_PRECISION
 
+########################################################################
 # useful for queries
-
+########################################################################
 from sqlalchemy.orm import aliased
 from sqlalchemy import func, desc
 
+########################################################################
 # bulk insertions
-
+########################################################################
 import psycopg2
 
 def get_cursor():
@@ -54,7 +59,6 @@ def get_cursor():
     return db, db.cursor()
 
 class bulk_insert:
-
     def __init__(self, table, fields, data, cursor=None):
         # prepare the iterator
         self.iter = iter(data)
@@ -85,7 +89,6 @@ class bulk_insert:
 
     readline = read
 
-
 def bulk_insert_ifnotexists(model, uniquekey, fields, data, cursor=None, do_stats=False):
     """
     Inserts bulk data with an intermediate check on a uniquekey

gargantext/util/ngramlists_tools.py

@@ -16,6 +16,7 @@ from gargantext.models           import Ngram, NodeNgram, NodeNodeNgram, \
 from gargantext.util.lists       import UnweightedList, Translations
 
 from gargantext.constants        import DEFAULT_CSV_DELIM, DEFAULT_CSV_DELIM_GROUP
+from gargantext.util.toolchain.mail_notification import notify_listMerged
 
 # import will implement the same text cleaning procedures as toolchain
 from gargantext.util.toolchain.parsing           import normalize_chars
@@ -30,6 +31,7 @@ from csv                 import writer, reader, QUOTE_MINIMAL
 from collections         import defaultdict
 from re                  import match, findall
 from io                  import StringIO # pseudo file to write CSV to memory
+from celery              import shared_task
 
 def query_list(list_id,
                 pagination_limit=None, pagination_offset=None,
@@ -127,7 +129,6 @@ def query_list(list_id,
 
     return query
 
-
 # helper func for exports
 def ngrams_to_csv_rows(ngram_objs, ngram_dico={}, group_infos={},
                         list_type="", groupings_delim=DEFAULT_CSV_DELIM_GROUP):
@@ -191,8 +192,6 @@ def ngrams_to_csv_rows(ngram_objs, ngram_dico={}, group_infos={},
 
     return csv_rows
 
-
-
 def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
     """
     export of the 3 lists under a corpus node (MAP, MAIN, STOP)
@@ -325,8 +324,6 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
         print("EXPORT: wrote %i ngrams to CSV file '%s'"
                % (len(this_corpus_all_rows), path.abspath(fname)))
 
-
-
 def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
                              group_delimiter=DEFAULT_CSV_DELIM_GROUP):
     '''
@@ -614,7 +611,6 @@ def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
     # print("IMPORT RESULT", result)
     return result
 
-
 def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
     """
     Integrates an external terms table to the current one:
@@ -648,6 +644,7 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
     NB: Uses group_tools.group_union() to merge the synonym links.
         Uses ngrams_addition.index_new_ngrams() to also add new ngrams to the docs
     """
+    print("Merging Ngram Lists")
     # log to send back to client-side (lines will be joined)
     my_log = []
 
@@ -836,12 +833,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
         print(msg)
 
     # return a log
+    notify_listMerged(onto_corpus)
     return("\n".join(my_log))
 
+
+@shared_task
 def import_and_merge_ngramlists(file_contents, onto_corpus_id):
     """
     A single function to run import_ngramlists and merge_ngramlists together
     """
+    print("import list")
     new_lists = import_ngramlists(file_contents)
 
     corpus_node = session.query(Node).filter(Node.id == onto_corpus_id).first()

gargantext/util/toolchain/mail_notification.py

@@ -4,10 +4,10 @@ from gargantext.util.db      import session
 from django.core.mail        import send_mail
 from gargantext.settings     import BASE_URL
 
-def notify_owner(corpus):
-    user = session.query(User).filter(User.id == corpus.user_id).first()
 
-    message = '''
+
+drafts = { 
+        'workflowEnd' : '''
     Bonjour,
     votre analyse sur Gargantext vient de se terminer.
 
@@ -22,15 +22,52 @@ def notify_owner(corpus):
     --
         L'équipe de Gargantext (CNRS)
 
-    ''' % (corpus.name, BASE_URL, corpus.parent_id, corpus.id)
+    ''',
+
+
+        'listMerged' : '''
+    Bonjour,
+    votre liste est mergée.
+
+    Vous pouvez accéder à votre corpus intitulé
+        \"%s\"
+    à l'adresse:
+
+    http://%s/projects/%d/corpora/%d
+
+    Nous restons à votre disposition pour tout complément d'information.
+    Cordialement
+    --
+        L'équipe de Gargantext (CNRS)
+
+    ''',
+
+
+        }
+
+
+
+def notification(corpus,draft):
+    user = session.query(User).filter(User.id == corpus.user_id).first()
+
+    message = draft % (corpus.name, BASE_URL, corpus.parent_id, corpus.id)
     
     if user.email != "" :
-        send_mail('[Gargantext] Votre analyse est terminée'
+        send_mail('[Gargantext] Update'
                  , message
-                 , 'team@gargantext.org'
+                 , 'contact@gargantext.org'
                  , [user.email], fail_silently=False )
+        print("Email sent")
     else:
         print("User %s (%d), has no email" % (user.username, user.id) )
 
 
+def notify_owner(corpus):
+    notification(corpus, drafts['workflowEnd'])
+
+
+def notify_listMerged(corpus):
+    notification(corpus, drafts['listMerged'])
+
+
 

gargantext/util/toolchain/main.py

@@ -182,7 +182,6 @@ def parse_extract_indexhyperdata(corpus):
     corpus.save_hyperdata()
     session.commit()
 
-
 @shared_task
 def recount(corpus):
     """
@@ -284,9 +283,5 @@ def recount(corpus):
     corpus.save_hyperdata()
     session.commit()
 
-
-
-
-
 def t():
     return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")

gargantext/views/api/ngramlists.py

@@ -90,8 +90,10 @@ class CSVLists(APIView):
         # import the csv
         # try:
         log_msg = "Async generation"
-        scheduled(import_and_merge_ngramlists)(csv_contents,
-                                              onto_corpus_id = corpus_node.id)
+        
+        corpus_node_id = corpus_node.id
+        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id)
+        
         return JsonHttpResponse({
             'log': log_msg,
             }, 200)

static/lib/graphExplorer/tinawebJS/methods.js

@@ -303,7 +303,7 @@ function updateLeftPanel_fix( sels , oppos ) {
     var ngram_actions = ''
     if(partialGraph.states[1].categories.length==1) {
         ngram_actions += '<center>'
-        ngram_actions += '<span><button class="delete" onclick="modify_ngrams(this.className)">Delete</button></span>'
+        ngram_actions += '<span><button class="delete" onclick="modify_ngrams(this.className)">Remove</button></span>'
         // if( Object.keys(sels).length>1)
         //     ngram_actions += '<span><button class="group" onclick="modify_ngrams(this.className)">Group</button></span>'
         ngram_actions += '</center>'

templates/pages/corpora/terms.html

@@ -434,7 +434,7 @@ function listmergeCsvPost(theFile){
                xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
              },
              success: function(response) {
-                 my_html  = '<h3 style="color:green">IMPORT OK</h3>'
+                 my_html  = '<h3 style="color:green">File upload, you will receive a notification email</h3>'
                  my_html += "<p class='note'>" + response['log'].replace(/\n/g, '<br/>') + "</p>"
                  my_html += "<p'>(this page will reload in 3s)</p>"
                  $('#formanswer').html(my_html);

templates/pages/menu.html

@@ -367,7 +367,7 @@
             <p>
                 Gargantext
                 <span class="glyphicon glyphicon-registration-mark" aria-hidden="true"></span>
-                , version 3.0.6,
+                , version 3.0.6.2,
                 <a href="http://www.cnrs.fr" target="blank" title="Institution that enables this project.">
                     Copyrights
                     <span class="glyphicon glyphicon-copyright-mark" aria-hidden="true"></span>