[MERGE] Async mode for list merge.

gargantext/util/ngramlists_tools.py

@@ -16,6 +16,7 @@ from gargantext.models           import Ngram, NodeNgram, NodeNodeNgram, \
 from gargantext.util.lists       import UnweightedList, Translations
 
 from gargantext.constants        import DEFAULT_CSV_DELIM, DEFAULT_CSV_DELIM_GROUP
+from gargantext.util.toolchain.mail_notification import notify_listMerged
 
 # import will implement the same text cleaning procedures as toolchain
 from gargantext.util.toolchain.parsing           import normalize_chars
@@ -30,6 +31,7 @@ from csv                 import writer, reader, QUOTE_MINIMAL
 from collections         import defaultdict
 from re                  import match, findall
 from io                  import StringIO # pseudo file to write CSV to memory
+from celery              import shared_task
 
 def query_list(list_id,
                 pagination_limit=None, pagination_offset=None,
@@ -127,7 +129,6 @@ def query_list(list_id,
 
     return query
 
-
 # helper func for exports
 def ngrams_to_csv_rows(ngram_objs, ngram_dico={}, group_infos={},
                         list_type="", groupings_delim=DEFAULT_CSV_DELIM_GROUP):
@@ -191,8 +192,6 @@ def ngrams_to_csv_rows(ngram_objs, ngram_dico={}, group_infos={},
 
     return csv_rows
 
-
-
 def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
     """
     export of the 3 lists under a corpus node (MAP, MAIN, STOP)
@@ -325,8 +324,6 @@ def export_ngramlists(node,fname=None,delimiter=DEFAULT_CSV_DELIM,titles=True):
         print("EXPORT: wrote %i ngrams to CSV file '%s'"
                % (len(this_corpus_all_rows), path.abspath(fname)))
 
-
-
 def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
                              group_delimiter=DEFAULT_CSV_DELIM_GROUP):
     '''
@@ -614,7 +611,6 @@ def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
     # print("IMPORT RESULT", result)
     return result
 
-
 def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
     """
     Integrates an external terms table to the current one:
@@ -648,6 +644,7 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
     NB: Uses group_tools.group_union() to merge the synonym links.
         Uses ngrams_addition.index_new_ngrams() to also add new ngrams to the docs
     """
+    print("Merging Ngram Lists")
     # log to send back to client-side (lines will be joined)
     my_log = []
 
@@ -836,12 +833,16 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
         print(msg)
 
     # return a log
+    notify_listMerged(onto_corpus)
     return("\n".join(my_log))
 
+
+@shared_task
 def import_and_merge_ngramlists(file_contents, onto_corpus_id):
     """
     A single function to run import_ngramlists and merge_ngramlists together
     """
+    print("import list")
     new_lists = import_ngramlists(file_contents)
 
     corpus_node = session.query(Node).filter(Node.id == onto_corpus_id).first()

gargantext/util/toolchain/mail_notification.py

@@ -4,10 +4,10 @@ from gargantext.util.db      import session
 from django.core.mail        import send_mail
 from gargantext.settings     import BASE_URL
 
-def notify_owner(corpus):
-    user = session.query(User).filter(User.id == corpus.user_id).first()
 
-    message = '''
+
+drafts = { 
+        'workflowEnd' : '''
     Bonjour,
     votre analyse sur Gargantext vient de se terminer.
 
@@ -22,15 +22,52 @@ def notify_owner(corpus):
     --
         L'équipe de Gargantext (CNRS)
 
-    ''' % (corpus.name, BASE_URL, corpus.parent_id, corpus.id)
+    ''',
+
+
+        'listMerged' : '''
+    Bonjour,
+    votre liste est mergée.
+
+    Vous pouvez accéder à votre corpus intitulé
+        \"%s\"
+    à l'adresse:
+
+    http://%s/projects/%d/corpora/%d
+
+    Nous restons à votre disposition pour tout complément d'information.
+    Cordialement
+    --
+        L'équipe de Gargantext (CNRS)
+
+    ''',
+
+
+        }
+
+
+
+def notification(corpus,draft):
+    user = session.query(User).filter(User.id == corpus.user_id).first()
+
+    message = draft % (corpus.name, BASE_URL, corpus.parent_id, corpus.id)
     
     if user.email != "" :
-        send_mail('[Gargantext] Votre analyse est terminée'
+        send_mail('[Gargantext] Update'
                  , message
-                 , 'team@gargantext.org'
+                 , 'contact@gargantext.org'
                  , [user.email], fail_silently=False )
+        print("Email sent")
     else:
         print("User %s (%d), has no email" % (user.username, user.id) )
 
 
+def notify_owner(corpus):
+    notification(corpus, drafts['workflowEnd'])
+
+
+def notify_listMerged(corpus):
+    notification(corpus, drafts['listMerged'])
+
+
 

gargantext/util/toolchain/main.py

@@ -182,7 +182,6 @@ def parse_extract_indexhyperdata(corpus):
     corpus.save_hyperdata()
     session.commit()
 
-
 @shared_task
 def recount(corpus):
     """
@@ -284,9 +283,5 @@ def recount(corpus):
     corpus.save_hyperdata()
     session.commit()
 
-
-
-
-
 def t():
     return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")

gargantext/views/api/ngramlists.py

@@ -90,8 +90,10 @@ class CSVLists(APIView):
         # import the csv
         # try:
         log_msg = "Async generation"
-        scheduled(import_and_merge_ngramlists)(csv_contents,
-                                              onto_corpus_id = corpus_node.id)
+        
+        corpus_node_id = corpus_node.id
+        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id)
+        
         return JsonHttpResponse({
             'log': log_msg,
             }, 200)

templates/pages/corpora/terms.html

@@ -434,7 +434,7 @@ function listmergeCsvPost(theFile){
                xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
              },
              success: function(response) {
-                 my_html  = '<h3 style="color:green">IMPORT OK</h3>'
+                 my_html  = '<h3 style="color:green">File upload, you will receive a notification email</h3>'
                  my_html += "<p class='note'>" + response['log'].replace(/\n/g, '<br/>') + "</p>"
                  my_html += "<p'>(this page will reload in 3s)</p>"
                  $('#formanswer').html(my_html);