FIX TAGERBOT default_lang

gargantext/util/toolchain/ngrams_extraction.py

@@ -52,8 +52,12 @@ def extract_ngrams(corpus, keys=DEFAULT_INDEX_FIELDS, do_subngrams = DEFAULT_IND
         tagger_bots = {lang: load_tagger(lang)() for lang in corpus.languages if lang != "__skipped__"}
         #sort docs by lang?
         # for lang, tagger in tagger_bots.items():
+
         for documents_count, document in enumerate(docs):
             language_iso2 = document.hyperdata.get('language_iso2')
+            if language_iso2 in source["default_languages"]:
+            #filtering out skipped_docsof parsing
+            #if document.id not in corpus.skipped_docs:
                 tagger = tagger_bots[language_iso2]
                 #print(language_iso2)
                 for key in keys:

gargantext/util/toolchain/parsing.py

@@ -146,14 +146,7 @@ def parse(corpus):
                         session.commit()
                         #adding skipped_docs for later processsing
                         skipped_docs.append(document.id)
-
-
-
-
-
                 #documents for this resources
-
-
                 session.add(corpus)
                 session.commit()
                 # update info about the resource
@@ -161,20 +154,27 @@ def parse(corpus):
                 #print( "resource n°",i, ":", d, "docs inside this file")
 
 
-        # add a corpus-level info about languages adding a __skipped__ info
-        print(len(skipped_docs), "docs skipped")
+
+        #skipped_docs
+        corpus.skipped_docs = list(set(skipped_docs))
+        print(len(corpus.skipped_docs), "docs skipped")
+        skipped_langs = dict(Counter(skipped_languages))
+        if len(corpus.skipped_docs) > 0:
+            print ("INFO in which:")
+            print (sum(skipped_langs.values()), "docs with unsupported lang")
+
         print(corpus.children("DOCUMENT").count(), "docs parsed")
-        #main language of the corpus
+        #language of corpus
         print(languages.items())
         corpus.language_id = sorted(languages.items(), key = lambda x: x[1], reverse=True)[0][0]
-        print(corpus.language_id)
-        languages['__skipped__'] = dict(Counter(skipped_languages))
-        corpus.languages = languages
-        corpus.skipped_docs = list(set(skipped_docs))
+        print("Default MAIN language of CORPUS", corpus.language_id)
+        corpus.languages = dict(languages)
+        corpus.languages["__skipped__"] = list(skipped_langs.keys())
+        print("Languages of CORPUS", corpus.languages)
         corpus.save_hyperdata()
         session.commit()
-        if len(corpus.skipped_docs) > 0:
-            print (sum(languages["__skipped__"].values()), "docs with unsupported lang")
+
+
             #assign main lang of the corpus to unsupported languages docs
             # for d_id in corpus.skipped_docs:
             #     document = session.query(Node).filter(Node.id == d_id, Node.typename == "DOCUMENT").first()