rewrite corpus.skipped_docs and corpus.languages => corpus.hyperdata + move...

rewrite corpus.skipped_docs and corpus.languages => corpus.hyperdata + move skipped languages from parsing.py to ngrams_extraction + test exceptions in ngrams_extraction

gargantext/util/parsers/_Parser.py

@@ -122,7 +122,8 @@ class Parser:
             if language_key in hyperdata:
                 try:
                     language_symbol = hyperdata[language_key]
-                    language = languages[language_symbol]
+                    if language_symbol is not None:
+                        language = languages[language_symbol]
                     if language:
                         break
                 except KeyError:

gargantext/util/toolchain/ngram_groups.py

@@ -26,7 +26,7 @@ def prepare_stemmers(corpus):
          and formatted
     """
     stemmers = {lang:SnowballStemmer(languages[lang].name.lower())  for lang \
-                    in corpus.languages.keys() if lang !="__skipped__"}
+                    in corpus.hyperdata['languages'].keys() if lang !="__skipped__"}
     stemmers['__unknown__'] = SnowballStemmer("english")
     return stemmers
 
@@ -56,7 +56,7 @@ def compute_groups(corpus, stoplist_id = None, overwrite_id = None):
 
     # preloop per doc to sort ngrams by language
     for doc in corpus.children('DOCUMENT'):
-        if doc.id not in corpus.skipped_docs:
+        if doc.id not in corpus.hyperdata['skipped_docs']:
             if ('language_iso2' in doc.hyperdata):
                 lgid = doc.hyperdata['language_iso2']
             else:

gargantext/util/toolchain/ngrams_extraction.py

@@ -47,29 +47,60 @@ def extract_ngrams(corpus, keys=DEFAULT_INDEX_FIELDS, do_subngrams = DEFAULT_IND
         resource = corpus.resources()[0]
         documents_count = 0
         source = get_resource(resource["type"])
-        #load available taggers for source default langage
-        docs = [doc for doc in corpus.children('DOCUMENT') if doc.id not in corpus.skipped_docs]
-        tagger_bots = {lang: load_tagger(lang)() for lang in corpus.languages if lang != "__skipped__"}
-        #sort docs by lang?
-        # for lang, tagger in tagger_bots.items():
+
+        # preload available taggers for corpus languages
+        tagger_bots = {}
+        skipped_languages = {}
+
+        for lang in corpus.hyperdata['languages']:
+            try:
+                tagger_bots[lang] = load_tagger(lang)()
+            except KeyError:
+                skipped_languages[lang] = True
+                print("WARNING skipping language:", lang)
+
+        # the list of todo docs
+        docs = [doc for doc in corpus.children('DOCUMENT') if doc.id not in corpus.hyperdata['skipped_docs']]
+
+        # go for the loop
         for documents_count, document in enumerate(docs):
+
             language_iso2 = document.hyperdata.get('language_iso2')
-            tagger = tagger_bots[language_iso2]
             #print(language_iso2)
+
+            # skip case if no tagger available
+            if language_iso2 in skipped_languages:
+                corpus.hyperdata['skipped_docs'][document.id] = True
+                corpus.save_hyperdata()
+                document.hyperdata["error"] = "Error: unsupported language"
+                document.save_hyperdata()
+                session.commit()
+                continue
+
+            # NORMAL CASE
+            tagger = tagger_bots[language_iso2]
             for key in keys:
+                key = str(key)
+                if key not in document.hyperdata:
+                    # print("DBG missing key in doc", key)
+                    # TODO test if document has no keys at all
+                    continue
+
+                # get a text value
+                value = document[key]
+
+                if not isinstance(value, str):
+                    print("DBG wrong content in doc for key", key)
+                    continue
+
                 try:
-                    value = document[str(key)]
-                    if not isinstance(value, str):
-                        continue
-                        # get ngrams
-                    for ngram in tagger.extract(value):
+                    # get ngrams
+                    ngrams = tagger.extract(value)
+                    for ngram in ngrams:
                         tokens = tuple(normalize_forms(token[0]) for token in ngram)
                         if do_subngrams:
                             # ex tokens = ["very", "cool", "exemple"]
-                            #    subterms = [['very', 'cool'],
-                            #                ['very', 'cool', 'exemple'],
-                            #                ['cool', 'exemple']]
-
+                            #    subterms = [['very', 'cool'],...]
                             subterms = subsequences(tokens)
                         else:
                             subterms = [tokens]
@@ -81,13 +112,11 @@ def extract_ngrams(corpus, keys=DEFAULT_INDEX_FIELDS, do_subngrams = DEFAULT_IND
                                 nodes_ngrams_count[(document.id, ngram)] += 1
                                 # add fields :   terms          n
                                 ngrams_data.add((ngram[:255], len(seqterm), ))
-                except:
-                    #value not in doc
+                except Exception as e:
+                    print('NGRAMS EXTRACTION skipping doc %i because of unknown error:' % document.id, str(e))
+                    # TODO add info to document.hyperdata['error']
                     pass
-            # except AttributeError:
-            #     print("ERROR NO language_iso2")
-            #     document.status("NGRAMS", error="No lang detected skipped Ngrams")
-            #     corpus.skipped_docs.append(document.id)
+
         # integrate ngrams and nodes-ngrams
         if len(nodes_ngrams_count) >= BATCH_NGRAMSEXTRACTION_SIZE:
             _integrate_associations(nodes_ngrams_count, ngrams_data, db, cursor)
@@ -105,9 +134,13 @@ def extract_ngrams(corpus, keys=DEFAULT_INDEX_FIELDS, do_subngrams = DEFAULT_IND
             nodes_ngrams_count.clear()
             ngrams_data.clear()
 
+        corpus.hyperdata['skipped_languages'] = skipped_languages
+        corpus.save_hyperdata()
+
         corpus.status('Ngrams', progress=documents_count+1, complete=True)
         corpus.save_hyperdata()
         session.commit()
+
     except Exception as error:
         corpus.status('Ngrams', error=error)
         corpus.save_hyperdata()

gargantext/util/toolchain/parsing.py

@@ -14,6 +14,10 @@ def parse(corpus):
         # shortcut to hyperdata's list of added resources (packs of docs)
         resources = corpus.resources()
 
+        # vars to gather some infos during parsing (=> will end up in hyperdata)
+        skipped_docs = defaultdict(bool)
+        observed_languages = defaultdict(int)
+
         # each resource contains a path to a file with the docs
         for i, resource in enumerate(resources):
 
@@ -28,14 +32,7 @@ def parse(corpus):
                 #corpus.status(error)
                 raise ValueError("Resource '%s' has no Parser" %resource["name"])
             else:
-                #observed langages in corpus docs
-                corpus.languages = defaultdict.fromkeys(source_infos["default_languages"], 0)
-                #remember the skipped docs in parsing
-                skipped_languages = []
-                corpus.skipped_docs = []
-                session.add(corpus)
-                session.commit()
-                #load the corresponding parser
+                # load the corresponding parser
                 parserbot = load_parser(source_infos)
                 # extract and insert documents from resource.path into database
                 default_lang_field = ["language_"+l for l in ["iso2", "iso3", "full_name"]]
@@ -49,14 +46,10 @@ def parse(corpus):
                             except Exception as error :
                                 hyperdata["error"] = "Error normalize_chars"
 
-
-                    #any parser should implement a language_iso2
+                    # any parserbot should implement a language_iso2
                     if "language_iso2" in hyperdata.keys():
-                        try:
-                            corpus.languages[hyperdata["language_iso2"]] +=1
-                        except KeyError:
-                            hyperdata["error"] = "Error: unsupported language"
-                            skipped_languages.append(hyperdata["language_iso2"])
+                        observed_languages[hyperdata["language_iso2"]] +=1
+
                     # this should be the responsability of the parserbot
                     # elif "language_iso3" in hyperdata.keys():
                     #     try:
@@ -67,22 +60,15 @@ def parse(corpus):
 
                     else:
                         print("[WARNING] no language_iso2 found in document [parsing.py]")
-                        #no language have been indexed
-                        #detectlang by index_fields
-
-                        text = " ".join([getattr(hyperdata, k) for k in DEFAULT_INDEX_FIELDS])
+                        # no language has been found by parserbot
+                        # => detectlang on index_fields
+                        text = " ".join([getattr(hyperdata, k, '') for k in DEFAULT_INDEX_FIELDS])
                         if len(text) < 10:
                             hyperdata["error"] = "Error: no TEXT fields to index"
-                            skipped_languages.append("__unknown__")
-
-                        hyperdata["language_iso2"] = detect_lang(text)
-                        try:
-                            corpus.languages[hyperdata["language_iso2"]] += 1
-                            corpus.languages[hyperdata["language_iso2"]] +=1
-                        except KeyError:
-                            hyperdata["error"] = "Error: unsupported language"
-                            skipped_languages.append(hyperdata["language_iso2"])
-
+                        else:
+                            predicted_lang = detect_lang(text)
+                            hyperdata["language_iso2"] = predicted_lang
+                            observed_languages[predicted_lang] += 1
 
                     # save as DB child
                     # ----------------
@@ -98,8 +84,10 @@ def parse(corpus):
                         document.status('Parsing', error= document.hyperdata["error"])
                         document.save_hyperdata()
                         session.commit()
-                        #adding skipped_docs for later processsing
-                        corpus.skipped_docs.append(document.id)
+
+                        # adding to skipped_docs for later processing
+                        skipped_docs[document.id] = True
+
                     documents_count += 1
 
                 # logging
@@ -113,18 +101,24 @@ def parse(corpus):
             corpus.hyperdata['resources'][i]['extracted'] = True
             corpus.save_hyperdata()
             session.commit()
-        # add a corpus-level info about languages adding a __skipped__ info
-        corpus.languages['__skipped__'] = Counter(skipped_languages)
+
+        print("PARSING:", len(skipped_docs), "docs skipped")
+
         print("LANGUES")
-        for n in corpus.languages.items():
+        for n in observed_languages.items():
             print(n)
-        #TO DO: give  the main language of the corpus to unsupported lang docs
-        print(len(corpus.skipped_docs), "docs skipped")
+
+        # add the infos to hyperdata at the end
+        corpus.hyperdata['skipped_docs'] = skipped_docs
+        corpus.hyperdata['languages'] = observed_languages
+        corpus.save_hyperdata()
+
         # commit all changes
         corpus.status('Docs', progress=documents_count, complete=True)
         corpus.save_hyperdata()
         session.add(corpus)
         session.commit()
+
     except Exception as error:
         corpus.status('Docs', error=error)
         corpus.save_hyperdata()