[CODE] Replaced lists with iterators in FileParser subclasses

https://forge.iscpif.fr/issues/1422

parsing/FileParsers/EuropressFileParser.py

@@ -48,11 +48,9 @@ class EuropressFileParser(FileParser):
                 print(error)
             
 
-        except:
-            return []
+        except Exception as error:
+            print(error)
 
-        # initialize the list of metadata
-        metadata_list = []
         # parse all the articles, one by one
         try:
             for html_article in html_articles:
@@ -201,16 +199,9 @@ class EuropressFileParser(FileParser):
                   #metadata_str = {}
                     for key, value in metadata.items():
                         metadata[key] = value.decode() if isinstance(value, bytes) else value
-                    metadata_list.append(metadata)
+                    yield metadata
                     count += 1
         
         except Exception as error:
             print(error)
             pass
-
-#       from pprint import pprint
-#       pprint(metadata_list)
-#       return []
-        return metadata_list
-
-

parsing/FileParsers/FileParser.py

@@ -109,9 +109,11 @@ class FileParser:
         # ...otherwise, let's parse it directly!
         else:
             try:
-                metadata_list += self._parse(file)
+                for metadata in self._parse(file):
+                    metadata_list.append(self.format_metadata(metadata))
             except Exception as error:
                 print(error)
         # return the list of formatted metadata
-        return map(self.format_metadata, metadata_list)
+        return metadata_list
+
 

parsing/FileParsers/PubmedFileParser.py

@@ -10,8 +10,6 @@ class PubmedFileParser(FileParser):
         xml_parser = etree.XMLParser(resolve_entities=False, recover=True)
         xml = etree.parse(file, parser=xml_parser)
         xml_articles = xml.findall('PubmedArticle')
-        # initialize the list of metadata
-        metadata_list = []
         # parse all the articles, one by one
         for xml_article in xml_articles:
             # extract data from the document
@@ -39,6 +37,4 @@ class PubmedFileParser(FileParser):
                         metadata[key] = xml_node.text
                 except:
                     pass
-            metadata_list.append(metadata)
-        # return the list of metadata
-        return metadata_list
+            yield metadata

parsing/FileParsers/RisFileParser.py

@@ -17,14 +17,12 @@ class RisFileParser(FileParser):
     }
 
     def _parse(self, file):
-        metadata_list = []
         metadata = {}
         last_key = None
         last_values = []
         for line in file:
             if len(line) > 2:
                 parameter_key = line[:2]
-#                print(parameter_key)
                 if parameter_key != b'  ' and parameter_key != last_key:
                     if last_key in self._parameters:
                         parameter = self._parameters[last_key]
@@ -32,17 +30,13 @@ class RisFileParser(FileParser):
                             separator = parameter["separator"] if "separator" in parameter else ""
                             metadata[parameter["key"]] = separator.join(last_values)
                         elif parameter["type"] == "delimiter":
-                            #language = self._languages_fullname[metadata["language"].lower()]
-                            #print(metadata)
                             try:
-                                #print("append")
                                 if 'language_fullname' not in metadata.keys():
                                     if 'language_iso3' not in metadata.keys():
                                         if 'language_iso2' not in metadata.keys():
                                             metadata['language_iso2'] = 'en'
-                                metadata_list.append(metadata)
+                                yield metadata
                                 metadata = {}
-                                #print("append succeeded")
                             except:
                                 pass
                     last_key = parameter_key
@@ -52,7 +46,3 @@ class RisFileParser(FileParser):
                 except Exception as error:
                     print(error)
                     pass
-        #print(len(metadata_list))
-        #print(metadata_list)
-
-        return metadata_list