Store scraping 'resources' in CorpusNode

gargantext/datasource/base.py

@@ -67,7 +67,9 @@ class Scraper(Spider):
 
         self._total = None
         self.counter = 0
+        self.status = {"succeeded": 0, "failed": 0, "remaining": 0}
         self.events = []
+        self.events_history = []
 
         # For errors/events reporting
         self.http = urllib3.PoolManager()
@@ -149,6 +151,7 @@ class Scraper(Spider):
             "level": level,
             "message": message,
         })
+        self.events_history.extend(self.events)
 
     def report(self, path, succeeded=0, failed=0, remaining=None, events=None):
         # Silently fail if not bounded to a crawler or callback is undefined
@@ -157,14 +160,15 @@ class Scraper(Spider):
 
         assert path in ('output', 'error', 'event')
 
+        self.status.update(succeeded=succeeded,
+                           failed=failed,
+                           remaining=remaining)
+
         events = self.events + (events or [])
         self.events = []
 
         url = '%s/%s' % (self.callback, path)
-        status = dict(succeeded=succeeded,
-                      failed=failed,
-                      remaining=remaining,
-                      events=events)
+        status = dict(events=events, **self.status)
         data = { k: v for k, v in status.items() if v is not None }
 
         try:

gargantext/datasource/pipelines.py

@@ -9,6 +9,7 @@ import logging
 
 from gargantext.models.nodes import DocumentNode, CorpusNode
 from gargantext.core.db import Session
+from gargantext.utils.dates import datetime
 from gargantext.utils.lang import lang_detect
 
 
@@ -68,8 +69,20 @@ class DatabasePipeline(object):
         self.corpus = self.db.query(CorpusNode).filter_by(id=scraper.corpus).one_or_none()
 
     def close_spider(self, scraper):
-        self.corpus["resources"] = [{"type": 0}]
+        resources = self.corpus.hyperdata.get('resources', [])
+        resources.append({
+            "date": datetime.now(),
+            # TODO Raw files storage and listing in paths
+            "paths": None,
+            "scraper": scraper.name,
+            "query": scraper.query,
+            "status": scraper.status,
+            "events": scraper.events_history,
+        })
+
+        self.corpus['resources'] = resources
         self.corpus.save_hyperdata()
+
         self.db.commit()
 
     def process_item(self, item, scraper):