[FIX] Bug #103: use timezone aware datetimes while parsing docs

gargantext/constants.py

@@ -36,7 +36,7 @@ import os
 import re
 import importlib
 from gargantext.util.lists import *
-from gargantext.util.tools import datetime, convert_to_date
+from gargantext.util import datetime, convert_to_datetime
 from .settings import BASE_DIR
 
 # types & models (nodes, lists, hyperdata, resource) ---------------------------------------------
@@ -108,9 +108,9 @@ INDEXED_HYPERDATA = {
 
     'publication_date':
         { 'id'             : 2
-        , 'type'           : datetime.datetime
-        , 'convert_to_db'  : convert_to_date
-        , 'convert_from_db': datetime.datetime.fromtimestamp
+        , 'type'           : datetime
+        , 'convert_to_db'  : convert_to_datetime
+        , 'convert_from_db': convert_to_datetime
         },
 
     'title':

gargantext/util/__init__.py

+from .dates import datetime, convert_to_datetime, MINYEAR

gargantext/util/dates.py

 import os
 from gargantext.settings import MEDIA_ROOT
 
-import datetime
-import dateutil
+from datetime import MINYEAR
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import datetime as _datetime, utc as UTC, now as utcnow
+
+__all__ = ['convert_to_datetime', 'datetime', 'MINYEAR']
+
+
+class datetime(_datetime):
+    @staticmethod
+    def now():
+        return utcnow()
+
+    @staticmethod
+    def utcfromtimestamp(ts):
+        return _datetime.utcfromtimestamp(ts).replace(tzinfo=UTC)
+
+    @staticmethod
+    def parse(s):
+        dt = parse_datetime(s)
+        return dt.astimezone(UTC) if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+def convert_to_datetime(dt):
+    if isinstance(dt, (int, float)):
+        return datetime.utcfromtimestamp(dt)
+
+    elif isinstance(dt, str):
+        return datetime.parse(dt)
+
+    elif isinstance(dt, _datetime):
+        args = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+        return datetime(*args, tzinfo=dt.tzinfo or UTC).astimezone(UTC)
 
-def convert_to_date(date):
-    if isinstance(date, (int, float)):
-        return datetime.datetime.timestamp(date)
     else:
-        return dateutil.parser.parse(date)
+        raise ValueError("Can't convert to datetime: %r" % dt)

gargantext/util/parsers/MULTIVAC.py

@@ -18,30 +18,30 @@ class MultivacParser(Parser):
         '''
         contents = filebuf.read().decode("UTF-8")
         data = json.loads(contents)
-        
+
         filebuf.close()
-        
+
         json_docs = data
         hyperdata_list = []
-        
+
         hyperdata_path = { "id"       : "id"
                          , "title"    : "title"
                          , "abstract" : "abstract"
                          , "type"     : "type"
                          }
-        
+
         for json_doc in json_docs:
 
             hyperdata = {}
-            
+
             doc = json_doc["_source"]
 
             for key, path in hyperdata_path.items():
                     hyperdata[key] = doc.get(path, "")
-            
+
             hyperdata["source"] = doc.get("serial"      , {})\
                                      .get("journaltitle", "REPEC Database")
-            
+
             try:
                 hyperdata["url"]    = doc.get("file", {})\
                                          .get("url" , "")
@@ -51,15 +51,15 @@ class MultivacParser(Parser):
             hyperdata["authors"] = ", ".join(
                                              [ p.get("person", {})
                                                 .get("name"  , "")
-                          
+
                                                for p in doc.get("hasauthor", [])
                                              ]
                                             )
-            
+
 
             year = doc.get("serial"  , {})\
                       .get("issuedate", None)
-            
+
             if year == "Invalide date":
                 year = doc.get("issuedate"  , None)
 
@@ -73,10 +73,7 @@ class MultivacParser(Parser):
                     date = datetime.now()
 
             hyperdata["publication_date"] = date
-            hyperdata["publication_year"]  = str(date.year)
-            hyperdata["publication_month"] = str(date.month)
-            hyperdata["publication_day"]   = str(date.day)
-            
+
             hyperdata_list.append(hyperdata)
-        
+
         return hyperdata_list

gargantext/util/parsers/_Parser.py

-import datetime
 import dateutil.parser
 import zipfile
 import re
 import dateparser as date_parser
 from gargantext.util.languages import languages
+from gargantext.util import datetime, convert_to_datetime, MINYEAR
 
 
-DEFAULT_DATE = datetime.datetime(datetime.MINYEAR, 1, 1)
+DEFAULT_DATE = datetime(MINYEAR, 1, 1)
 
 
 class Parser:
@@ -34,29 +34,29 @@ class Parser:
     def format_hyperdata_dates(self, hyperdata):
         """Format the dates found in the hyperdata.
         Examples:
-            {"publication_date": "2014-10-23 09:57:42"}
-            -> {"publication_date": "2014-10-23 09:57:42", "publication_year": "2014", ...}
+            {"publication_date": "2014-10-23 09:57:42+00:00"}
+            -> {"publication_date": "2014-10-23 09:57:42+00:00", "publication_year": "2014", ...}
             {"publication_year": "2014"}
-            -> {"publication_date": "2014-01-01 00:00:00", "publication_year": "2014", ...}
+            -> {"publication_date": "2014-01-01 00:00:00+00:00", "publication_year": "2014", ...}
         """
 
         # First, check the split dates...
         # This part mainly deal with Zotero data but can be usefull for others
         # parts
-        date_string = hyperdata.get('publication_date_to_parse', None)
+        date_string = hyperdata.get('publication_date_to_parse')
         if date_string is not None:
             date_string = re.sub(r'\/\/+(\w*|\d*)', '', date_string)
             try:
-                hyperdata['publication' + "_date"] = dateutil.parser.parse(
+                hyperdata['publication_date'] = dateutil.parser.parse(
                     date_string,
                     default=DEFAULT_DATE
-                ).strftime("%Y-%m-%d %H:%M:%S")
+                )
             except Exception as error:
                 print(error, 'Date not parsed for:', date_string)
-                hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                hyperdata['publication_date'] = datetime.now()
 
 
-        elif hyperdata.get('publication_year', None) is not None:
+        elif hyperdata.get('publication_year') is not None:
 
             prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_year"]
             # eg prefixes : ['publication']
@@ -64,56 +64,45 @@ class Parser:
             for prefix in prefixes:
                 date_string = hyperdata[prefix + "_year"]
 
-                # FIXME: except for year is it necessary to test that key exists
-                #        when we have a default value in .get(key, "01") ??
-                key = prefix + "_month"
-                if key in hyperdata:
-                    date_string += " " + hyperdata.get(key, "01")
-                    key = prefix + "_day"
-                    if key in hyperdata:
-                        date_string += " " + hyperdata.get(key, "01")
-                        key = prefix + "_hour"
-                        if key in hyperdata:
-                            date_string += " " + hyperdata.get(key, "01")
-                            key = prefix + "_minute"
-                            if key in hyperdata:
-                                date_string += ":" + hyperdata.get(key, "01")
-                                key = prefix + "_second"
-                                if key in hyperdata:
-                                    date_string += ":" + hyperdata.get(key, "01")
+                for part in ('month', 'day', 'hour', 'minute', 'second'):
+                    key = prefix + '_' + part
+                    if key not in hyperdata:
+                        break
+
+                    sep = ":" if key in ('minute', 'second') else " "
+                    date_string += sep + hyperdata.get(key, '01')
+
                 try:
-                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string).strftime("%Y-%m-%d %H:%M:%S")
+                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string)
                 except Exception as error:
                     try:
                         print("_Parser: error in full date parse", error, date_string)
                         # Date format:  1994 NOV-DEC
-                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8]).strftime("%Y-%m-%d %H:%M:%S")
+                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8])
 
                     except Exception as error:
                         try:
                             print("_Parser: error in short date parse", error)
                             # FIXME Date format:  1994 SPR
                             # By default, we take the year only
-                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4]).strftime("%Y-%m-%d %H:%M:%S")
+                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4])
 
                         except Exception as error:
                             print("_Parser:", error)
         else:
             print("WARNING: Date unknown at _Parser level, using now()")
-            hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            hyperdata['publication_date'] = datetime.now()
 
         # ...then parse all the "date" fields, to parse it into separate elements
         prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_date"]
         for prefix in prefixes:
-            date = dateutil.parser.parse(hyperdata[prefix + "_date"])
-            #print(date)
-
-            hyperdata[prefix + "_year"]      = date.strftime("%Y")
-            hyperdata[prefix + "_month"]     = date.strftime("%m")
-            hyperdata[prefix + "_day"]       = date.strftime("%d")
-            hyperdata[prefix + "_hour"]      = date.strftime("%H")
-            hyperdata[prefix + "_minute"]    = date.strftime("%M")
-            hyperdata[prefix + "_second"]    = date.strftime("%S")
+            name = prefix + "_date"
+            date = hyperdata[name]
+            hyperdata[name] = str(convert_to_datetime(date))
+
+            for part in ('year', 'month', 'day', 'hour', 'minute', 'second'):
+                hyperdata[prefix + '_' + part] = getattr(date, part)
+
         # print("line 116", hyperdata['publication_date'])
         # finally, return the transformed result!
         return hyperdata

gargantext/util/toolchain/hyperdata_indexing.py

@@ -43,8 +43,7 @@ def _nodes_hyperdata_generator(corpus):
                             key['id'],
                             None,
                             None,
-                            value.strftime("%Y-%m-%d %H:%M:%S"), 
-                            # FIXME check timestamp +%Z
+                            str(value),
                             None,
                             None,
                         )

gargantext/util/toolchain/ngram_coocs.py

@@ -9,7 +9,6 @@ from gargantext.util.db        import get_engine
 from gargantext.util.db_cache  import cache
 from gargantext.constants      import DEFAULT_COOC_THRESHOLD, NODETYPES
 from gargantext.constants      import INDEXED_HYPERDATA
-from gargantext.util.tools     import datetime, convert_to_date
 
 def compute_coocs(  corpus,
                     overwrite_id    = None,
@@ -95,7 +94,7 @@ def compute_coocs(  corpus,
 
     # 2b) stating the filters
     cooc_filter_sql = """
-        WHERE 
+        WHERE
             n.typename  = {nodetype_id}
         AND n.parent_id = {corpus_id}
         GROUP BY 1,2
@@ -105,7 +104,7 @@ def compute_coocs(  corpus,
         """.format( nodetype_id = NODETYPES.index('DOCUMENT')
                   , corpus_id=corpus.id
                   )
-    
+
     # 3) taking the cooccurrences of ngram x2
     ngram_filter_A_sql += """
         -- STEP 1: X axis of the matrix
@@ -162,25 +161,25 @@ def compute_coocs(  corpus,
     # 4) prepare the synonyms
     if groupings_id:
         ngram_filter_A_sql += """
-        LEFT JOIN  nodes_ngrams_ngrams 
-               AS grA  ON wlA.ngram_id = grA.ngram1_id 
+        LEFT JOIN  nodes_ngrams_ngrams
+               AS grA  ON wlA.ngram_id = grA.ngram1_id
                       AND grA.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
         LEFT JOIN  nodes_ngrams
                AS wlAA ON grA.ngram2_id = wlAA.ngram_id
-                      AND wlAA.node_id  = wlA.node_id 
+                      AND wlAA.node_id  = wlA.node_id
         -- \--> adding (joining) ngrams that are not grouped
         --LEFT JOIN  ngrams        AS wlAA ON grA.ngram2_id = wlAA.id
         -- \--> for joining all synonyms even if they are not in the main list (white list)
 
         """.format(groupings_id = groupings_id)
-        
+
         ngram_filter_B_sql += """
         LEFT JOIN  nodes_ngrams_ngrams
-               AS grB  ON wlB.ngram_id = grB.ngram1_id 
+               AS grB  ON wlB.ngram_id = grB.ngram1_id
                       AND grB.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
-        LEFT JOIN  nodes_ngrams 
+        LEFT JOIN  nodes_ngrams
                AS wlBB ON grB.ngram2_id = wlBB.ngram_id
                       AND wlBB.node_id   = wlB.node_id
         -- \--> adding (joining) ngrams that are not grouped