Commit 048f004a authored by Administrator's avatar Administrator

[COSMETICS] pls Samuel... :), what does it mean ?

parent 5fc5153d
...@@ -9,7 +9,6 @@ from gargantext_web.db import * ...@@ -9,7 +9,6 @@ from gargantext_web.db import *
from .parsers_config import parsers as _parsers from .parsers_config import parsers as _parsers
class DebugTime: class DebugTime:
def __init__(self, prefix): def __init__(self, prefix):
...@@ -29,7 +28,6 @@ class DebugTime: ...@@ -29,7 +28,6 @@ class DebugTime:
# keep all the parsers in a cache # keep all the parsers in a cache
class Parsers(defaultdict): class Parsers(defaultdict):
def __init__(self): def __init__(self):
self._parsers = _parsers self._parsers = _parsers
...@@ -44,9 +42,7 @@ class Parsers(defaultdict): ...@@ -44,9 +42,7 @@ class Parsers(defaultdict):
parsers = Parsers() parsers = Parsers()
# resources management # resources management
def add_resource(corpus, **kwargs): def add_resource(corpus, **kwargs):
# only for tests # only for tests
session = Session() session = Session()
...@@ -83,7 +79,6 @@ def add_resource(corpus, **kwargs): ...@@ -83,7 +79,6 @@ def add_resource(corpus, **kwargs):
# return result # return result
return resource return resource
def parse_resources(corpus, user=None, user_id=None): def parse_resources(corpus, user=None, user_id=None):
dbg = DebugTime('Corpus #%d - parsing' % corpus.id) dbg = DebugTime('Corpus #%d - parsing' % corpus.id)
session = Session() session = Session()
...@@ -102,7 +97,6 @@ def parse_resources(corpus, user=None, user_id=None): ...@@ -102,7 +97,6 @@ def parse_resources(corpus, user=None, user_id=None):
.filter(Node_Resource.parsed == False) .filter(Node_Resource.parsed == False)
) )
# make a new node for every parsed document of the corpus # make a new node for every parsed document of the corpus
print("HERE MOFOs")
print(resources_query) print(resources_query)
dbg.show('analyze documents') dbg.show('analyze documents')
nodes = list() nodes = list()
...@@ -147,13 +141,13 @@ def parse_resources(corpus, user=None, user_id=None): ...@@ -147,13 +141,13 @@ def parse_resources(corpus, user=None, user_id=None):
hyperdata.name: hyperdata hyperdata.name: hyperdata
for hyperdata in session.query(Hyperdata) for hyperdata in session.query(Hyperdata)
} }
print('hyperdata_types', hyperdata_types)
for node in nodes: for node in nodes:
node_id = node.id node_id = node.id
for hyperdata_key, hyperdata_value in node.hyperdata.items(): for hyperdata_key, hyperdata_value in node.hyperdata.items():
try: try:
hyperdata = hyperdata_types[hyperdata_key] hyperdata = hyperdata_types[hyperdata_key]
except KeyError: except KeyError:
# Why silent continue here ?
continue continue
if hyperdata.type == 'string': if hyperdata.type == 'string':
hyperdata_value = hyperdata_value[:255] hyperdata_value = hyperdata_value[:255]
...@@ -163,16 +157,17 @@ def parse_resources(corpus, user=None, user_id=None): ...@@ -163,16 +157,17 @@ def parse_resources(corpus, user=None, user_id=None):
hyperdata_value, hyperdata_value,
)) ))
print('I am here', node_hyperdata_lists.items())
for key, values in node_hyperdata_lists.items(): for key, values in node_hyperdata_lists.items():
print('here', key, values)
bulk_insert(Node_Hyperdata, ['node_id', 'hyperdata_id', 'value_'+key], values) bulk_insert(Node_Hyperdata, ['node_id', 'hyperdata_id', 'value_'+key], values)
# mark the corpus as parsed # mark the corpus as parsed
corpus.parsed = True corpus.parsed = True
# ngrams extraction # ngrams extraction
from .NgramsExtractors import EnglishNgramsExtractor, FrenchNgramsExtractor, NgramsExtractor from .NgramsExtractors import EnglishNgramsExtractor, FrenchNgramsExtractor, NgramsExtractor
class NgramsExtractors(defaultdict): class NgramsExtractors(defaultdict):
def __init__(self): def __init__(self):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment